From 621d2f01c3b990c9d86b7c160d71cb269b7e5b9b Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 16:22:27 -0800 Subject: [PATCH 01/27] feat: add scenario improvements and agent factory - Add Scenario.as_agent_tool() for creating tools that spawn fresh agents - Add Scenario.from_remote() for remote scenario handles via MCP - Add hud.scenario() module-level helper - Add create_agent() factory in hud/agents for programmatic agent creation - Resolve stash merge conflicts (taskset_id naming) --- hud/__init__.py | 51 +++++- hud/agents/__init__.py | 63 ++++++- hud/agents/base.py | 47 ++++- hud/datasets/loader.py | 2 +- hud/datasets/tests/test_loader.py | 6 +- hud/environment/__init__.py | 4 +- hud/environment/scenarios.py | 277 +++++++++++++++++++++++++----- hud/eval/task.py | 48 ++++++ 8 files changed, 448 insertions(+), 50 deletions(-) diff --git a/hud/__init__.py b/hud/__init__.py index 1fb747b1a..7c3dd3f48 100644 --- a/hud/__init__.py +++ b/hud/__init__.py @@ -9,7 +9,7 @@ # Apply patches to third-party libraries early, before other imports from . import patches as _patches # noqa: F401 -from .environment import Environment +from .environment import Environment, Scenario, ScenarioTool from .eval import EvalContext from .eval import run_eval as eval from .telemetry.instrument import instrument @@ -29,11 +29,60 @@ def trace(*args: object, **kwargs: object) -> EvalContext: return eval(*args, **kwargs) # type: ignore[arg-type] +def scenario( + env: Environment, + name: str, + *, + description: str | None = None, +) -> Scenario: + """Load a scenario from an environment (local or remote). + + This is a convenience function for creating Scenario handles, + especially for remote scenarios accessed via MCP. + + Args: + env: Environment where the scenario is defined. + name: Scenario name (with or without env prefix like "env:scenario"). + description: Optional description override. + + Returns: + Scenario object that can create Tasks or be converted to tools. + + Example: + ```python + import hud + + # Connect to remote environment + env = await hud.Environment.connect_hub("http://hub:8000") + + # Load scenario + checkout = hud.scenario(env, "checkout") + + # Create task from scenario + task = checkout(user="alice", product_id="123") + + # Or convert to a tool for sub-agent use + tool = checkout.as_agent_tool("claude") + ``` + """ + # Check if scenario is already registered locally + if hasattr(env, "get_scenario"): + local_scenario = env.get_scenario(name) + if local_scenario is not None: + return local_scenario + + # Otherwise, create a remote scenario handle + return Scenario.from_remote(env, name, description=description) + + __all__ = [ "Environment", "EvalContext", + "Scenario", + "ScenarioTool", "eval", "instrument", + "scenario", "trace", # Deprecated alias for eval ] diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 547d876b8..ce01cb5b4 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -1,19 +1,80 @@ from __future__ import annotations -from .base import MCPAgent +from typing import TYPE_CHECKING, Any + +from .base import MCPAgent, Rollout from .openai import OpenAIAgent from .openai_chat import OpenAIChatAgent from .operator import OperatorAgent +if TYPE_CHECKING: + from hud.types import AgentType + # Note: These agents are not exported here to avoid requiring optional dependencies. # Import directly if needed: # from hud.agents.claude import ClaudeAgent # requires anthropic # from hud.agents.gemini import GeminiAgent # requires google-genai # from hud.agents.gemini_cua import GeminiCUAAgent # requires google-genai + +def create_agent( + agent_type: str | AgentType, + **kwargs: Any, +) -> MCPAgent: + """Create an agent from a type string or AgentType enum. + + This is the recommended factory for creating agents programmatically. + The agent type maps to a specific agent class via AgentType.cls. + + Args: + agent_type: Agent type ("claude", "openai", "gemini", etc.) or AgentType enum. + **kwargs: Parameters passed to the agent's create() method. + Common params: model, max_tokens, temperature, system_prompt. + + Returns: + Configured MCPAgent instance ready to use with hud.eval(). + + Example: + ```python + from hud.agents import create_agent + + # Create Claude agent + agent = create_agent("claude", model="claude-sonnet-4-5") + + # Create OpenAI agent + agent = create_agent("openai", model="gpt-4o") + + # Use with hud.eval() + async with hud.eval(task) as ctx: + await agent.run(ctx) + ``` + + Supported agent types: + - "claude": ClaudeAgent (Anthropic Claude) + - "openai": OpenAIAgent (OpenAI with responses API) + - "operator": OperatorAgent (OpenAI Computer Use) + - "gemini": GeminiAgent (Google Gemini) + - "gemini_cua": GeminiCUAAgent (Gemini Computer Use) + - "openai_compatible": OpenAIChatAgent (OpenAI-compatible endpoints) + """ + from hud.types import AgentType as AT + + # Normalize to AgentType enum + if isinstance(agent_type, str): + agent_type_enum = AT(agent_type) + else: + agent_type_enum = agent_type + + # Get agent class and create instance + agent_cls = agent_type_enum.cls + return agent_cls.create(**kwargs) + + __all__ = [ "MCPAgent", "OpenAIAgent", "OpenAIChatAgent", "OperatorAgent", + "Rollout", + "create_agent", ] diff --git a/hud/agents/base.py b/hud/agents/base.py index 8c3c04d5b..fe6182cf2 100644 --- a/hud/agents/base.py +++ b/hud/agents/base.py @@ -6,7 +6,7 @@ import json import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, ClassVar, Literal +from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, runtime_checkable import mcp.types as types from pydantic import BaseModel, ConfigDict @@ -21,6 +21,49 @@ logger = logging.getLogger(__name__) +__all__ = ["BaseCreateParams", "MCPAgent", "Rollout"] + + +# ============================================================================= +# Rollout Protocol +# ============================================================================= + + +@runtime_checkable +class Rollout(Protocol): + """Protocol for agent rollouts - any execution strategy. + + A Rollout takes an EvalContext and runs an agent loop, + returning a result (typically a Trace). + + MCPAgent inherits from Rollout, so any MCPAgent can be used + wherever a Rollout is expected. + + Custom rollouts can implement this protocol for non-MCPAgent + execution strategies (e.g., LangGraph, custom loops). + + Example: + class MyCustomRollout(Rollout): + async def run(self, ctx: EvalContext) -> Trace: + # Custom agent logic + return Trace(content="result", done=True) + + # Use with scenario.as_tool() + tool = investigate.as_tool(MyCustomRollout()) + """ + + async def run(self, ctx: Any, **kwargs: Any) -> Any: + """Run the agent loop on the given context. + + Args: + ctx: EvalContext containing prompt and tools + **kwargs: Additional arguments (e.g., max_steps) + + Returns: + Trace or result object with content + """ + ... + class BaseCreateParams(BaseModel): """Runtime parameters for agent creation.""" @@ -34,7 +77,7 @@ class BaseCreateParams(BaseModel): verbose: bool = False -class MCPAgent(ABC): +class MCPAgent(Rollout, ABC): """ Base class for MCP-enabled agents. diff --git a/hud/datasets/loader.py b/hud/datasets/loader.py index 0c1982f29..0957870bd 100644 --- a/hud/datasets/loader.py +++ b/hud/datasets/loader.py @@ -303,7 +303,7 @@ def save_tasks( ) response.raise_for_status() data = response.json() - taskset_id = data.get("evalset_id") or data.get("id") or name + taskset_id = data.get("taskset_id") or data.get("evalset_id") or data.get("id") or name logger.info("Saved %d tasks to taskset: %s", len(tasks), taskset_id) return taskset_id except httpx.HTTPStatusError as e: diff --git a/hud/datasets/tests/test_loader.py b/hud/datasets/tests/test_loader.py index 5c6658709..ef8c9cd89 100644 --- a/hud/datasets/tests/test_loader.py +++ b/hud/datasets/tests/test_loader.py @@ -22,10 +22,10 @@ def test_load_tasks_success( mock_settings.api_key = "test_key" mock_response = MagicMock() - # EvalsetTasksResponse format: tasks keyed by task ID + # TasksetTasksResponse format: tasks keyed by task ID mock_response.json.return_value = { - "evalset_id": "evalset-123", - "evalset_name": "test-dataset", + "taskset_id": "taskset-123", + "taskset_name": "test-dataset", "tasks": { "task-1": { "env": {"name": "test"}, diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index 9aad37a0d..b7dab8cff 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -28,7 +28,7 @@ from hud.environment.environment import Environment from hud.environment.mock import MockMixin, generate_mock_value from hud.environment.router import ConflictResolution, ToolRouter -from hud.environment.scenarios import ScenarioMixin +from hud.environment.scenarios import Scenario, ScenarioMixin, ScenarioTool from hud.environment.types import EnvConfig from hud.environment.utils import ToolFormat, format_result, parse_tool_call, parse_tool_calls @@ -40,7 +40,9 @@ "EnvConfig", "Environment", "MockMixin", + "Scenario", "ScenarioMixin", + "ScenarioTool", "ToolFormat", "ToolRouter", "format_result", diff --git a/hud/environment/scenarios.py b/hud/environment/scenarios.py index 566422d35..6d025c281 100644 --- a/hud/environment/scenarios.py +++ b/hud/environment/scenarios.py @@ -1,4 +1,4 @@ -"""Scenario decorator for Environment - defines setup/evaluate phases.""" +"""Scenario decorator and classes for Environment setup/evaluate phases.""" from __future__ import annotations @@ -8,6 +8,11 @@ import uuid from typing import TYPE_CHECKING, Any, get_type_hints +from fastmcp.tools.tool import ToolResult +from mcp.types import TextContent + +from hud.tools.base import BaseTool + if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable @@ -15,35 +20,211 @@ from fastmcp.resources import ResourceManager from fastmcp.tools import ToolManager -__all__ = ["ScenarioMixin"] + from hud.agents.base import Rollout + from hud.environment import Environment + from hud.types import AgentType + +__all__ = ["Scenario", "ScenarioMixin", "ScenarioTool"] logger = logging.getLogger(__name__) -class ScenarioMixin: - """Mixin providing @env.scenario decorator for setup/evaluate phases. +class Scenario: + """Scenario template created by @env.scenario() - can create Tasks or Tools. - Scenarios are async generators that yield twice: - - First yield: prompt string (setup phase) - - Second yield: reward float (evaluate phase) + Example: + @env.scenario() + async def investigate(issue_id: str): + yield f"Investigate {issue_id}" + yield 1.0 - The scenario can receive the agent's answer via yield: - answer = yield "Do the task" - yield 1.0 if "success" in answer else 0.0 + task = investigate(issue_id="123") # Create Task + tool = investigate.as_tool(my_agent) # Create Tool + """ - The answer is passed via the hud_submit tool or ctx.submit(). + def __init__( + self, + fn: Callable[..., AsyncGenerator[Any, Any]], + env: Environment, + name: str, + description: str | None = None, + ) -> None: + self.fn = fn + self.env = env + self.name = name + self.description = description or fn.__doc__ or f"Scenario: {name}" + self._signature = inspect.signature(fn) + + @property + def input_schema(self) -> dict[str, Any]: + """JSON schema for the scenario's parameters.""" + properties: dict[str, Any] = {} + required: list[str] = [] + + for pname, param in self._signature.parameters.items(): + prop: dict[str, Any] = {} + if param.annotation is not inspect.Parameter.empty: + try: + from pydantic import TypeAdapter + param_schema = TypeAdapter(param.annotation).json_schema() + if "type" in param_schema: + prop["type"] = param_schema["type"] + elif "$ref" in param_schema or "anyOf" in param_schema: + prop = param_schema # Complex type - store full schema + else: + prop["type"] = "string" + except Exception: + prop["type"] = "string" + else: + prop["type"] = "string" - The decorator registers both an MCP prompt and resource with the same - identifier ({env_name}:{scenario_name}), linked by session state. + properties[pname] = prop + if param.default is inspect.Parameter.empty: + required.append(pname) - Example: - @env.scenario() - async def search_cats(url: str): - await env.call_tool("navigate", url=url) - answer = yield "Find all cat images on the page" - result = await env.call_tool("count_cats") - yield float(result > 0 or "found" in answer.lower()) - """ + return {"type": "object", "properties": properties, "required": required} + + def __call__(self, **kwargs: Any) -> Any: + """Create a Task from this scenario.""" + from hud.eval.task import Task + return Task(env=self.env, scenario=self.name, args=kwargs) + + def as_tool( + self, + rollout: Rollout, + *, + name: str | None = None, + trace: bool = False, + ) -> ScenarioTool: + """Convert to a tool backed by an agent/rollout.""" + return ScenarioTool(scenario=self, rollout=rollout, name=name, trace=trace) + + def as_agent_tool( + self, + agent_type: str | AgentType, + *, + agent_params: dict[str, Any] | None = None, + name: str | None = None, + trace: bool = False, + ) -> ScenarioTool: + """Convert to a tool that spawns a fresh agent for each call. + + This is a convenience over as_tool() when you want the agent created + automatically from a model type rather than passing an existing rollout. + + Args: + agent_type: Agent type string ("claude", "openai", etc.) or AgentType enum. + agent_params: Parameters passed to agent.create() (model, max_tokens, etc.). + name: Override tool name (defaults to scenario name). + trace: Whether to trace the sub-agent's execution. + + Returns: + ScenarioTool that creates a fresh agent per invocation. + + Example: + tool = investigate.as_agent_tool( + "claude", + agent_params={"model": "claude-sonnet-4-5"}, + ) + """ + from hud.agents import create_agent + + # Create a rollout wrapper that creates a fresh agent per call + class AgentRollout: + def __init__( + self, + agent_type: str | AgentType, + agent_params: dict[str, Any] | None, + ) -> None: + self._agent_type = agent_type + self._agent_params = agent_params or {} + + async def run(self, ctx: Any, **kwargs: Any) -> Any: + agent = create_agent(self._agent_type, **self._agent_params) + return await agent.run(ctx, **kwargs) + + rollout = AgentRollout(agent_type, agent_params) + return ScenarioTool(scenario=self, rollout=rollout, name=name, trace=trace) + + @classmethod + def from_remote( + cls, + env: Environment, + scenario_name: str, + *, + description: str | None = None, + ) -> Scenario: + """Create a Scenario handle for a remote scenario (via MCP). + + Use this when the scenario is defined on a remote environment (hub) + rather than locally. The scenario runs via MCP prompt/resource calls. + + Args: + env: Environment connected to the hub where scenario is defined. + scenario_name: Name of the scenario (with or without env prefix). + description: Optional description override. + + Returns: + Scenario object that can create Tasks or be converted to tools. + + Example: + env = await Environment.connect_hub("http://hub:8000") + scenario = Scenario.from_remote(env, "checkout") + task = scenario(user="alice") # Creates Task for remote execution + """ + # Remote scenarios don't have a local function + # Create a placeholder that works with Task creation + async def _remote_placeholder(**kwargs: Any) -> AsyncGenerator[Any, Any]: + # This generator is never actually called for remote scenarios + # Task execution goes through MCP prompt/resource + raise RuntimeError( + f"Scenario '{scenario_name}' is remote - " + "use Task execution, not direct generator call" + ) + yield # Make it a generator + + return cls( + fn=_remote_placeholder, + env=env, + name=scenario_name, + description=description or f"Remote scenario: {scenario_name}", + ) + + +class ScenarioTool(BaseTool): + """Tool wrapping a Scenario + Rollout for hierarchical agent patterns.""" + + def __init__( + self, + *, + scenario: Scenario, + rollout: Rollout, + name: str | None = None, + trace: bool = False, + ) -> None: + self._scenario = scenario + self._rollout = rollout + self._trace = trace + super().__init__( + name=name or scenario.name, + description=scenario.description, + meta={"input_schema": scenario.input_schema}, + ) + + async def __call__(self, **kwargs: Any) -> ToolResult: + """Execute scenario with rollout and return answer.""" + from hud.eval.manager import run_eval + from hud.eval.task import Task + + task = Task(env=self._scenario.env, scenario=self._scenario.name, args=kwargs) + async with run_eval(task, trace=self._trace) as ctx: + result = await self._rollout.run(ctx) + content = result.content if hasattr(result, "content") and result.content else "" + return ToolResult(content=[TextContent(type="text", text=content)]) + + +class ScenarioMixin: + """Mixin providing @env.scenario decorator for setup/evaluate phases.""" # These come from Environment/MCPServer name: str @@ -52,7 +233,7 @@ async def search_cats(url: str): _tool_manager: ToolManager # Scenario state - _scenarios: dict[str, Callable[..., AsyncGenerator[Any, Any]]] + _scenarios: dict[str, Scenario] # scenario_name -> Scenario object _scenario_sessions: dict[str, AsyncGenerator[Any, Any]] # session_id -> generator _scenario_latest: dict[str, str] # scenario_name -> latest session_id _scenario_answers: dict[str, str] # scenario_name -> submitted answer @@ -148,8 +329,8 @@ async def run_scenario_setup(self, scenario_name: str, args: dict[str, Any]) -> # Check if scenario is registered locally if scenario_name in self._scenarios: # Local scenario - run setup via generator - scenario_fn = self._scenarios[scenario_name] - gen = scenario_fn(**args) + scenario_obj = self._scenarios[scenario_name] + gen = scenario_obj.fn(**args) # Run setup phase (code before first yield) prompt = await gen.__anext__() @@ -298,13 +479,14 @@ def scenario( self, name: str | None = None, description: str | None = None, - ) -> Callable[ - [Callable[..., AsyncGenerator[Any, None]]], - Callable[..., AsyncGenerator[Any, None]], - ]: + ) -> Callable[[Callable[..., AsyncGenerator[Any, None]]], Scenario]: """Decorator to register a scenario with setup and evaluate phases. - Creates both a prompt and resource with identifier scenario:{name}. + Returns a Scenario object that can be: + - Called with args to create a Task: scenario(issue_id="123") + - Converted to a tool: scenario.as_tool(agent) + + Creates both a prompt and resource with identifier {env_name}:{name}. The scenario function should yield twice: - First yield: the prompt string (returned from prompt) - Second yield: the reward float (returned from resource) @@ -321,15 +503,14 @@ async def search_cats(url: str): result = await env.call_tool("count_cats") yield float(result > 0) - # MCP client usage: - # 1. get_prompt("{env_name}:search_cats", {url: "..."}) -> prompt messages - # 2. agent runs... - # 3. read_resource("{env_name}:search_cats") -> {"reward": 0.95} + # Create Task for evaluation + task = search_cats(url="https://example.com") + + # Create Tool for subagent orchestration + tool = search_cats.as_tool(my_agent) """ - def decorator( - fn: Callable[..., AsyncGenerator[Any, None]], - ) -> Callable[..., AsyncGenerator[Any, None]]: + def decorator(fn: Callable[..., AsyncGenerator[Any, None]]) -> Scenario: scenario_name = name or fn.__name__ # Sanitize env name for URI scheme (no underscores allowed) safe_env_name = self.name.replace("_", "-") @@ -347,11 +528,14 @@ def decorator( ) source_code = None - # Store the generator function - self._scenarios[scenario_name] = fn + # Create Scenario object + scenario_obj = Scenario( + fn=fn, env=self, name=scenario_name, description=scenario_desc # type: ignore[arg-type] + ) + self._scenarios[scenario_name] = scenario_obj # Get function signature for prompt arguments with type info - sig = inspect.signature(fn) + sig = scenario_obj._signature prompt_args: list[dict[str, Any]] = [] for p in sig.parameters.values(): is_required = p.default is inspect.Parameter.empty @@ -390,7 +574,7 @@ def decorator( # Register PROMPT - runs setup, returns prompt messages # We need a reference to self and the outer variables scenario_self = self - scenario_fn = fn + scenario_fn = scenario_obj.fn scenario_name_ref = scenario_name # Resolve parameter type hints for deserialization @@ -537,6 +721,17 @@ async def resource_handler() -> str: scenario_id, ) - return fn + return scenario_obj return decorator + + def get_scenario(self, name: str) -> Scenario | None: + """Get a Scenario object by name. + + Args: + name: Name of the scenario + + Returns: + Scenario object or None if not found + """ + return self._scenarios.get(name) diff --git a/hud/eval/task.py b/hud/eval/task.py index 085f1bf86..c7b0aa8f2 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -338,3 +338,51 @@ def copy(self) -> Task: args=self.args.copy() if self.args else {}, validation=self.validation.copy() if self.validation else None, ) + + async def run( + self, + rollout: Any, + *, + trace: bool = True, + max_steps: int = 10, + ) -> str: + """Run this task with a rollout/agent and return the answer. + + This is a convenience method for quick task execution. + For full control over the evaluation context, use hud.eval(task) instead. + + Args: + rollout: Rollout or MCPAgent to execute the task. + Must have a run(ctx) method that takes EvalContext. + trace: Whether to send trace data to backend (default True) + max_steps: Maximum agent steps (passed to rollout if supported) + + Returns: + The agent's final answer as a string + + Example: + task = investigate(issue_id="123") + result = await task.run(OpenAIChatAgent.create(model="gpt-4o")) + + # Equivalent to: + async with hud.eval(task) as ctx: + result = await agent.run(ctx) + """ + from hud.eval.manager import run_eval + + async with run_eval(self, trace=trace) as ctx: + # Call rollout.run(ctx) - works for both MCPAgent and Rollout protocol + # MCPAgent.run accepts max_steps, but we check if supported + try: + result = await rollout.run(ctx, max_steps=max_steps) + except TypeError: + # Rollout doesn't accept max_steps + result = await rollout.run(ctx) + + # Extract content from result + if hasattr(result, "content"): + return result.content if result.content else "" + elif isinstance(result, str): + return result + else: + return str(result) if result else "" \ No newline at end of file From d70d2b061755e8f491fbda18ae552f2a1585df60 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 16:49:28 -0800 Subject: [PATCH 02/27] scenario as tool simplification --- hud/__init__.py | 51 +------ hud/agents/__init__.py | 37 +---- hud/environment/__init__.py | 4 +- hud/environment/scenarios.py | 277 ++++++----------------------------- hud/eval/context.py | 10 ++ hud/eval/task.py | 7 +- hud/tools/__init__.py | 2 + hud/tools/agent.py | 95 ++++++++++++ 8 files changed, 156 insertions(+), 327 deletions(-) create mode 100644 hud/tools/agent.py diff --git a/hud/__init__.py b/hud/__init__.py index 7c3dd3f48..1fb747b1a 100644 --- a/hud/__init__.py +++ b/hud/__init__.py @@ -9,7 +9,7 @@ # Apply patches to third-party libraries early, before other imports from . import patches as _patches # noqa: F401 -from .environment import Environment, Scenario, ScenarioTool +from .environment import Environment from .eval import EvalContext from .eval import run_eval as eval from .telemetry.instrument import instrument @@ -29,60 +29,11 @@ def trace(*args: object, **kwargs: object) -> EvalContext: return eval(*args, **kwargs) # type: ignore[arg-type] -def scenario( - env: Environment, - name: str, - *, - description: str | None = None, -) -> Scenario: - """Load a scenario from an environment (local or remote). - - This is a convenience function for creating Scenario handles, - especially for remote scenarios accessed via MCP. - - Args: - env: Environment where the scenario is defined. - name: Scenario name (with or without env prefix like "env:scenario"). - description: Optional description override. - - Returns: - Scenario object that can create Tasks or be converted to tools. - - Example: - ```python - import hud - - # Connect to remote environment - env = await hud.Environment.connect_hub("http://hub:8000") - - # Load scenario - checkout = hud.scenario(env, "checkout") - - # Create task from scenario - task = checkout(user="alice", product_id="123") - - # Or convert to a tool for sub-agent use - tool = checkout.as_agent_tool("claude") - ``` - """ - # Check if scenario is already registered locally - if hasattr(env, "get_scenario"): - local_scenario = env.get_scenario(name) - if local_scenario is not None: - return local_scenario - - # Otherwise, create a remote scenario handle - return Scenario.from_remote(env, name, description=description) - - __all__ = [ "Environment", "EvalContext", - "Scenario", - "ScenarioTool", "eval", "instrument", - "scenario", "trace", # Deprecated alias for eval ] diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index ce01cb5b4..e495b2f1c 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -21,42 +21,7 @@ def create_agent( agent_type: str | AgentType, **kwargs: Any, ) -> MCPAgent: - """Create an agent from a type string or AgentType enum. - - This is the recommended factory for creating agents programmatically. - The agent type maps to a specific agent class via AgentType.cls. - - Args: - agent_type: Agent type ("claude", "openai", "gemini", etc.) or AgentType enum. - **kwargs: Parameters passed to the agent's create() method. - Common params: model, max_tokens, temperature, system_prompt. - - Returns: - Configured MCPAgent instance ready to use with hud.eval(). - - Example: - ```python - from hud.agents import create_agent - - # Create Claude agent - agent = create_agent("claude", model="claude-sonnet-4-5") - - # Create OpenAI agent - agent = create_agent("openai", model="gpt-4o") - - # Use with hud.eval() - async with hud.eval(task) as ctx: - await agent.run(ctx) - ``` - - Supported agent types: - - "claude": ClaudeAgent (Anthropic Claude) - - "openai": OpenAIAgent (OpenAI with responses API) - - "operator": OperatorAgent (OpenAI Computer Use) - - "gemini": GeminiAgent (Google Gemini) - - "gemini_cua": GeminiCUAAgent (Gemini Computer Use) - - "openai_compatible": OpenAIChatAgent (OpenAI-compatible endpoints) - """ + """Create an agent from a type string or AgentType enum.""" from hud.types import AgentType as AT # Normalize to AgentType enum diff --git a/hud/environment/__init__.py b/hud/environment/__init__.py index b7dab8cff..9aad37a0d 100644 --- a/hud/environment/__init__.py +++ b/hud/environment/__init__.py @@ -28,7 +28,7 @@ from hud.environment.environment import Environment from hud.environment.mock import MockMixin, generate_mock_value from hud.environment.router import ConflictResolution, ToolRouter -from hud.environment.scenarios import Scenario, ScenarioMixin, ScenarioTool +from hud.environment.scenarios import ScenarioMixin from hud.environment.types import EnvConfig from hud.environment.utils import ToolFormat, format_result, parse_tool_call, parse_tool_calls @@ -40,9 +40,7 @@ "EnvConfig", "Environment", "MockMixin", - "Scenario", "ScenarioMixin", - "ScenarioTool", "ToolFormat", "ToolRouter", "format_result", diff --git a/hud/environment/scenarios.py b/hud/environment/scenarios.py index 6d025c281..566422d35 100644 --- a/hud/environment/scenarios.py +++ b/hud/environment/scenarios.py @@ -1,4 +1,4 @@ -"""Scenario decorator and classes for Environment setup/evaluate phases.""" +"""Scenario decorator for Environment - defines setup/evaluate phases.""" from __future__ import annotations @@ -8,11 +8,6 @@ import uuid from typing import TYPE_CHECKING, Any, get_type_hints -from fastmcp.tools.tool import ToolResult -from mcp.types import TextContent - -from hud.tools.base import BaseTool - if TYPE_CHECKING: from collections.abc import AsyncGenerator, Callable @@ -20,211 +15,35 @@ from fastmcp.resources import ResourceManager from fastmcp.tools import ToolManager - from hud.agents.base import Rollout - from hud.environment import Environment - from hud.types import AgentType - -__all__ = ["Scenario", "ScenarioMixin", "ScenarioTool"] +__all__ = ["ScenarioMixin"] logger = logging.getLogger(__name__) -class Scenario: - """Scenario template created by @env.scenario() - can create Tasks or Tools. - - Example: - @env.scenario() - async def investigate(issue_id: str): - yield f"Investigate {issue_id}" - yield 1.0 - - task = investigate(issue_id="123") # Create Task - tool = investigate.as_tool(my_agent) # Create Tool - """ - - def __init__( - self, - fn: Callable[..., AsyncGenerator[Any, Any]], - env: Environment, - name: str, - description: str | None = None, - ) -> None: - self.fn = fn - self.env = env - self.name = name - self.description = description or fn.__doc__ or f"Scenario: {name}" - self._signature = inspect.signature(fn) - - @property - def input_schema(self) -> dict[str, Any]: - """JSON schema for the scenario's parameters.""" - properties: dict[str, Any] = {} - required: list[str] = [] - - for pname, param in self._signature.parameters.items(): - prop: dict[str, Any] = {} - if param.annotation is not inspect.Parameter.empty: - try: - from pydantic import TypeAdapter - param_schema = TypeAdapter(param.annotation).json_schema() - if "type" in param_schema: - prop["type"] = param_schema["type"] - elif "$ref" in param_schema or "anyOf" in param_schema: - prop = param_schema # Complex type - store full schema - else: - prop["type"] = "string" - except Exception: - prop["type"] = "string" - else: - prop["type"] = "string" - - properties[pname] = prop - if param.default is inspect.Parameter.empty: - required.append(pname) - - return {"type": "object", "properties": properties, "required": required} - - def __call__(self, **kwargs: Any) -> Any: - """Create a Task from this scenario.""" - from hud.eval.task import Task - return Task(env=self.env, scenario=self.name, args=kwargs) - - def as_tool( - self, - rollout: Rollout, - *, - name: str | None = None, - trace: bool = False, - ) -> ScenarioTool: - """Convert to a tool backed by an agent/rollout.""" - return ScenarioTool(scenario=self, rollout=rollout, name=name, trace=trace) - - def as_agent_tool( - self, - agent_type: str | AgentType, - *, - agent_params: dict[str, Any] | None = None, - name: str | None = None, - trace: bool = False, - ) -> ScenarioTool: - """Convert to a tool that spawns a fresh agent for each call. - - This is a convenience over as_tool() when you want the agent created - automatically from a model type rather than passing an existing rollout. - - Args: - agent_type: Agent type string ("claude", "openai", etc.) or AgentType enum. - agent_params: Parameters passed to agent.create() (model, max_tokens, etc.). - name: Override tool name (defaults to scenario name). - trace: Whether to trace the sub-agent's execution. - - Returns: - ScenarioTool that creates a fresh agent per invocation. - - Example: - tool = investigate.as_agent_tool( - "claude", - agent_params={"model": "claude-sonnet-4-5"}, - ) - """ - from hud.agents import create_agent - - # Create a rollout wrapper that creates a fresh agent per call - class AgentRollout: - def __init__( - self, - agent_type: str | AgentType, - agent_params: dict[str, Any] | None, - ) -> None: - self._agent_type = agent_type - self._agent_params = agent_params or {} - - async def run(self, ctx: Any, **kwargs: Any) -> Any: - agent = create_agent(self._agent_type, **self._agent_params) - return await agent.run(ctx, **kwargs) - - rollout = AgentRollout(agent_type, agent_params) - return ScenarioTool(scenario=self, rollout=rollout, name=name, trace=trace) - - @classmethod - def from_remote( - cls, - env: Environment, - scenario_name: str, - *, - description: str | None = None, - ) -> Scenario: - """Create a Scenario handle for a remote scenario (via MCP). - - Use this when the scenario is defined on a remote environment (hub) - rather than locally. The scenario runs via MCP prompt/resource calls. - - Args: - env: Environment connected to the hub where scenario is defined. - scenario_name: Name of the scenario (with or without env prefix). - description: Optional description override. - - Returns: - Scenario object that can create Tasks or be converted to tools. - - Example: - env = await Environment.connect_hub("http://hub:8000") - scenario = Scenario.from_remote(env, "checkout") - task = scenario(user="alice") # Creates Task for remote execution - """ - # Remote scenarios don't have a local function - # Create a placeholder that works with Task creation - async def _remote_placeholder(**kwargs: Any) -> AsyncGenerator[Any, Any]: - # This generator is never actually called for remote scenarios - # Task execution goes through MCP prompt/resource - raise RuntimeError( - f"Scenario '{scenario_name}' is remote - " - "use Task execution, not direct generator call" - ) - yield # Make it a generator - - return cls( - fn=_remote_placeholder, - env=env, - name=scenario_name, - description=description or f"Remote scenario: {scenario_name}", - ) - - -class ScenarioTool(BaseTool): - """Tool wrapping a Scenario + Rollout for hierarchical agent patterns.""" +class ScenarioMixin: + """Mixin providing @env.scenario decorator for setup/evaluate phases. - def __init__( - self, - *, - scenario: Scenario, - rollout: Rollout, - name: str | None = None, - trace: bool = False, - ) -> None: - self._scenario = scenario - self._rollout = rollout - self._trace = trace - super().__init__( - name=name or scenario.name, - description=scenario.description, - meta={"input_schema": scenario.input_schema}, - ) + Scenarios are async generators that yield twice: + - First yield: prompt string (setup phase) + - Second yield: reward float (evaluate phase) - async def __call__(self, **kwargs: Any) -> ToolResult: - """Execute scenario with rollout and return answer.""" - from hud.eval.manager import run_eval - from hud.eval.task import Task + The scenario can receive the agent's answer via yield: + answer = yield "Do the task" + yield 1.0 if "success" in answer else 0.0 - task = Task(env=self._scenario.env, scenario=self._scenario.name, args=kwargs) - async with run_eval(task, trace=self._trace) as ctx: - result = await self._rollout.run(ctx) - content = result.content if hasattr(result, "content") and result.content else "" - return ToolResult(content=[TextContent(type="text", text=content)]) + The answer is passed via the hud_submit tool or ctx.submit(). + The decorator registers both an MCP prompt and resource with the same + identifier ({env_name}:{scenario_name}), linked by session state. -class ScenarioMixin: - """Mixin providing @env.scenario decorator for setup/evaluate phases.""" + Example: + @env.scenario() + async def search_cats(url: str): + await env.call_tool("navigate", url=url) + answer = yield "Find all cat images on the page" + result = await env.call_tool("count_cats") + yield float(result > 0 or "found" in answer.lower()) + """ # These come from Environment/MCPServer name: str @@ -233,7 +52,7 @@ class ScenarioMixin: _tool_manager: ToolManager # Scenario state - _scenarios: dict[str, Scenario] # scenario_name -> Scenario object + _scenarios: dict[str, Callable[..., AsyncGenerator[Any, Any]]] _scenario_sessions: dict[str, AsyncGenerator[Any, Any]] # session_id -> generator _scenario_latest: dict[str, str] # scenario_name -> latest session_id _scenario_answers: dict[str, str] # scenario_name -> submitted answer @@ -329,8 +148,8 @@ async def run_scenario_setup(self, scenario_name: str, args: dict[str, Any]) -> # Check if scenario is registered locally if scenario_name in self._scenarios: # Local scenario - run setup via generator - scenario_obj = self._scenarios[scenario_name] - gen = scenario_obj.fn(**args) + scenario_fn = self._scenarios[scenario_name] + gen = scenario_fn(**args) # Run setup phase (code before first yield) prompt = await gen.__anext__() @@ -479,14 +298,13 @@ def scenario( self, name: str | None = None, description: str | None = None, - ) -> Callable[[Callable[..., AsyncGenerator[Any, None]]], Scenario]: + ) -> Callable[ + [Callable[..., AsyncGenerator[Any, None]]], + Callable[..., AsyncGenerator[Any, None]], + ]: """Decorator to register a scenario with setup and evaluate phases. - Returns a Scenario object that can be: - - Called with args to create a Task: scenario(issue_id="123") - - Converted to a tool: scenario.as_tool(agent) - - Creates both a prompt and resource with identifier {env_name}:{name}. + Creates both a prompt and resource with identifier scenario:{name}. The scenario function should yield twice: - First yield: the prompt string (returned from prompt) - Second yield: the reward float (returned from resource) @@ -503,14 +321,15 @@ async def search_cats(url: str): result = await env.call_tool("count_cats") yield float(result > 0) - # Create Task for evaluation - task = search_cats(url="https://example.com") - - # Create Tool for subagent orchestration - tool = search_cats.as_tool(my_agent) + # MCP client usage: + # 1. get_prompt("{env_name}:search_cats", {url: "..."}) -> prompt messages + # 2. agent runs... + # 3. read_resource("{env_name}:search_cats") -> {"reward": 0.95} """ - def decorator(fn: Callable[..., AsyncGenerator[Any, None]]) -> Scenario: + def decorator( + fn: Callable[..., AsyncGenerator[Any, None]], + ) -> Callable[..., AsyncGenerator[Any, None]]: scenario_name = name or fn.__name__ # Sanitize env name for URI scheme (no underscores allowed) safe_env_name = self.name.replace("_", "-") @@ -528,14 +347,11 @@ def decorator(fn: Callable[..., AsyncGenerator[Any, None]]) -> Scenario: ) source_code = None - # Create Scenario object - scenario_obj = Scenario( - fn=fn, env=self, name=scenario_name, description=scenario_desc # type: ignore[arg-type] - ) - self._scenarios[scenario_name] = scenario_obj + # Store the generator function + self._scenarios[scenario_name] = fn # Get function signature for prompt arguments with type info - sig = scenario_obj._signature + sig = inspect.signature(fn) prompt_args: list[dict[str, Any]] = [] for p in sig.parameters.values(): is_required = p.default is inspect.Parameter.empty @@ -574,7 +390,7 @@ def decorator(fn: Callable[..., AsyncGenerator[Any, None]]) -> Scenario: # Register PROMPT - runs setup, returns prompt messages # We need a reference to self and the outer variables scenario_self = self - scenario_fn = scenario_obj.fn + scenario_fn = fn scenario_name_ref = scenario_name # Resolve parameter type hints for deserialization @@ -721,17 +537,6 @@ async def resource_handler() -> str: scenario_id, ) - return scenario_obj + return fn return decorator - - def get_scenario(self, name: str) -> Scenario | None: - """Get a Scenario object by name. - - Args: - name: Name of the scenario - - Returns: - Scenario object or None if not found - """ - return self._scenarios.get(name) diff --git a/hud/eval/context.py b/hud/eval/context.py index ca0704f52..ad129d3b5 100644 --- a/hud/eval/context.py +++ b/hud/eval/context.py @@ -302,10 +302,20 @@ def from_task( code_snippet: Code being evaluated trace: Whether to send traces to backend quiet: Whether to suppress output + + Raises: + ValueError: If task.args is None (template tasks cannot be run directly) """ from hud.environment import Environment from hud.eval.task import build_eval_name + # Validate that task has args (not a template) + if task.args is None: + raise ValueError( + f"Cannot run task with args=None (this is a template). " + f"Provide args when creating the task: env('{task.scenario}', **args)" + ) + eval_name = name or build_eval_name(task.scenario, task.args) # task.env is guaranteed to be Environment after Task.__post_init__ diff --git a/hud/eval/task.py b/hud/eval/task.py index c7b0aa8f2..e322ec2c6 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -148,7 +148,10 @@ class Task(BaseModel): env: Any = Field(default=None) # Typed as Any for input flexibility, validated below scenario: str | None = None id: str | None = None - args: dict[str, Any] = Field(default_factory=dict) + args: dict[str, Any] | None = Field( + default=None, + description="Scenario arguments. None indicates a template (args filled in later).", + ) validation: list[MCPToolCall] | None = None # Agent config - settings passed to agent (system_prompt, etc.) @@ -385,4 +388,4 @@ async def run( elif isinstance(result, str): return result else: - return str(result) if result else "" \ No newline at end of file + return str(result) if result else "" diff --git a/hud/tools/__init__.py b/hud/tools/__init__.py index 8451a04f3..26495d332 100644 --- a/hud/tools/__init__.py +++ b/hud/tools/__init__.py @@ -4,6 +4,7 @@ from typing import TYPE_CHECKING, Any +from .agent import AgentTool from .base import BaseHub, BaseTool from .bash import BashTool from .edit import EditTool @@ -21,6 +22,7 @@ ) __all__ = [ + "AgentTool", "AnthropicComputerTool", "BaseHub", "BaseTool", diff --git a/hud/tools/agent.py b/hud/tools/agent.py new file mode 100644 index 000000000..36dcce3f6 --- /dev/null +++ b/hud/tools/agent.py @@ -0,0 +1,95 @@ +"""AgentTool - run a Task with an agent as a tool.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +from fastmcp.tools.tool import ToolResult +from mcp.types import TextContent + +from hud.tools.base import BaseTool + +if TYPE_CHECKING: + from hud.eval.task import Task + from hud.types import AgentType + +__all__ = ["AgentTool"] + + +class AgentTool(BaseTool): + """Tool that runs a Task template with an agent. + + Takes a Task as a template (typically with empty args) and runs it + with a fresh agent when called. Call-time kwargs are merged into + the task's args. + + Works for both local scenarios (defined with @env.scenario) and + remote scenarios (from connected hubs). + + Example: + ```python + # Create task template + template = env("checkout") # Task with args={} + + # Wrap in AgentTool + tool = AgentTool(template, "claude", agent_params={"model": "claude-sonnet-4-5"}) + + # Call with args - spawns fresh agent + result = await tool(user="alice") + ``` + """ + + def __init__( + self, + task: Task, + agent_type: str | AgentType, + agent_params: dict[str, Any] | None = None, + *, + name: str | None = None, + description: str | None = None, + trace: bool = False, + ) -> None: + """Create an AgentTool. + + Args: + task: Task template (scenario + env, typically with empty args). + agent_type: Agent type ("claude", "openai", etc.) or AgentType enum. + agent_params: Parameters passed to agent.create(). + name: Override tool name (defaults to scenario name). + description: Override tool description. + trace: Whether to trace the sub-agent's execution. + """ + self._task = task + self._agent_type = agent_type + self._agent_params = agent_params or {} + self._trace = trace + + tool_name = name or task.scenario or "agent_tool" + tool_desc = description or f"Run scenario: {task.scenario}" + + super().__init__(name=tool_name, description=tool_desc) + + async def __call__(self, **kwargs: Any) -> ToolResult: + """Execute the task with a fresh agent. + + Args: + **kwargs: Arguments merged into the template's args. + + Returns: + ToolResult with the agent's response content. + """ + from hud.agents import create_agent + from hud.eval.manager import run_eval + + # Merge call kwargs with template args (None means empty template) + base_args = self._task.args if self._task.args is not None else {} + merged_args = {**base_args, **kwargs} + task = self._task.model_copy(update={"args": merged_args}) + + # Run with fresh agent + async with run_eval(task, trace=self._trace) as ctx: + agent = create_agent(self._agent_type, **self._agent_params) + result = await agent.run(ctx) + content = result.content if hasattr(result, "content") and result.content else "" + return ToolResult(content=[TextContent(type="text", text=content)]) + From b2de659b2964e133225f716109a42d48393fbe71 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 17:54:20 -0800 Subject: [PATCH 03/27] change agent resolution for easier model switching --- hud/agents/__init__.py | 75 +++++++++++++++++++++++++++--------------- hud/agents/base.py | 47 ++------------------------ hud/agents/gateway.py | 43 ++++++++++++++++++++++++ hud/agents/resolver.py | 70 +++++++++++++++++++++++++++++++++++++++ hud/cli/eval.py | 54 ++++++++++-------------------- hud/datasets/runner.py | 19 +++++------ hud/eval/task.py | 50 +--------------------------- hud/tools/agent.py | 52 +++++++++++++++++++++-------- 8 files changed, 228 insertions(+), 182 deletions(-) create mode 100644 hud/agents/gateway.py create mode 100644 hud/agents/resolver.py diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index e495b2f1c..1b5deb1e2 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -2,44 +2,65 @@ from typing import TYPE_CHECKING, Any -from .base import MCPAgent, Rollout +from .base import MCPAgent from .openai import OpenAIAgent from .openai_chat import OpenAIChatAgent from .operator import OperatorAgent +from .resolver import resolve_cls if TYPE_CHECKING: from hud.types import AgentType -# Note: These agents are not exported here to avoid requiring optional dependencies. -# Import directly if needed: -# from hud.agents.claude import ClaudeAgent # requires anthropic -# from hud.agents.gemini import GeminiAgent # requires google-genai -# from hud.agents.gemini_cua import GeminiCUAAgent # requires google-genai - - -def create_agent( - agent_type: str | AgentType, - **kwargs: Any, -) -> MCPAgent: - """Create an agent from a type string or AgentType enum.""" - from hud.types import AgentType as AT - - # Normalize to AgentType enum - if isinstance(agent_type, str): - agent_type_enum = AT(agent_type) - else: - agent_type_enum = agent_type - - # Get agent class and create instance - agent_cls = agent_type_enum.cls - return agent_cls.create(**kwargs) - - __all__ = [ "MCPAgent", "OpenAIAgent", "OpenAIChatAgent", "OperatorAgent", - "Rollout", "create_agent", + "resolve_cls", ] + + +def create_agent(model: str | AgentType, **kwargs: Any) -> MCPAgent: + """Create an agent from a model string or AgentType. + + Args: + model: AgentType ("claude"), or gateway model name ("gpt-4o"). + **kwargs: Params passed to agent.create(). + + Example: + ```python + agent = create_agent("claude", model="claude-sonnet-4-5") + agent = create_agent("gpt-4o") # auto-configures gateway + ``` + """ + from hud.types import AgentType as AT + + # AgentType enum → just create + if isinstance(model, AT): + return model.cls.create(**kwargs) + + # Resolve class and optional gateway info + agent_cls, gateway_info = resolve_cls(model) + + # If not a gateway model, just create + if gateway_info is None: + return agent_cls.create(**kwargs) + + # Build gateway params + model_id = gateway_info.get("model") or gateway_info.get("id") or model + kwargs.setdefault("model", model_id) + kwargs.setdefault("validate_api_key", False) + + # Build model_client based on provider + if "model_client" not in kwargs and "openai_client" not in kwargs: + from hud.agents.gateway import build_gateway_client + + provider = gateway_info.get("provider", "openai_compatible") + client = build_gateway_client(provider) + + # OpenAIChatAgent uses openai_client key, others use model_client + key = "openai_client" if agent_cls == OpenAIChatAgent else "model_client" + kwargs[key] = client + + return agent_cls.create(**kwargs) diff --git a/hud/agents/base.py b/hud/agents/base.py index fe6182cf2..8c3c04d5b 100644 --- a/hud/agents/base.py +++ b/hud/agents/base.py @@ -6,7 +6,7 @@ import json import logging from abc import ABC, abstractmethod -from typing import TYPE_CHECKING, Any, ClassVar, Literal, Protocol, runtime_checkable +from typing import TYPE_CHECKING, Any, ClassVar, Literal import mcp.types as types from pydantic import BaseModel, ConfigDict @@ -21,49 +21,6 @@ logger = logging.getLogger(__name__) -__all__ = ["BaseCreateParams", "MCPAgent", "Rollout"] - - -# ============================================================================= -# Rollout Protocol -# ============================================================================= - - -@runtime_checkable -class Rollout(Protocol): - """Protocol for agent rollouts - any execution strategy. - - A Rollout takes an EvalContext and runs an agent loop, - returning a result (typically a Trace). - - MCPAgent inherits from Rollout, so any MCPAgent can be used - wherever a Rollout is expected. - - Custom rollouts can implement this protocol for non-MCPAgent - execution strategies (e.g., LangGraph, custom loops). - - Example: - class MyCustomRollout(Rollout): - async def run(self, ctx: EvalContext) -> Trace: - # Custom agent logic - return Trace(content="result", done=True) - - # Use with scenario.as_tool() - tool = investigate.as_tool(MyCustomRollout()) - """ - - async def run(self, ctx: Any, **kwargs: Any) -> Any: - """Run the agent loop on the given context. - - Args: - ctx: EvalContext containing prompt and tools - **kwargs: Additional arguments (e.g., max_steps) - - Returns: - Trace or result object with content - """ - ... - class BaseCreateParams(BaseModel): """Runtime parameters for agent creation.""" @@ -77,7 +34,7 @@ class BaseCreateParams(BaseModel): verbose: bool = False -class MCPAgent(Rollout, ABC): +class MCPAgent(ABC): """ Base class for MCP-enabled agents. diff --git a/hud/agents/gateway.py b/hud/agents/gateway.py new file mode 100644 index 000000000..1249df893 --- /dev/null +++ b/hud/agents/gateway.py @@ -0,0 +1,43 @@ +"""Gateway client utilities for HUD inference gateway.""" + +from __future__ import annotations + +from typing import Any + + +def build_gateway_client(provider: str) -> Any: + """Build a client configured for HUD gateway routing. + + Args: + provider: Provider name ("anthropic", "openai", "gemini", etc.) + + Returns: + Configured async client for the provider. + """ + from hud.settings import settings + + provider = provider.lower() + + if provider == "anthropic": + from anthropic import AsyncAnthropic + + return AsyncAnthropic(api_key=settings.api_key, base_url=settings.hud_gateway_url) + + if provider == "gemini": + from google import genai + from google.genai.types import HttpOptions + + return genai.Client( + api_key="PLACEHOLDER", + http_options=HttpOptions( + api_version="v1beta", + base_url=settings.hud_gateway_url, + headers={"Authorization": f"Bearer {settings.api_key}"}, + ), + ) + + # OpenAI-compatible (openai, azure, together, groq, fireworks, etc.) + from openai import AsyncOpenAI + + return AsyncOpenAI(api_key=settings.api_key, base_url=settings.hud_gateway_url) + diff --git a/hud/agents/resolver.py b/hud/agents/resolver.py new file mode 100644 index 000000000..b2d01d7cd --- /dev/null +++ b/hud/agents/resolver.py @@ -0,0 +1,70 @@ +"""Model resolution - maps model strings to agent classes.""" + +from __future__ import annotations + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from hud.agents.base import MCPAgent + +__all__ = ["resolve_cls"] + +_models_cache: list[dict[str, Any]] | None = None + +# Provider name → AgentType value (only anthropic differs) +_PROVIDER_TO_AGENT = {"anthropic": "claude"} + + +def _fetch_gateway_models() -> list[dict[str, Any]]: + """Fetch available models from HUD gateway (cached).""" + global _models_cache + if _models_cache is not None: + return _models_cache + + import httpx + + from hud.settings import settings + + if not settings.api_key: + return [] + + try: + resp = httpx.get( + f"{settings.hud_gateway_url}/models", + headers={"Authorization": f"Bearer {settings.api_key}"}, + timeout=10.0, + ) + resp.raise_for_status() + data = resp.json() + _models_cache = data.get("data", data) if isinstance(data, dict) else data + return _models_cache or [] + except Exception: + return [] + + +def resolve_cls(model: str) -> tuple[type[MCPAgent], dict[str, Any] | None]: + """Resolve model string to (agent_class, gateway_info). + + Returns: + (agent_class, None) for known AgentTypes + (agent_class, gateway_model_info) for gateway models + """ + from hud.types import AgentType + + # Known AgentType → no gateway info + try: + return AgentType(model).cls, None + except ValueError: + pass + + # Gateway lookup + for m in _fetch_gateway_models(): + if model in (m.get("id"), m.get("name"), m.get("model")): + provider = m.get("provider", "openai_compatible").lower() + agent_str = _PROVIDER_TO_AGENT.get(provider, provider) + try: + return AgentType(agent_str).cls, m + except ValueError: + return AgentType.OPENAI_COMPATIBLE.cls, m + + raise ValueError(f"Model '{model}' not found") diff --git a/hud/cli/eval.py b/hud/cli/eval.py index eb13ce34f..faedb107d 100644 --- a/hud/cli/eval.py +++ b/hud/cli/eval.py @@ -338,47 +338,27 @@ def get_agent_kwargs(self) -> dict[str, Any]: # Configure gateway mode - route LLM API calls through HUD gateway if self.gateway: - hud_api_key = settings.api_key - if not hud_api_key: + if not settings.api_key: raise typer.Exit(1) # Already validated in validate_api_keys() - if self.agent_type == AgentType.CLAUDE: - from anthropic import AsyncAnthropic - - kwargs["model_client"] = AsyncAnthropic( - api_key=hud_api_key, - base_url=settings.hud_gateway_url, - ) - hud_console.info("🌐 Using HUD Gateway for Claude API") - elif self.agent_type in (AgentType.OPENAI, AgentType.OPERATOR): - from openai import AsyncOpenAI + from hud.agents.gateway import build_gateway_client - kwargs["model_client"] = AsyncOpenAI( - api_key=hud_api_key, - base_url=settings.hud_gateway_url, - ) - hud_console.info("🌐 Using HUD Gateway for OpenAI API") - elif self.agent_type == AgentType.OPENAI_COMPATIBLE: - from openai import AsyncOpenAI + # Map AgentType to provider + agent_to_provider = { + AgentType.CLAUDE: "anthropic", + AgentType.OPENAI: "openai", + AgentType.OPERATOR: "openai", + AgentType.GEMINI: "gemini", + AgentType.GEMINI_CUA: "gemini", + AgentType.OPENAI_COMPATIBLE: "openai", + } + provider = agent_to_provider.get(self.agent_type, "openai") + client = build_gateway_client(provider) - kwargs["openai_client"] = AsyncOpenAI( - api_key=hud_api_key, - base_url=settings.hud_gateway_url, - ) - hud_console.info("🌐 Using HUD Gateway for OpenAI-compatible API") - elif self.agent_type in (AgentType.GEMINI, AgentType.GEMINI_CUA): - from google import genai - from google.genai.types import HttpOptions - - kwargs["model_client"] = genai.Client( - api_key="PLACEHOLDER", - http_options=HttpOptions( - api_version="v1beta", - base_url=settings.hud_gateway_url, - headers={"Authorization": f"Bearer {hud_api_key}"}, - ), - ) - hud_console.info("🌐 Using HUD Gateway for Gemini API") + # OpenAI-compatible uses openai_client key + is_oai_compat = self.agent_type == AgentType.OPENAI_COMPATIBLE + kwargs["openai_client" if is_oai_compat else "model_client"] = client + hud_console.info(f"🌐 Using HUD Gateway for {provider} API") return kwargs diff --git a/hud/datasets/runner.py b/hud/datasets/runner.py index 3b4b11629..acb79fb95 100644 --- a/hud/datasets/runner.py +++ b/hud/datasets/runner.py @@ -86,10 +86,6 @@ async def run_dataset( if not task_list: raise ValueError("No tasks to run") - # Resolve agent class - agent_type_enum = agent_type if isinstance(agent_type, AgentType) else AgentType(agent_type) - agent_cls = agent_type_enum.cls - # Use hud.eval() for both single and parallel execution async with hud.eval( task_list, @@ -97,8 +93,10 @@ async def run_dataset( max_concurrent=max_concurrent, quiet=quiet, ) as ctx: - # Create agent fresh for each context (ensures correct tool initialization) - agent = agent_cls.create(**(agent_params or {})) + # Create agent (handles AgentType, gateway models, etc.) + from hud.agents import create_agent + + agent = create_agent(agent_type, **(agent_params or {})) await agent.run(ctx, max_steps=max_steps) # Reward is computed by EvalContext.__aexit__ from evaluate tools @@ -112,7 +110,7 @@ async def run_dataset( async def run_single_task( task: Task, *, - agent_type: AgentType, + agent_type: str | AgentType, agent_params: dict[str, Any] | None = None, max_steps: int = 10, job_id: str | None = None, @@ -198,9 +196,10 @@ async def run_single_task( if ctx.system_prompt and "system_prompt" not in final_agent_params: final_agent_params["system_prompt"] = ctx.system_prompt - # Create agent inside ctx so it has access to context-derived values - agent_cls = agent_type.cls - agent = agent_cls.create(**final_agent_params) + # Create agent (handles AgentType, gateway models, etc.) + from hud.agents import create_agent + + agent = create_agent(agent_type, **final_agent_params) # Store metadata if provided if metadata: diff --git a/hud/eval/task.py b/hud/eval/task.py index e322ec2c6..ea621a9ab 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -338,54 +338,6 @@ def copy(self) -> Task: id=self.id, env=self.env, # Share reference scenario=self.scenario, - args=self.args.copy() if self.args else {}, + args=self.args.copy() if self.args else None, validation=self.validation.copy() if self.validation else None, ) - - async def run( - self, - rollout: Any, - *, - trace: bool = True, - max_steps: int = 10, - ) -> str: - """Run this task with a rollout/agent and return the answer. - - This is a convenience method for quick task execution. - For full control over the evaluation context, use hud.eval(task) instead. - - Args: - rollout: Rollout or MCPAgent to execute the task. - Must have a run(ctx) method that takes EvalContext. - trace: Whether to send trace data to backend (default True) - max_steps: Maximum agent steps (passed to rollout if supported) - - Returns: - The agent's final answer as a string - - Example: - task = investigate(issue_id="123") - result = await task.run(OpenAIChatAgent.create(model="gpt-4o")) - - # Equivalent to: - async with hud.eval(task) as ctx: - result = await agent.run(ctx) - """ - from hud.eval.manager import run_eval - - async with run_eval(self, trace=trace) as ctx: - # Call rollout.run(ctx) - works for both MCPAgent and Rollout protocol - # MCPAgent.run accepts max_steps, but we check if supported - try: - result = await rollout.run(ctx, max_steps=max_steps) - except TypeError: - # Rollout doesn't accept max_steps - result = await rollout.run(ctx) - - # Extract content from result - if hasattr(result, "content"): - return result.content if result.content else "" - elif isinstance(result, str): - return result - else: - return str(result) if result else "" diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 36dcce3f6..479dee3a0 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -10,8 +10,8 @@ from hud.tools.base import BaseTool if TYPE_CHECKING: + from hud.agents.base import MCPAgent from hud.eval.task import Task - from hud.types import AgentType __all__ = ["AgentTool"] @@ -19,7 +19,7 @@ class AgentTool(BaseTool): """Tool that runs a Task template with an agent. - Takes a Task as a template (typically with empty args) and runs it + Takes a Task as a template (typically with args=None) and runs it with a fresh agent when called. Call-time kwargs are merged into the task's args. @@ -28,11 +28,16 @@ class AgentTool(BaseTool): Example: ```python + from hud.tools import AgentTool + # Create task template - template = env("checkout") # Task with args={} + template = env("checkout") # Task with args=None + + # Option 1: Use built-in agent type + tool = AgentTool(template, model="claude") - # Wrap in AgentTool - tool = AgentTool(template, "claude", agent_params={"model": "claude-sonnet-4-5"}) + # Option 2: Use custom agent class + tool = AgentTool(template, agent=MyCustomAgent) # Call with args - spawns fresh agent result = await tool(user="alice") @@ -42,9 +47,10 @@ class AgentTool(BaseTool): def __init__( self, task: Task, - agent_type: str | AgentType, - agent_params: dict[str, Any] | None = None, *, + model: str | None = None, + agent: type[MCPAgent] | None = None, + agent_params: dict[str, Any] | None = None, name: str | None = None, description: str | None = None, trace: bool = False, @@ -52,15 +58,27 @@ def __init__( """Create an AgentTool. Args: - task: Task template (scenario + env, typically with empty args). - agent_type: Agent type ("claude", "openai", etc.) or AgentType enum. - agent_params: Parameters passed to agent.create(). + task: Task template (scenario + env, typically with args=None). + model: Agent type string ("claude", "openai", "gemini", etc.). + Uses the same resolution as hud eval CLI. + agent: Custom agent class (must have .create() method). + Use this for custom agent implementations. + agent_params: Parameters passed to agent.create() (model name, etc.). name: Override tool name (defaults to scenario name). description: Override tool description. trace: Whether to trace the sub-agent's execution. + + Note: + Must provide either `model` or `agent`, not both. """ + if model is None and agent is None: + raise ValueError("Must provide either 'model' or 'agent'") + if model is not None and agent is not None: + raise ValueError("Cannot provide both 'model' and 'agent'") + self._task = task - self._agent_type = agent_type + self._model = model + self._agent_cls = agent self._agent_params = agent_params or {} self._trace = trace @@ -78,7 +96,6 @@ async def __call__(self, **kwargs: Any) -> ToolResult: Returns: ToolResult with the agent's response content. """ - from hud.agents import create_agent from hud.eval.manager import run_eval # Merge call kwargs with template args (None means empty template) @@ -88,8 +105,15 @@ async def __call__(self, **kwargs: Any) -> ToolResult: # Run with fresh agent async with run_eval(task, trace=self._trace) as ctx: - agent = create_agent(self._agent_type, **self._agent_params) + # Create agent from model string or custom class + if self._model is not None: + from hud.agents import create_agent + + agent = create_agent(self._model, **self._agent_params) + else: + # Custom agent class - call .create() directly + agent = self._agent_cls.create(**self._agent_params) # type: ignore[union-attr] + result = await agent.run(ctx) content = result.content if hasattr(result, "content") and result.content else "" return ToolResult(content=[TextContent(type="text", text=content)]) - From 32b31183cce2f0c4d584cd75df2524edf7777b42 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:17:17 -0800 Subject: [PATCH 04/27] agent tool does not get optional params (eval params) --- hud/tools/agent.py | 136 +++++++++++++++++++++++++++------------------ 1 file changed, 83 insertions(+), 53 deletions(-) diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 479dee3a0..01b224a2c 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -2,7 +2,8 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +import inspect +from typing import TYPE_CHECKING, Any, get_args, get_origin from fastmcp.tools.tool import ToolResult from mcp.types import TextContent @@ -16,31 +17,33 @@ __all__ = ["AgentTool"] +def _is_eval_only(param: inspect.Parameter) -> bool: + """Check if param is eval-only: has None default AND None in type union.""" + if param.default is not None: + return False + if param.annotation is inspect.Parameter.empty: + return False + origin = get_origin(param.annotation) + if origin is not None: + return type(None) in get_args(param.annotation) + return False + + class AgentTool(BaseTool): """Tool that runs a Task template with an agent. - Takes a Task as a template (typically with args=None) and runs it - with a fresh agent when called. Call-time kwargs are merged into - the task's args. - - Works for both local scenarios (defined with @env.scenario) and - remote scenarios (from connected hubs). + Parameters with `| None = None` are eval-only and hidden from the tool schema. Example: ```python - from hud.tools import AgentTool - - # Create task template - template = env("checkout") # Task with args=None - - # Option 1: Use built-in agent type - tool = AgentTool(template, model="claude") - - # Option 2: Use custom agent class - tool = AgentTool(template, agent=MyCustomAgent) - - # Call with args - spawns fresh agent - result = await tool(user="alice") + @env.scenario() + async def investigate( + issue_id: str, # Required - orchestrator sees + expected_cause: str | None = None # Eval only - hidden + ): + yield {"task": f"Investigate {issue_id}"} + + seer = AgentTool(env("investigate"), model="ft:seer-v2") ``` """ @@ -55,22 +58,6 @@ def __init__( description: str | None = None, trace: bool = False, ) -> None: - """Create an AgentTool. - - Args: - task: Task template (scenario + env, typically with args=None). - model: Agent type string ("claude", "openai", "gemini", etc.). - Uses the same resolution as hud eval CLI. - agent: Custom agent class (must have .create() method). - Use this for custom agent implementations. - agent_params: Parameters passed to agent.create() (model name, etc.). - name: Override tool name (defaults to scenario name). - description: Override tool description. - trace: Whether to trace the sub-agent's execution. - - Note: - Must provide either `model` or `agent`, not both. - """ if model is None and agent is None: raise ValueError("Must provide either 'model' or 'agent'") if model is not None and agent is not None: @@ -82,37 +69,80 @@ def __init__( self._agent_params = agent_params or {} self._trace = trace + # Get visible params from scenario function + self._visible_params: set[str] = set() + self._param_schema: dict[str, Any] | None = None + + if task.env and task.scenario: + scenario_fn = task.env._scenarios.get(task.scenario) + if scenario_fn: + sig = inspect.signature(scenario_fn) + visible = { + name: p for name, p in sig.parameters.items() + if not _is_eval_only(p) + } + self._visible_params = set(visible.keys()) + self._param_schema = self._build_schema(visible) + tool_name = name or task.scenario or "agent_tool" tool_desc = description or f"Run scenario: {task.scenario}" super().__init__(name=tool_name, description=tool_desc) - async def __call__(self, **kwargs: Any) -> ToolResult: - """Execute the task with a fresh agent. + def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]: + """Build JSON schema using Pydantic TypeAdapter.""" + from pydantic import TypeAdapter - Args: - **kwargs: Arguments merged into the template's args. + properties: dict[str, Any] = {} + required: list[str] = [] - Returns: - ToolResult with the agent's response content. - """ + for name, param in params.items(): + if param.annotation is not inspect.Parameter.empty: + try: + adapter = TypeAdapter(param.annotation) + properties[name] = adapter.json_schema() + except Exception: + properties[name] = {"type": "string"} + else: + properties[name] = {"type": "string"} + + if param.default is inspect.Parameter.empty: + required.append(name) + elif param.default is not None: + properties[name]["default"] = param.default + + return {"type": "object", "properties": properties, "required": required} + + @property + def mcp(self) -> Any: + """Get as FastMCP FunctionTool with filtered schema.""" + if not hasattr(self, "_mcp_tool"): + from fastmcp.tools import FunctionTool + + self._mcp_tool = FunctionTool.from_function( + self, name=self.name, description=self.description + ) + if self._param_schema: + self._mcp_tool.parameters = self._param_schema + return self._mcp_tool + + async def __call__(self, **kwargs: Any) -> ToolResult: + """Execute the task with a fresh agent.""" from hud.eval.manager import run_eval - # Merge call kwargs with template args (None means empty template) - base_args = self._task.args if self._task.args is not None else {} - merged_args = {**base_args, **kwargs} - task = self._task.model_copy(update={"args": merged_args}) + # Filter to visible params only + filtered = {k: v for k, v in kwargs.items() if k in self._visible_params} + + # Merge with template args + base_args = self._task.args or {} + task = self._task.model_copy(update={"args": {**base_args, **filtered}}) - # Run with fresh agent async with run_eval(task, trace=self._trace) as ctx: - # Create agent from model string or custom class - if self._model is not None: + if self._model: from hud.agents import create_agent - agent = create_agent(self._model, **self._agent_params) else: - # Custom agent class - call .create() directly - agent = self._agent_cls.create(**self._agent_params) # type: ignore[union-attr] + agent = self._agent_cls.create(**self._agent_params) # type: ignore result = await agent.run(ctx) content = result.content if hasattr(result, "content") and result.content else "" From 8f9f2ba00cde1573c63789c4fd6f4bcf6200f1aa Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:17:51 -0800 Subject: [PATCH 05/27] fix tests --- hud/datasets/loader.py | 9 ++++++--- hud/eval/tests/test_eval.py | 2 +- hud/telemetry/tests/test_eval_telemetry.py | 16 ++++++++-------- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/hud/datasets/loader.py b/hud/datasets/loader.py index 0957870bd..969bf75cd 100644 --- a/hud/datasets/loader.py +++ b/hud/datasets/loader.py @@ -63,7 +63,8 @@ def _load_from_file(path: Path) -> list[Task]: from hud.eval.task import Task raw_items = _load_raw_from_file(path) - return [Task(**item) for item in raw_items] + # Default args to {} for runnable tasks (None = template) + return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items] def _load_raw_from_huggingface(dataset_name: str) -> list[dict[str, Any]]: @@ -99,7 +100,8 @@ def _load_from_huggingface(dataset_name: str) -> list[Task]: raw_items = _load_raw_from_huggingface(dataset_name) from hud.eval.task import Task - return [Task(**item) for item in raw_items] + # Default args to {} for runnable tasks (None = template) + return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items] def _load_raw_from_api(dataset_name: str) -> list[dict[str, Any]]: @@ -138,7 +140,8 @@ def _load_from_api(dataset_name: str) -> list[Task]: from hud.eval.task import Task raw_items = _load_raw_from_api(dataset_name) - return [Task(**item) for item in raw_items] + # Default args to {} for runnable tasks (None = template) + return [Task(**{**item, "args": item.get("args") or {}}) for item in raw_items] @overload diff --git a/hud/eval/tests/test_eval.py b/hud/eval/tests/test_eval.py index 6d4708089..ea958af49 100644 --- a/hud/eval/tests/test_eval.py +++ b/hud/eval/tests/test_eval.py @@ -16,7 +16,7 @@ def test_init_defaults(self) -> None: assert task.env is None assert task.scenario is None - assert task.args == {} + assert task.args is None # None = template, {} = runnable with no args def test_init_with_env_dict(self) -> None: """Task auto-converts env dict to Environment via validator.""" diff --git a/hud/telemetry/tests/test_eval_telemetry.py b/hud/telemetry/tests/test_eval_telemetry.py index 8849cd13c..bfb610044 100644 --- a/hud/telemetry/tests/test_eval_telemetry.py +++ b/hud/telemetry/tests/test_eval_telemetry.py @@ -49,8 +49,8 @@ async def greet(name: str) -> str: """Say hello.""" return f"Hello, {name}!" - # Create task from environment - task = Task(env=env) + # Create task from environment (args={} = runnable, args=None = template) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, @@ -110,7 +110,7 @@ async def failing_tool() -> str: """Always fails.""" raise ValueError("Tool error") - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, @@ -162,7 +162,7 @@ async def multiply(a: int, b: int) -> int: """Multiply two numbers.""" return a * b - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, @@ -195,7 +195,7 @@ async def test_flush_called_on_context_exit(self): async def simple_tool() -> str: return "done" - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.eval.context.flush") as mock_flush, @@ -229,7 +229,7 @@ def should_not_be_called(*args: Any, **kwargs: Any) -> bool: async def test_tool() -> str: return "ok" - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, @@ -272,7 +272,7 @@ def capture_upload( async def echo(message: str) -> str: return message - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, @@ -329,7 +329,7 @@ def capture_upload( async def noop() -> None: pass - task = Task(env=env) + task = Task(env=env, args={}) with ( patch("hud.settings.settings") as mock_settings, From b957818d0eef7cba67bdffab5cc8aab85d1579fe Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:32:54 -0800 Subject: [PATCH 06/27] change routing logic and add tests --- hud/agents/__init__.py | 67 +++--- hud/agents/gateway.py | 1 - hud/agents/tests/test_resolver.py | 192 +++++++++++++++++ hud/datasets/loader.py | 2 +- hud/datasets/runner.py | 18 +- hud/datasets/tests/test_loader.py | 6 +- hud/tools/agent.py | 11 +- hud/tools/tests/test_agent_tool.py | 325 +++++++++++++++++++++++++++++ 8 files changed, 567 insertions(+), 55 deletions(-) create mode 100644 hud/agents/tests/test_resolver.py create mode 100644 hud/tools/tests/test_agent_tool.py diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index 1b5deb1e2..b49b7489a 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -1,15 +1,11 @@ from __future__ import annotations -from typing import TYPE_CHECKING, Any +from typing import Any from .base import MCPAgent from .openai import OpenAIAgent from .openai_chat import OpenAIChatAgent from .operator import OperatorAgent -from .resolver import resolve_cls - -if TYPE_CHECKING: - from hud.types import AgentType __all__ = [ "MCPAgent", @@ -17,50 +13,55 @@ "OpenAIChatAgent", "OperatorAgent", "create_agent", - "resolve_cls", ] -def create_agent(model: str | AgentType, **kwargs: Any) -> MCPAgent: - """Create an agent from a model string or AgentType. +def create_agent(model: str, **kwargs: Any) -> MCPAgent: + """Create an agent for a gateway model. + + This routes ALL requests through the HUD gateway. For direct API access + (using your own API keys), use the agent classes directly. Args: - model: AgentType ("claude"), or gateway model name ("gpt-4o"). - **kwargs: Params passed to agent.create(). + model: Model name (e.g., "gpt-4o", "claude-sonnet-4-5"). + **kwargs: Additional params passed to agent.create(). + + Returns: + Configured MCPAgent instance with gateway routing. Example: ```python - agent = create_agent("claude", model="claude-sonnet-4-5") - agent = create_agent("gpt-4o") # auto-configures gateway + # Gateway routing (recommended) + agent = create_agent("gpt-4o") + agent = create_agent("claude-sonnet-4-5", temperature=0.7) + + # Direct API access (use agent classes) + from hud.agents.claude import ClaudeAgent + + agent = ClaudeAgent.create(model="claude-sonnet-4-5") ``` """ - from hud.types import AgentType as AT + from hud.agents.gateway import build_gateway_client + from hud.agents.resolver import resolve_cls - # AgentType enum → just create - if isinstance(model, AT): - return model.cls.create(**kwargs) - - # Resolve class and optional gateway info + # Resolve class and gateway info agent_cls, gateway_info = resolve_cls(model) - # If not a gateway model, just create - if gateway_info is None: - return agent_cls.create(**kwargs) + # Get model ID from gateway info or use input + model_id = model + if gateway_info: + model_id = gateway_info.get("model") or gateway_info.get("id") or model + + # Build gateway client + provider = gateway_info.get("provider", "openai") if gateway_info else "openai" + client = build_gateway_client(provider) - # Build gateway params - model_id = gateway_info.get("model") or gateway_info.get("id") or model + # Set up kwargs kwargs.setdefault("model", model_id) kwargs.setdefault("validate_api_key", False) - # Build model_client based on provider - if "model_client" not in kwargs and "openai_client" not in kwargs: - from hud.agents.gateway import build_gateway_client - - provider = gateway_info.get("provider", "openai_compatible") - client = build_gateway_client(provider) - - # OpenAIChatAgent uses openai_client key, others use model_client - key = "openai_client" if agent_cls == OpenAIChatAgent else "model_client" - kwargs[key] = client + # Use correct client key + client_key = "openai_client" if agent_cls == OpenAIChatAgent else "model_client" + kwargs.setdefault(client_key, client) return agent_cls.create(**kwargs) diff --git a/hud/agents/gateway.py b/hud/agents/gateway.py index 1249df893..4d0973f8f 100644 --- a/hud/agents/gateway.py +++ b/hud/agents/gateway.py @@ -40,4 +40,3 @@ def build_gateway_client(provider: str) -> Any: from openai import AsyncOpenAI return AsyncOpenAI(api_key=settings.api_key, base_url=settings.hud_gateway_url) - diff --git a/hud/agents/tests/test_resolver.py b/hud/agents/tests/test_resolver.py new file mode 100644 index 000000000..04e6f51ed --- /dev/null +++ b/hud/agents/tests/test_resolver.py @@ -0,0 +1,192 @@ +"""Tests for model resolution and create_agent.""" + +from __future__ import annotations + +from unittest.mock import MagicMock, patch + +import pytest + +from hud.agents import create_agent +from hud.agents.resolver import resolve_cls + + +class TestResolveCls: + """Tests for resolve_cls function.""" + + def test_resolves_known_agent_type(self) -> None: + """Known AgentType strings resolve to their class.""" + from hud.agents.claude import ClaudeAgent + + cls, gateway_info = resolve_cls("claude") + assert cls == ClaudeAgent + assert gateway_info is None + + def test_resolves_openai(self) -> None: + """Resolves 'openai' to OpenAIAgent.""" + from hud.agents import OpenAIAgent + + cls, _gateway_info = resolve_cls("openai") + assert cls == OpenAIAgent + + def test_resolves_gemini(self) -> None: + """Resolves 'gemini' to GeminiAgent.""" + from hud.agents.gemini import GeminiAgent + + cls, _gateway_info = resolve_cls("gemini") + assert cls == GeminiAgent + + def test_unknown_model_without_gateway_raises(self) -> None: + """Unknown model with no gateway models raises ValueError.""" + with ( + patch("hud.agents.resolver._fetch_gateway_models", return_value=[]), + pytest.raises(ValueError, match="not found"), + ): + resolve_cls("unknown-model-xyz") + + def test_resolves_gateway_model(self) -> None: + """Resolves model found in gateway.""" + from hud.agents import OpenAIAgent + + mock_models = [ + {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"}, + ] + + with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models): + cls, info = resolve_cls("gpt-4o") + assert cls == OpenAIAgent + assert info is not None + assert info["id"] == "gpt-4o" + + def test_resolves_anthropic_provider_to_claude(self) -> None: + """Provider 'anthropic' maps to ClaudeAgent.""" + from hud.agents.claude import ClaudeAgent + + mock_models = [ + {"id": "claude-sonnet", "model": "claude-3-sonnet", "provider": "anthropic"}, + ] + + with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models): + cls, _info = resolve_cls("claude-sonnet") + assert cls == ClaudeAgent + + def test_resolves_unknown_provider_to_openai_compatible(self) -> None: + """Unknown provider maps to OpenAIChatAgent.""" + from hud.agents.openai_chat import OpenAIChatAgent + + mock_models = [ + {"id": "custom-model", "model": "custom", "provider": "custom-provider"}, + ] + + with patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models): + cls, _info = resolve_cls("custom-model") + assert cls == OpenAIChatAgent + + +class TestCreateAgent: + """Tests for create_agent function - gateway-only.""" + + def test_creates_with_gateway_client(self) -> None: + """create_agent always uses gateway routing.""" + from hud.agents import OpenAIAgent + + mock_models = [ + {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"}, + ] + + with ( + patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models), + patch.object(OpenAIAgent, "create") as mock_create, + patch("hud.agents.gateway.build_gateway_client") as mock_build_client, + ): + mock_client = MagicMock() + mock_build_client.return_value = mock_client + mock_agent = MagicMock() + mock_create.return_value = mock_agent + + agent = create_agent("gpt-4o") + + # Should have set model and model_client + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs["model"] == "gpt-4o" + assert "model_client" in call_kwargs + assert agent == mock_agent + + def test_passes_kwargs_to_create(self) -> None: + """Extra kwargs are passed to agent.create().""" + from hud.agents import OpenAIAgent + + mock_models = [ + {"id": "gpt-4o", "model": "gpt-4o", "provider": "openai"}, + ] + + with ( + patch("hud.agents.resolver._fetch_gateway_models", return_value=mock_models), + patch.object(OpenAIAgent, "create") as mock_create, + patch("hud.agents.gateway.build_gateway_client"), + ): + mock_create.return_value = MagicMock() + + create_agent("gpt-4o", temperature=0.5, max_tokens=1000) + + call_kwargs = mock_create.call_args.kwargs + assert call_kwargs["temperature"] == 0.5 + assert call_kwargs["max_tokens"] == 1000 + + def test_known_agent_type_also_uses_gateway(self) -> None: + """Even 'claude' string uses gateway (it's a gateway shortcut).""" + from hud.agents.claude import ClaudeAgent + + with ( + patch.object(ClaudeAgent, "create") as mock_create, + patch("hud.agents.gateway.build_gateway_client") as mock_build_client, + ): + mock_client = MagicMock() + mock_build_client.return_value = mock_client + mock_create.return_value = MagicMock() + + create_agent("claude") + + # Should still build gateway client + mock_build_client.assert_called_once() + call_kwargs = mock_create.call_args.kwargs + assert "model_client" in call_kwargs + + +class TestBuildGatewayClient: + """Tests for build_gateway_client function.""" + + def test_builds_anthropic_client(self) -> None: + """Builds AsyncAnthropic for anthropic provider.""" + from hud.agents.gateway import build_gateway_client + + with patch("hud.settings.settings") as mock_settings: + mock_settings.api_key = "test-key" + mock_settings.hud_gateway_url = "https://gateway.hud.ai" + + with patch("anthropic.AsyncAnthropic") as mock_client_cls: + build_gateway_client("anthropic") + mock_client_cls.assert_called_once() + + def test_builds_openai_client_for_openai(self) -> None: + """Builds AsyncOpenAI for openai provider.""" + from hud.agents.gateway import build_gateway_client + + with patch("hud.settings.settings") as mock_settings: + mock_settings.api_key = "test-key" + mock_settings.hud_gateway_url = "https://gateway.hud.ai" + + with patch("openai.AsyncOpenAI") as mock_client_cls: + build_gateway_client("openai") + mock_client_cls.assert_called_once() + + def test_builds_openai_client_for_unknown(self) -> None: + """Builds AsyncOpenAI for unknown providers (openai-compatible).""" + from hud.agents.gateway import build_gateway_client + + with patch("hud.settings.settings") as mock_settings: + mock_settings.api_key = "test-key" + mock_settings.hud_gateway_url = "https://gateway.hud.ai" + + with patch("openai.AsyncOpenAI") as mock_client_cls: + build_gateway_client("together") + mock_client_cls.assert_called_once() diff --git a/hud/datasets/loader.py b/hud/datasets/loader.py index 969bf75cd..d313228ea 100644 --- a/hud/datasets/loader.py +++ b/hud/datasets/loader.py @@ -306,7 +306,7 @@ def save_tasks( ) response.raise_for_status() data = response.json() - taskset_id = data.get("taskset_id") or data.get("evalset_id") or data.get("id") or name + taskset_id = data.get("evalset_id") or data.get("id") or name logger.info("Saved %d tasks to taskset: %s", len(tasks), taskset_id) return taskset_id except httpx.HTTPStatusError as e: diff --git a/hud/datasets/runner.py b/hud/datasets/runner.py index acb79fb95..393e7858e 100644 --- a/hud/datasets/runner.py +++ b/hud/datasets/runner.py @@ -22,7 +22,7 @@ async def run_dataset( tasks: str | TaskInput | Sequence[TaskInput], - agent_type: str | AgentType, + agent_type: AgentType, *, agent_params: dict[str, Any] | None = None, max_steps: int = 10, @@ -40,7 +40,7 @@ async def run_dataset( - A source string (file path, API slug) - loaded via load_tasks() - A single TaskInput (Task, LegacyTask, or dict) - A list of TaskInput objects - agent_type: Type of agent to create (e.g., "claude", "openai", AgentType.CLAUDE). + agent_type: AgentType enum specifying the agent to use. agent_params: Parameters to pass to agent.create(). max_steps: Maximum steps per task. max_concurrent: Maximum concurrent tasks (for parallel execution). @@ -93,10 +93,8 @@ async def run_dataset( max_concurrent=max_concurrent, quiet=quiet, ) as ctx: - # Create agent (handles AgentType, gateway models, etc.) - from hud.agents import create_agent - - agent = create_agent(agent_type, **(agent_params or {})) + # Create agent using AgentType.cls.create() + agent = agent_type.cls.create(**(agent_params or {})) await agent.run(ctx, max_steps=max_steps) # Reward is computed by EvalContext.__aexit__ from evaluate tools @@ -110,7 +108,7 @@ async def run_dataset( async def run_single_task( task: Task, *, - agent_type: str | AgentType, + agent_type: AgentType, agent_params: dict[str, Any] | None = None, max_steps: int = 10, job_id: str | None = None, @@ -196,10 +194,8 @@ async def run_single_task( if ctx.system_prompt and "system_prompt" not in final_agent_params: final_agent_params["system_prompt"] = ctx.system_prompt - # Create agent (handles AgentType, gateway models, etc.) - from hud.agents import create_agent - - agent = create_agent(agent_type, **final_agent_params) + # Create agent using AgentType.cls.create() + agent = agent_type.cls.create(**final_agent_params) # Store metadata if provided if metadata: diff --git a/hud/datasets/tests/test_loader.py b/hud/datasets/tests/test_loader.py index ef8c9cd89..5c6658709 100644 --- a/hud/datasets/tests/test_loader.py +++ b/hud/datasets/tests/test_loader.py @@ -22,10 +22,10 @@ def test_load_tasks_success( mock_settings.api_key = "test_key" mock_response = MagicMock() - # TasksetTasksResponse format: tasks keyed by task ID + # EvalsetTasksResponse format: tasks keyed by task ID mock_response.json.return_value = { - "taskset_id": "taskset-123", - "taskset_name": "test-dataset", + "evalset_id": "evalset-123", + "evalset_name": "test-dataset", "tasks": { "task-1": { "env": {"name": "test"}, diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 01b224a2c..12948cafd 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -38,11 +38,12 @@ class AgentTool(BaseTool): ```python @env.scenario() async def investigate( - issue_id: str, # Required - orchestrator sees - expected_cause: str | None = None # Eval only - hidden + issue_id: str, # Required - orchestrator sees + expected_cause: str | None = None, # Eval only - hidden ): yield {"task": f"Investigate {issue_id}"} + seer = AgentTool(env("investigate"), model="ft:seer-v2") ``` """ @@ -77,10 +78,7 @@ def __init__( scenario_fn = task.env._scenarios.get(task.scenario) if scenario_fn: sig = inspect.signature(scenario_fn) - visible = { - name: p for name, p in sig.parameters.items() - if not _is_eval_only(p) - } + visible = {name: p for name, p in sig.parameters.items() if not _is_eval_only(p)} self._visible_params = set(visible.keys()) self._param_schema = self._build_schema(visible) @@ -140,6 +138,7 @@ async def __call__(self, **kwargs: Any) -> ToolResult: async with run_eval(task, trace=self._trace) as ctx: if self._model: from hud.agents import create_agent + agent = create_agent(self._model, **self._agent_params) else: agent = self._agent_cls.create(**self._agent_params) # type: ignore diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py new file mode 100644 index 000000000..b0bc8651f --- /dev/null +++ b/hud/tools/tests/test_agent_tool.py @@ -0,0 +1,325 @@ +"""Tests for AgentTool - scenario-to-agent composition.""" + +from __future__ import annotations + +import inspect +from unittest.mock import AsyncMock, MagicMock, patch + +import pytest + +from hud.environment import Environment +from hud.eval.task import Task +from hud.tools.agent import AgentTool, _is_eval_only + + +class TestIsEvalOnly: + """Tests for _is_eval_only helper function.""" + + def test_required_param_not_eval_only(self) -> None: + """Required params (no default) are not eval-only.""" + + def fn(x: str) -> None: + pass + + sig = inspect.signature(fn) + param = sig.parameters["x"] + assert not _is_eval_only(param) + + def test_optional_with_value_not_eval_only(self) -> None: + """Optional params with non-None default are not eval-only.""" + + def fn(x: str = "default") -> None: + pass + + sig = inspect.signature(fn) + param = sig.parameters["x"] + assert not _is_eval_only(param) + + def test_optional_none_without_union_not_eval_only(self) -> None: + """Optional with None default but no None in type is not eval-only.""" + + def fn(x: str = None) -> None: # type: ignore[assignment] # noqa: RUF013 + pass + + sig = inspect.signature(fn) + param = sig.parameters["x"] + assert not _is_eval_only(param) + + def test_optional_none_with_union_is_eval_only(self) -> None: + """Params with `X | None = None` pattern are eval-only.""" + + def fn(x: str | None = None) -> None: + pass + + sig = inspect.signature(fn) + param = sig.parameters["x"] + assert _is_eval_only(param) + + def test_optional_int_none_is_eval_only(self) -> None: + """Works with int | None = None too.""" + + def fn(x: int | None = None) -> None: + pass + + sig = inspect.signature(fn) + param = sig.parameters["x"] + assert _is_eval_only(param) + + +class TestAgentToolInit: + """Tests for AgentTool initialization.""" + + def test_requires_model_or_agent(self) -> None: + """Must provide either model or agent.""" + task = Task(args={}) + + with pytest.raises(ValueError, match="Must provide either"): + AgentTool(task) + + def test_cannot_provide_both_model_and_agent(self) -> None: + """Cannot provide both model and agent.""" + task = Task(args={}) + mock_agent = MagicMock() + + with pytest.raises(ValueError, match="Cannot provide both"): + AgentTool(task, model="claude", agent=mock_agent) # type: ignore[arg-type] + + def test_accepts_model_string(self) -> None: + """Can create with model string.""" + task = Task(scenario="test", args={}) + tool = AgentTool(task, model="claude") + + assert tool._model == "claude" + assert tool._agent_cls is None + + def test_accepts_agent_class(self) -> None: + """Can create with custom agent class.""" + task = Task(scenario="test", args={}) + mock_agent_cls = MagicMock() + tool = AgentTool(task, agent=mock_agent_cls) # type: ignore[arg-type] + + assert tool._model is None + assert tool._agent_cls is mock_agent_cls + + def test_name_defaults_to_scenario(self) -> None: + """Tool name defaults to scenario name.""" + task = Task(scenario="investigate", args={}) + tool = AgentTool(task, model="claude") + + assert tool.name == "investigate" + + def test_name_can_be_overridden(self) -> None: + """Tool name can be overridden.""" + task = Task(scenario="investigate", args={}) + tool = AgentTool(task, model="claude", name="custom_name") + + assert tool.name == "custom_name" + + +class TestAgentToolParamFiltering: + """Tests for parameter filtering (eval-only params hidden).""" + + def test_filters_eval_only_params(self) -> None: + """Eval-only params (| None = None) are filtered from visible_params.""" + env = Environment("test") + + @env.scenario() + async def investigate( + issue_id: str, + include_traces: bool = True, + expected_cause: str | None = None, # Eval only + ): + yield {"task": f"Investigate {issue_id}"} + + task = env("investigate") + tool = AgentTool(task, model="claude") + + # visible_params should only have issue_id and include_traces + assert "issue_id" in tool._visible_params + assert "include_traces" in tool._visible_params + assert "expected_cause" not in tool._visible_params + + def test_all_required_params_visible(self) -> None: + """All required params are visible.""" + env = Environment("test") + + @env.scenario() + async def search(query: str, limit: int): + yield {"task": f"Search: {query}"} + + task = env("search") + tool = AgentTool(task, model="claude") + + assert "query" in tool._visible_params + assert "limit" in tool._visible_params + + def test_optional_with_default_visible(self) -> None: + """Optional params with non-None defaults are visible.""" + env = Environment("test") + + @env.scenario() + async def fetch(url: str, request_timeout: int = 30, retries: int = 3): + yield {"task": f"Fetch {url}"} + + task = env("fetch") + tool = AgentTool(task, model="claude") + + assert "url" in tool._visible_params + assert "request_timeout" in tool._visible_params + assert "retries" in tool._visible_params + + +class TestAgentToolSchema: + """Tests for JSON schema generation.""" + + def test_builds_json_schema(self) -> None: + """Builds proper JSON schema from visible params.""" + env = Environment("test") + + @env.scenario() + async def investigate(issue_id: str, verbose: bool = False): + yield {"task": f"Investigate {issue_id}"} + + task = env("investigate") + tool = AgentTool(task, model="claude") + + schema = tool._param_schema + assert schema is not None + assert schema["type"] == "object" + assert "issue_id" in schema["properties"] + assert "verbose" in schema["properties"] + assert "issue_id" in schema["required"] + assert "verbose" not in schema["required"] # Has default + + def test_schema_excludes_eval_only(self) -> None: + """Schema excludes eval-only params.""" + env = Environment("test") + + @env.scenario() + async def check( + item_id: str, + expected_status: str | None = None, # Eval only + ): + yield {"task": f"Check {item_id}"} + + task = env("check") + tool = AgentTool(task, model="claude") + + schema = tool._param_schema + assert schema is not None + assert "item_id" in schema["properties"] + assert "expected_status" not in schema["properties"] + + +class TestAgentToolMCP: + """Tests for MCP tool integration.""" + + def test_mcp_property_returns_function_tool(self) -> None: + """The mcp property returns a FunctionTool.""" + from fastmcp.tools import FunctionTool + + env = Environment("test") + + @env.scenario() + async def greet(name: str): + yield {"task": f"Greet {name}"} + + task = env("greet") + tool = AgentTool(task, model="claude") + + mcp_tool = tool.mcp + assert isinstance(mcp_tool, FunctionTool) + + def test_mcp_has_filtered_parameters(self) -> None: + """MCP tool has filtered parameter schema.""" + env = Environment("test") + + @env.scenario() + async def analyze( + data: str, + expected_result: str | None = None, # Eval only + ): + yield {"task": f"Analyze {data}"} + + task = env("analyze") + tool = AgentTool(task, model="claude") + + mcp_tool = tool.mcp + params = mcp_tool.parameters + + assert "data" in params["properties"] + assert "expected_result" not in params["properties"] + + +class TestAgentToolCall: + """Tests for AgentTool.__call__.""" + + @pytest.mark.asyncio + async def test_filters_kwargs_to_visible_only(self) -> None: + """Call filters kwargs to visible params only.""" + env = Environment("test") + + @env.scenario() + async def process(item: str, expected: str | None = None): + yield {"task": f"Process {item}"} + + task = env("process") + tool = AgentTool(task, model="claude") + + # Mock the eval context and agent + with ( + patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.agents.create_agent") as mock_create_agent, + ): + mock_ctx = AsyncMock() + mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) + mock_ctx.__aexit__ = AsyncMock(return_value=None) + mock_run_eval.return_value = mock_ctx + + mock_agent = MagicMock() + mock_agent.run = AsyncMock(return_value=MagicMock(content="result")) + mock_create_agent.return_value = mock_agent + + # Call with both visible and eval-only params + await tool(item="test", expected="should_be_filtered") + + # Check that task was created with filtered args + call_args = mock_run_eval.call_args + task_arg = call_args[0][0] + assert "item" in task_arg.args + assert "expected" not in task_arg.args # Filtered out + + @pytest.mark.asyncio + async def test_merges_template_args(self) -> None: + """Call merges kwargs with template args.""" + env = Environment("test") + + @env.scenario() + async def search(query: str, limit: int = 10): + yield {"task": f"Search {query}"} + + # Create template with some args pre-filled + task = env("search", limit=5) + tool = AgentTool(task, model="claude") + + with ( + patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.agents.create_agent") as mock_create_agent, + ): + mock_ctx = AsyncMock() + mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) + mock_ctx.__aexit__ = AsyncMock(return_value=None) + mock_run_eval.return_value = mock_ctx + + mock_agent = MagicMock() + mock_agent.run = AsyncMock(return_value=MagicMock(content="result")) + mock_create_agent.return_value = mock_agent + + # Call with additional arg + await tool(query="test query") + + # Check merged args + call_args = mock_run_eval.call_args + task_arg = call_args[0][0] + assert task_arg.args["query"] == "test query" + assert task_arg.args["limit"] == 5 # From template From 219f255b89433d92a6c164a2e657861e8a60246c Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:34:09 -0800 Subject: [PATCH 07/27] lint --- hud/eval/context.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hud/eval/context.py b/hud/eval/context.py index ad129d3b5..e7d4be141 100644 --- a/hud/eval/context.py +++ b/hud/eval/context.py @@ -353,7 +353,7 @@ async def _run_task_scenario_setup(self) -> None: if self._task is None or self._task.scenario is None: return - prompt = await self.run_scenario_setup(self._task.scenario, self._task.args) + prompt = await self.run_scenario_setup(self._task.scenario, self._task.args or {}) if prompt: self.prompt = prompt From 627a6e3716e55d3c2c58ae41a5ffdac572e780d5 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:37:42 -0800 Subject: [PATCH 08/27] add convenience back --- hud/datasets/runner.py | 8 ++++++-- hud/eval/task.py | 2 +- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/hud/datasets/runner.py b/hud/datasets/runner.py index 393e7858e..1a27074f2 100644 --- a/hud/datasets/runner.py +++ b/hud/datasets/runner.py @@ -22,7 +22,7 @@ async def run_dataset( tasks: str | TaskInput | Sequence[TaskInput], - agent_type: AgentType, + agent_type: str | AgentType, *, agent_params: dict[str, Any] | None = None, max_steps: int = 10, @@ -40,7 +40,7 @@ async def run_dataset( - A source string (file path, API slug) - loaded via load_tasks() - A single TaskInput (Task, LegacyTask, or dict) - A list of TaskInput objects - agent_type: AgentType enum specifying the agent to use. + agent_type: Agent type (e.g., "claude", "openai", AgentType.CLAUDE). agent_params: Parameters to pass to agent.create(). max_steps: Maximum steps per task. max_concurrent: Maximum concurrent tasks (for parallel execution). @@ -70,6 +70,10 @@ async def run_dataset( from hud.datasets.loader import load_tasks from hud.eval.task import Task + # Normalize agent_type to AgentType enum + if isinstance(agent_type, str): + agent_type = AgentType(agent_type) + # Normalize tasks to list[Task] task_list: list[Task] if isinstance(tasks, str): diff --git a/hud/eval/task.py b/hud/eval/task.py index ea621a9ab..cfa6d64a9 100644 --- a/hud/eval/task.py +++ b/hud/eval/task.py @@ -338,6 +338,6 @@ def copy(self) -> Task: id=self.id, env=self.env, # Share reference scenario=self.scenario, - args=self.args.copy() if self.args else None, + args=self.args.copy() if self.args is not None else None, validation=self.validation.copy() if self.validation else None, ) From 85aad9815cee91e03e97724fc1445a84391cdc6c Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:41:41 -0800 Subject: [PATCH 09/27] fix edge cases --- hud/tools/agent.py | 78 +++++++++++++++++++++++------- hud/tools/tests/test_agent_tool.py | 36 +++++++++++--- 2 files changed, 90 insertions(+), 24 deletions(-) diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 12948cafd..a0e9af4b0 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -3,10 +3,10 @@ from __future__ import annotations import inspect -from typing import TYPE_CHECKING, Any, get_args, get_origin +from typing import TYPE_CHECKING, Any, Union, get_args, get_origin from fastmcp.tools.tool import ToolResult -from mcp.types import TextContent +from mcp.types import TextContent, Tool from hud.tools.base import BaseTool @@ -18,14 +18,44 @@ def _is_eval_only(param: inspect.Parameter) -> bool: - """Check if param is eval-only: has None default AND None in type union.""" + """Check if param is eval-only: has None default AND None in type union. + + Handles both runtime types and string annotations (PEP 563). + """ + # Must have default of None if param.default is not None: return False if param.annotation is inspect.Parameter.empty: return False - origin = get_origin(param.annotation) - if origin is not None: - return type(None) in get_args(param.annotation) + + annotation = param.annotation + + # Handle string annotations (from __future__ annotations or quoted) + if isinstance(annotation, str): + # Check if it looks like "X | None", "Union[X, None]", or "Optional[X]" + return ( + "| None" in annotation + or "None |" in annotation + or "Optional[" in annotation + or ("Union[" in annotation and "None" in annotation) + ) + + # Handle runtime type annotations + origin = get_origin(annotation) + + # Union types (X | None or Union[X, None]) + if origin is Union: + return type(None) in get_args(annotation) + + # For Python 3.10+ union syntax at runtime (types.UnionType) + try: + import types + + if isinstance(annotation, types.UnionType): + return type(None) in get_args(annotation) + except (ImportError, AttributeError): + pass + return False @@ -72,13 +102,19 @@ def __init__( # Get visible params from scenario function self._visible_params: set[str] = set() - self._param_schema: dict[str, Any] | None = None + self._param_schema: dict[str, Any] = { + "type": "object", + "properties": {}, + "required": [], + } if task.env and task.scenario: scenario_fn = task.env._scenarios.get(task.scenario) if scenario_fn: sig = inspect.signature(scenario_fn) - visible = {name: p for name, p in sig.parameters.items() if not _is_eval_only(p)} + visible = { + name: p for name, p in sig.parameters.items() if not _is_eval_only(p) + } self._visible_params = set(visible.keys()) self._param_schema = self._build_schema(visible) @@ -97,7 +133,17 @@ def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]: for name, param in params.items(): if param.annotation is not inspect.Parameter.empty: try: - adapter = TypeAdapter(param.annotation) + # Handle string annotations + annotation = param.annotation + if isinstance(annotation, str): + # Try to evaluate the annotation + try: + annotation = eval(annotation) # noqa: S307 + except Exception: + properties[name] = {"type": "string"} + continue + + adapter = TypeAdapter(annotation) properties[name] = adapter.json_schema() except Exception: properties[name] = {"type": "string"} @@ -112,16 +158,14 @@ def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]: return {"type": "object", "properties": properties, "required": required} @property - def mcp(self) -> Any: - """Get as FastMCP FunctionTool with filtered schema.""" + def mcp(self) -> Tool: + """Get as MCP Tool with filtered schema.""" if not hasattr(self, "_mcp_tool"): - from fastmcp.tools import FunctionTool - - self._mcp_tool = FunctionTool.from_function( - self, name=self.name, description=self.description + self._mcp_tool = Tool( + name=self.name, + description=self.description, + inputSchema=self._param_schema, ) - if self._param_schema: - self._mcp_tool.parameters = self._param_schema return self._mcp_tool async def __call__(self, **kwargs: Any) -> ToolResult: diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py index b0bc8651f..a58346b49 100644 --- a/hud/tools/tests/test_agent_tool.py +++ b/hud/tools/tests/test_agent_tool.py @@ -65,6 +65,27 @@ def fn(x: int | None = None) -> None: param = sig.parameters["x"] assert _is_eval_only(param) + def test_string_annotation_with_none_union(self) -> None: + """Handles string annotations like 'str | None'.""" + # Simulate string annotation + param = inspect.Parameter( + "x", + inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=None, + annotation="str | None", + ) + assert _is_eval_only(param) + + def test_string_annotation_without_none(self) -> None: + """String annotations without None are not eval-only.""" + param = inspect.Parameter( + "x", + inspect.Parameter.POSITIONAL_OR_KEYWORD, + default=None, + annotation="str", + ) + assert not _is_eval_only(param) + class TestAgentToolInit: """Tests for AgentTool initialization.""" @@ -123,6 +144,7 @@ def test_filters_eval_only_params(self) -> None: """Eval-only params (| None = None) are filtered from visible_params.""" env = Environment("test") + # Use Union syntax for consistency across Python versions @env.scenario() async def investigate( issue_id: str, @@ -214,9 +236,9 @@ async def check( class TestAgentToolMCP: """Tests for MCP tool integration.""" - def test_mcp_property_returns_function_tool(self) -> None: - """The mcp property returns a FunctionTool.""" - from fastmcp.tools import FunctionTool + def test_mcp_property_returns_tool(self) -> None: + """The mcp property returns an MCP Tool.""" + from mcp.types import Tool env = Environment("test") @@ -228,7 +250,7 @@ async def greet(name: str): tool = AgentTool(task, model="claude") mcp_tool = tool.mcp - assert isinstance(mcp_tool, FunctionTool) + assert isinstance(mcp_tool, Tool) def test_mcp_has_filtered_parameters(self) -> None: """MCP tool has filtered parameter schema.""" @@ -245,7 +267,7 @@ async def analyze( tool = AgentTool(task, model="claude") mcp_tool = tool.mcp - params = mcp_tool.parameters + params = mcp_tool.inputSchema assert "data" in params["properties"] assert "expected_result" not in params["properties"] @@ -268,7 +290,7 @@ async def process(item: str, expected: str | None = None): # Mock the eval context and agent with ( - patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.eval.manager.run_eval") as mock_run_eval, patch("hud.agents.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() @@ -303,7 +325,7 @@ async def search(query: str, limit: int = 10): tool = AgentTool(task, model="claude") with ( - patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.eval.manager.run_eval") as mock_run_eval, patch("hud.agents.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() From 8e6b1862f8942b85532d219fba7cafdbae1713fe Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:42:15 -0800 Subject: [PATCH 10/27] mock path fixes --- hud/tools/tests/test_agent_tool.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py index a58346b49..f3d1bf658 100644 --- a/hud/tools/tests/test_agent_tool.py +++ b/hud/tools/tests/test_agent_tool.py @@ -290,8 +290,8 @@ async def process(item: str, expected: str | None = None): # Mock the eval context and agent with ( - patch("hud.eval.manager.run_eval") as mock_run_eval, - patch("hud.agents.create_agent") as mock_create_agent, + patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.tools.agent.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) @@ -325,8 +325,8 @@ async def search(query: str, limit: int = 10): tool = AgentTool(task, model="claude") with ( - patch("hud.eval.manager.run_eval") as mock_run_eval, - patch("hud.agents.create_agent") as mock_create_agent, + patch("hud.tools.agent.run_eval") as mock_run_eval, + patch("hud.tools.agent.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) From 760f6c8fee77d04c7bdb15b148fd5ddfbbd8ba01 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:44:05 -0800 Subject: [PATCH 11/27] change import paths --- hud/tools/tests/test_agent_tool.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py index f3d1bf658..c6aafa52f 100644 --- a/hud/tools/tests/test_agent_tool.py +++ b/hud/tools/tests/test_agent_tool.py @@ -279,6 +279,10 @@ class TestAgentToolCall: @pytest.mark.asyncio async def test_filters_kwargs_to_visible_only(self) -> None: """Call filters kwargs to visible params only.""" + # Import modules first so patches work + import hud.agents # noqa: F401 + import hud.eval.manager # noqa: F401 + env = Environment("test") @env.scenario() @@ -290,8 +294,8 @@ async def process(item: str, expected: str | None = None): # Mock the eval context and agent with ( - patch("hud.tools.agent.run_eval") as mock_run_eval, - patch("hud.tools.agent.create_agent") as mock_create_agent, + patch("hud.eval.manager.run_eval") as mock_run_eval, + patch("hud.agents.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) @@ -314,6 +318,10 @@ async def process(item: str, expected: str | None = None): @pytest.mark.asyncio async def test_merges_template_args(self) -> None: """Call merges kwargs with template args.""" + # Import modules first so patches work + import hud.agents # noqa: F401 + import hud.eval.manager # noqa: F401 + env = Environment("test") @env.scenario() @@ -325,8 +333,8 @@ async def search(query: str, limit: int = 10): tool = AgentTool(task, model="claude") with ( - patch("hud.tools.agent.run_eval") as mock_run_eval, - patch("hud.tools.agent.create_agent") as mock_create_agent, + patch("hud.eval.manager.run_eval") as mock_run_eval, + patch("hud.agents.create_agent") as mock_create_agent, ): mock_ctx = AsyncMock() mock_ctx.__aenter__ = AsyncMock(return_value=mock_ctx) From 2a5f10bd3766ef7b1940ead6cabd0dc4230537ca Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 18:44:21 -0800 Subject: [PATCH 12/27] format --- hud/tools/agent.py | 4 +--- hud/tools/tests/test_agent_tool.py | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/hud/tools/agent.py b/hud/tools/agent.py index a0e9af4b0..e93203caa 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -112,9 +112,7 @@ def __init__( scenario_fn = task.env._scenarios.get(task.scenario) if scenario_fn: sig = inspect.signature(scenario_fn) - visible = { - name: p for name, p in sig.parameters.items() if not _is_eval_only(p) - } + visible = {name: p for name, p in sig.parameters.items() if not _is_eval_only(p)} self._visible_params = set(visible.keys()) self._param_schema = self._build_schema(visible) diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py index c6aafa52f..db5395ad2 100644 --- a/hud/tools/tests/test_agent_tool.py +++ b/hud/tools/tests/test_agent_tool.py @@ -280,7 +280,7 @@ class TestAgentToolCall: async def test_filters_kwargs_to_visible_only(self) -> None: """Call filters kwargs to visible params only.""" # Import modules first so patches work - import hud.agents # noqa: F401 + import hud.agents import hud.eval.manager # noqa: F401 env = Environment("test") @@ -319,7 +319,7 @@ async def process(item: str, expected: str | None = None): async def test_merges_template_args(self) -> None: """Call merges kwargs with template args.""" # Import modules first so patches work - import hud.agents # noqa: F401 + import hud.agents import hud.eval.manager # noqa: F401 env = Environment("test") From 99fd3c2cb692ecee49e73d577eabdeb75f0292ce Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 19:45:39 -0800 Subject: [PATCH 13/27] fix agent edge cases --- hud/agents/__init__.py | 11 +++++++---- hud/tools/agent.py | 27 +++++++++++++++++++-------- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index b49b7489a..d8aa198ba 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -58,10 +58,13 @@ def create_agent(model: str, **kwargs: Any) -> MCPAgent: # Set up kwargs kwargs.setdefault("model", model_id) - kwargs.setdefault("validate_api_key", False) - # Use correct client key - client_key = "openai_client" if agent_cls == OpenAIChatAgent else "model_client" - kwargs.setdefault(client_key, client) + # Use correct client key based on agent type + if agent_cls == OpenAIChatAgent: + kwargs.setdefault("openai_client", client) + else: + # Claude and other agents use model_client and validate_api_key + kwargs.setdefault("model_client", client) + kwargs.setdefault("validate_api_key", False) return agent_cls.create(**kwargs) diff --git a/hud/tools/agent.py b/hud/tools/agent.py index e93203caa..a59191771 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -5,8 +5,8 @@ import inspect from typing import TYPE_CHECKING, Any, Union, get_args, get_origin -from fastmcp.tools.tool import ToolResult -from mcp.types import TextContent, Tool +from fastmcp.tools.tool import FunctionTool, ToolResult +from mcp.types import TextContent from hud.tools.base import BaseTool @@ -156,18 +156,26 @@ def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]: return {"type": "object", "properties": properties, "required": required} @property - def mcp(self) -> Tool: - """Get as MCP Tool with filtered schema.""" + def mcp(self) -> FunctionTool: + """Get as FastMCP FunctionTool with filtered schema.""" if not hasattr(self, "_mcp_tool"): - self._mcp_tool = Tool( + # Directly instantiate FunctionTool with our callable and schema + # This bypasses from_function's signature parsing + self._mcp_tool = FunctionTool( name=self.name, - description=self.description, - inputSchema=self._param_schema, + description=self.description or "", + parameters=self._param_schema, + fn=self._execute_with_args, ) return self._mcp_tool + async def _execute_with_args(self, **kwargs: Any) -> ToolResult: + """Internal executor that FastMCP calls with parsed arguments.""" + return await self(**kwargs) + async def __call__(self, **kwargs: Any) -> ToolResult: """Execute the task with a fresh agent.""" + from hud.eval.context import get_current_trace_id from hud.eval.manager import run_eval # Filter to visible params only @@ -177,7 +185,10 @@ async def __call__(self, **kwargs: Any) -> ToolResult: base_args = self._task.args or {} task = self._task.model_copy(update={"args": {**base_args, **filtered}}) - async with run_eval(task, trace=self._trace) as ctx: + # Use parent trace if available (for hierarchical agents) + parent_trace_id = get_current_trace_id() + + async with run_eval(task, trace=self._trace, trace_id=parent_trace_id, quiet=True) as ctx: if self._model: from hud.agents import create_agent From d74edb472e16e9fd13a2880ddcbb317deff791cf Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 20:09:57 -0800 Subject: [PATCH 14/27] nested tracing --- hud/tools/agent.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/hud/tools/agent.py b/hud/tools/agent.py index a59191771..59391007b 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -187,8 +187,17 @@ async def __call__(self, **kwargs: Any) -> ToolResult: # Use parent trace if available (for hierarchical agents) parent_trace_id = get_current_trace_id() - - async with run_eval(task, trace=self._trace, trace_id=parent_trace_id, quiet=True) as ctx: + + # If nested (has parent), skip subagent's enter/exit registration but keep tool instrumentation + # Tool calls are still recorded via the shared trace_id's context + is_nested = parent_trace_id is not None + + async with run_eval( + task, + trace=not is_nested, # Skip enter/exit for nested agents + trace_id=parent_trace_id, + quiet=True + ) as ctx: if self._model: from hud.agents import create_agent From 641576266f528017e1c0d457750f1fc1d2126862 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 20:45:14 -0800 Subject: [PATCH 15/27] fix tests --- hud/agents/__init__.py | 16 ++++++++++++++-- hud/tools/agent.py | 19 +++++++++++-------- hud/tools/tests/test_agent_tool.py | 8 ++++---- 3 files changed, 29 insertions(+), 14 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index d8aa198ba..b9be1b6d2 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -52,8 +52,20 @@ def create_agent(model: str, **kwargs: Any) -> MCPAgent: if gateway_info: model_id = gateway_info.get("model") or gateway_info.get("id") or model - # Build gateway client - provider = gateway_info.get("provider", "openai") if gateway_info else "openai" + # Determine provider: from gateway info, or infer from agent class + if gateway_info: + provider = gateway_info.get("provider", "openai") + else: + # Map agent class to provider for known types + from hud.agents.claude import ClaudeAgent + from hud.agents.gemini import GeminiAgent + + _AGENT_TO_PROVIDER = { + ClaudeAgent: "anthropic", + GeminiAgent: "google", + } + provider = _AGENT_TO_PROVIDER.get(agent_cls, "openai") + client = build_gateway_client(provider) # Set up kwargs diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 59391007b..3932f5191 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -89,9 +89,9 @@ def __init__( description: str | None = None, trace: bool = False, ) -> None: - if model is None and agent is None: + if not model and agent is None: raise ValueError("Must provide either 'model' or 'agent'") - if model is not None and agent is not None: + if model and agent is not None: raise ValueError("Cannot provide both 'model' and 'agent'") self._task = task @@ -187,16 +187,19 @@ async def __call__(self, **kwargs: Any) -> ToolResult: # Use parent trace if available (for hierarchical agents) parent_trace_id = get_current_trace_id() - - # If nested (has parent), skip subagent's enter/exit registration but keep tool instrumentation + + # If nested (has parent), skip subagent's enter/exit registration # Tool calls are still recorded via the shared trace_id's context is_nested = parent_trace_id is not None + # Trace if explicitly requested AND not nested (nested uses parent trace) + should_trace = self._trace and not is_nested + async with run_eval( - task, - trace=not is_nested, # Skip enter/exit for nested agents - trace_id=parent_trace_id, - quiet=True + task, + trace=should_trace, + trace_id=parent_trace_id, + quiet=True, ) as ctx: if self._model: from hud.agents import create_agent diff --git a/hud/tools/tests/test_agent_tool.py b/hud/tools/tests/test_agent_tool.py index db5395ad2..de8196c38 100644 --- a/hud/tools/tests/test_agent_tool.py +++ b/hud/tools/tests/test_agent_tool.py @@ -237,8 +237,8 @@ class TestAgentToolMCP: """Tests for MCP tool integration.""" def test_mcp_property_returns_tool(self) -> None: - """The mcp property returns an MCP Tool.""" - from mcp.types import Tool + """The mcp property returns a FastMCP FunctionTool.""" + from fastmcp.tools import FunctionTool env = Environment("test") @@ -250,7 +250,7 @@ async def greet(name: str): tool = AgentTool(task, model="claude") mcp_tool = tool.mcp - assert isinstance(mcp_tool, Tool) + assert isinstance(mcp_tool, FunctionTool) def test_mcp_has_filtered_parameters(self) -> None: """MCP tool has filtered parameter schema.""" @@ -267,7 +267,7 @@ async def analyze( tool = AgentTool(task, model="claude") mcp_tool = tool.mcp - params = mcp_tool.inputSchema + params = mcp_tool.parameters # FunctionTool uses 'parameters' assert "data" in params["properties"] assert "expected_result" not in params["properties"] From 7027550e134ff75ca1f73e93856b66d7d3c9b8af Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 23:01:24 -0800 Subject: [PATCH 16/27] agent tool examples --- docs/cookbook/ops-diagnostics.mdx | 478 ++++++++++++++++++++++++++++++ docs/reference/tools.mdx | 87 ++++++ 2 files changed, 565 insertions(+) create mode 100644 docs/cookbook/ops-diagnostics.mdx diff --git a/docs/cookbook/ops-diagnostics.mdx b/docs/cookbook/ops-diagnostics.mdx new file mode 100644 index 000000000..558d3d2dc --- /dev/null +++ b/docs/cookbook/ops-diagnostics.mdx @@ -0,0 +1,478 @@ +--- +title: "Ops Diagnostics Agent" +description: "How we built a hierarchical agent to diagnose production issues across our infrastructure" +icon: "stethoscope" +--- + +At HUD, we run a complex stack: Sentry for errors, Supabase for data, Railway for deployments, and Kubernetes for orchestration. When something breaks, we wanted an agent that could investigate across all services and provide a unified diagnosis. + +This cookbook walks through how we built it—focusing on **environment design**, **hierarchical delegation**, and **practical patterns** for production agent systems. + +## Why Hierarchical? + +When you connect multiple MCP servers to a single environment, the agent sees all tools at once. For diagnostics across four services, this meant 60+ tools in a flat list. The cognitive load made it harder for the model to select the right tool for the job. + +We restructured into a hierarchy: an orchestrator that delegates to specialized subagents. + +```mermaid +flowchart TD + subgraph orch["Orchestrator"] + O["4 subagent tools"] + end + + subgraph sentry["Sentry Agent"] + S1["search_issues"] + S2["get_issue_details"] + S3["analyze_with_seer"] + end + + subgraph supabase["Supabase Agent"] + SU1["list_tables"] + SU2["execute_sql"] + SU3["get_logs"] + end + + subgraph railway["Railway Agent"] + R1["list_projects"] + R2["get_deployments"] + R3["get_logs"] + end + + subgraph kubectl["kubectl Agent"] + K1["get_pods"] + K2["get_events"] + K3["describe_pod"] + end + + O --> sentry + O --> supabase + O --> railway + O --> kubectl +``` + +The orchestrator sees only 4 tools—one per specialist. Each specialist has a focused toolset for its domain. + +## Environment Design + +Good environment design is the foundation. Each subagent is an `Environment` with: +- A **focused toolset** (only what's needed for this domain) +- A **single scenario** that defines the interface +- **Read-only constraints** for safety + +### Connecting to MCP Servers + +For services with official MCP servers (Sentry, Supabase), connect via `connect_mcp_config`: + +```python +# environments/sentry.py +from hud import Environment +import os +import platform + +sentry_env = Environment(name="sentry-agent") + +IS_WINDOWS = platform.system() == "Windows" +token = os.getenv("SENTRY_AUTH_TOKEN") + +if token: + config = { + "command": "cmd" if IS_WINDOWS else "npx", + "args": ["/c", "npx", "-y", "@sentry/mcp-server@latest"] if IS_WINDOWS + else ["-y", "@sentry/mcp-server@latest"], + "env": {"SENTRY_ACCESS_TOKEN": token} + } + sentry_env.connect_mcp_config({"sentry": config}) +``` + +### Custom Tools When Needed + +Railway's MCP server requires browser OAuth—not ideal for headless agents. We built custom tools using their GraphQL API: + +```python +# environments/tools/railway.py +from hud.server import MCPRouter +import httpx +import os + +router = MCPRouter() +RAILWAY_API = "https://backboard.railway.com/graphql/v2" + + +async def _graphql(query: str, variables: dict | None = None) -> dict: + token = os.getenv("RAILWAY_API_TOKEN") + async with httpx.AsyncClient() as client: + resp = await client.post( + RAILWAY_API, + headers={"Authorization": f"Bearer {token}"}, + json={"query": query, "variables": variables} + ) + return resp.json() + + +@router.tool() +async def railway_list_projects() -> dict: + """List all projects with their services.""" + return await _graphql(""" + query { + projects { + edges { node { id name } } + } + } + """) + + +@router.tool() +async def railway_get_deployment_logs(deployment_id: str) -> dict: + """Get logs for a deployment.""" + return await _graphql(""" + query($id: String!) { + deploymentLogs(deploymentId: $id) { + ... on Log { message timestamp severity } + } + } + """, {"id": deployment_id}) +``` + +Then include the router in your environment: + +```python +# environments/railway.py +from hud import Environment +from .tools.railway import router + +railway_env = Environment(name="railway-agent") +railway_env.include_router(router) +``` + +### Defining the Scenario + +The scenario is the contract between orchestrator and subagent: + +```python +@sentry_env.scenario("investigate") +async def investigate_issue( + query: str, # Orchestrator provides this + expected_finding: str | None = None, # Hidden from orchestrator (eval-only) +): + """Investigate errors in Sentry.""" + + prompt = f"""You are a Sentry specialist. Investigate: + +**Query:** {query} + +**IMPORTANT: This is a READ-ONLY investigation.** + +Provide findings, root cause analysis, and recommended fixes.""" + + response = yield prompt + + # Scoring for evals + if expected_finding and response: + yield 1.0 if expected_finding.lower() in response.lower() else 0.5 + else: + yield 1.0 if response else 0.0 +``` + + +**Eval-only parameters**: Parameters with `| None = None` are automatically hidden from the orchestrator's tool schema but available for evaluation scoring. + + +## Building the Orchestrator + +The orchestrator wraps each subagent's scenario as an `AgentTool`: + +```python +# orchestrator.py +from hud import Environment +from hud.tools import AgentTool +from hud.agents import create_agent +import hud + +from environments import sentry_env, supabase_env, railway_env, kubectl_env + + +async def diagnose(query: str, model: str = "claude-sonnet-4-5"): + orchestrator = Environment(name="ops-orchestrator") + + # Wrap each subagent as a tool + for name, env, desc in [ + ("investigate_sentry", sentry_env, "Check error monitoring"), + ("investigate_supabase", supabase_env, "Check database/auth"), + ("investigate_railway", railway_env, "Check deployments"), + ("investigate_kubernetes", kubectl_env, "Check cluster health"), + ]: + tool = AgentTool( + env("investigate"), + model=model, + name=name, + description=desc, + ) + orchestrator.add_tool(tool.mcp) + + @orchestrator.scenario("diagnose") + async def run_diagnosis(issue: str): + yield f"""You are an ops diagnostics orchestrator. + +**Issue:** {issue} + +You have READ-ONLY subagents for Sentry, Supabase, Railway, and Kubernetes. +Investigate systematically and correlate findings across services.""" + + task = orchestrator("diagnose", issue=query) + + async with hud.eval(task) as ctx: + agent = create_agent(model) + return await agent.run(ctx, max_steps=20) +``` + +### Trace Continuity + +All subagent activity appears in a single trace on the HUD platform. When the orchestrator calls a subagent tool, the inference and tool calls are recorded under the parent trace—no separate URLs to track. + +## The READ-ONLY Constraint + + +We tested and operated this environment directly on our production systems, so all scenarios enforce read-only constraints. We removed mutation tools like `kubectl_exec`, `railway_redeploy`, and Supabase DDL operations. + +Every prompt includes: **"This is a READ-ONLY investigation."** + + +## Sample Output + +Running against a real production issue: + +```bash +python orchestrator.py --model claude-sonnet-4-5 \ + "Failed to delete pod: 429 Too Many Requests. 7451 events, escalating." +``` + +The orchestrator delegates to `investigate_sentry`, `investigate_railway`, and `investigate_supabase`, then correlates findings across services. After about 5 minutes: + +```text Diagnosis +COMPREHENSIVE DIAGNOSIS REPORT + +Issue Summary + - Error: Failed to delete pod ████████████████████████████████████: 429 Too Many Requests + - Impact: 7,451 events over 5 days, 16 users affected, escalating state + - Project: Orchestrator / mcp-server + - Alert ID: ORCHESTRATOR-AC + +ROOT CAUSE ANALYSIS + + Primary Root Cause: Kubernetes API Rate Limiting + + The orchestrator service is hitting Kubernetes API server rate limits when + attempting to delete pods at scale. This is occurring in the + ████████.hud_gym.utils.kubernetes module. + + Key Contributing Factors: + + 1. Excessive Deletion Frequency: ~1,491 errors/day (~62/hour) indicates + aggressive pod deletion attempts + 2. No Retry/Backoff Logic: Code lacks exponential backoff when encountering + 429 responses + 3. High Concurrency: Service runs with 50 uvicorn workers + 32 Railway + replicas, amplifying concurrent API calls + 4. Burst Traffic Pattern: Correlated with API usage spikes (313 inference + calls/minute at peak) + 5. No Client-Side Rate Limiting: Kubernetes client not configured with QPS + limits + +CORRELATED FINDINGS ACROSS SERVICES + + Sentry (Error Tracking) + - 7,455 occurrences of the 429 error between ██████████████ + - Last occurrence: ████████████████████ + - Error originates from: ████████.hud_gym.utils.kubernetes logger + - Associated with HTTP PATCH to Supabase /rest/v1/environments endpoint + - Part of environment update/cleanup workflow + + Railway (Deployment Platform) + - Production service: 32 replicas in us-west2 + - Latest successful deployment: ████████████████████ (30 min AFTER last + Sentry error) + - Historical failures (██████): AWS EKS credential issues (now resolved) + - No current rate limiting errors in deployment logs + - Pod deletions working normally post-fix + + Supabase (Database/API) + - API burst traffic spike: 313 calls/minute at ████████████████████ + - ████ Team (22 members, free tier): 15,933 inference calls/24h - prime + candidate for "16 users" + - Connection pool saturation: 49 waiting connections out of 52 + - Security vulnerabilities: 38 tables with RLS enabled but NO policies + - Performance issues: 52 unindexed foreign keys, inefficient RLS policies + - 429 errors occur at API gateway layer (not visible in Postgres logs) + + Kubernetes + - Investigation unavailable due to response size (cluster likely healthy + but under load) + +CORRELATION & TIMELINE + + ██████████████: 7,455 pod deletion failures (continuous) + - ████████████████████: Last 429 error recorded in Sentry + - ████████████████████: New production deployment (likely contained fix) + - ████████████████████: API traffic spike (313 req/min) + + Pattern Identified: + + 1. Orchestrator creates ephemeral pods for task execution (inference + workloads) + 2. High inference API traffic (15,933 calls/day from ████ team) triggers + frequent pod creation + 3. Cleanup attempts to delete pods rapidly after task completion + 4. Kubernetes API rate limits exceeded due to: + - 50 uvicorn workers x 32 Railway replicas = 1,600 concurrent processes + - No client-side rate limiting or request coordination + - No exponential backoff on 429 responses + 5. Failed deletions likely retry aggressively, compounding the problem + +RECOMMENDED ACTIONS FOR HUMANS + + CRITICAL - Immediate Actions + + 1. Implement Exponential Backoff + Location: ████████.hud_gym.utils.kubernetes + + def delete_pod_with_retry(api_client, pod_name, namespace, max_retries=5): + for attempt in range(max_retries): + try: + api_client.delete_namespaced_pod(pod_name, namespace) + return True + except ApiException as e: + if e.status == 429 and attempt < max_retries - 1: + wait_time = (2 ** attempt) + random.uniform(0, 1) + logger.warning(f"Rate limited, retrying in {wait_time:.1f}s") + time.sleep(wait_time) + continue + raise + return False + + 2. Configure Kubernetes Client Rate Limiting + + configuration = client.Configuration() + configuration.qps = 5.0 # Max 5 queries per second per client + configuration.burst = 10 # Allow bursts up to 10 + + 3. Add Concurrency Control + + k8s_api_semaphore = Semaphore(10) # Max 10 concurrent API calls + + async def delete_pod_rate_limited(pod_name, namespace): + async with k8s_api_semaphore: + return await delete_pod_with_retry(pod_name, namespace) + + HIGH PRIORITY - Within 48 Hours + + 4. Optimize Worker Configuration + - Current: 50 uvicorn workers x 32 Railway replicas = 1,600 processes + - Recommendation: Reduce uvicorn workers to 10-20 per replica + - Why: Excessive concurrency amplifies K8s API load + + 5. Implement Pod Deletion Queue + - Use background queue (Redis, Celery) for pod deletions + - Process deletions with controlled rate (e.g., 100/minute globally) + - Provides visibility into deletion backlog + + 6. Fix Supabase Security Issues + - URGENT: Add RLS policies to 38 tables currently without policies + - Enable leaked password protection + - Reduce OTP expiry to < 1 hour + - Index 52 foreign keys for query performance + - Remove 5 duplicate indexes + + 7. Upgrade ████ Team or Implement Graduated Rate Limits + - ████ team (22 members, free tier) using 15,933 API calls/day + (enterprise-level) + - Either upgrade to paid tier or implement request throttling + - Add monitoring for teams exceeding tier limits + + MEDIUM PRIORITY - Within 1 Week + + 8. Add Monitoring & Alerting + - Track pod deletion success/failure rates + - Monitor K8s API rate limit headers (X-RateLimit-Remaining) + - Alert when deletion failure rate > 5% + - Add dashboards for pod lifecycle metrics + + 9. Implement Circuit Breaker Pattern + + k8s_breaker = CircuitBreaker(fail_max=5, timeout_duration=60) + + @k8s_breaker + def delete_pod_protected(pod_name, namespace): + return delete_pod_with_retry(pod_name, namespace) + + 10. Optimize Pod Lifecycle + - Review if pods can be longer-lived (reduce churn) + - Consider pod pooling/reuse for similar tasks + - Use K8s native garbage collection where possible + - Set propagationPolicy=Background for async cleanup + + 11. Fix Supabase Connection Pool + - Switch auth server to percentage-based connection allocation + - Current: 49 waiting connections out of 52 (saturation) + - Monitor connection wait times and adjust pool size + + LOW PRIORITY - Technical Debt + + 12. Update Deprecated Dependencies + - Replace close() with aclose() for Redis connections + - Update Supabase client for new parameter configuration + - Address deprecation warnings in logs + + 13. Add Request Coalescing + - Batch multiple pod deletions into single API calls where possible + - Implement request deduplication for identical operations + +VALIDATION STEPS + + After implementing fixes, validate with: + + 1. Sentry: Monitor ORCHESTRATOR-AC for decreased error frequency (target: 0 + errors) + 2. Kubernetes: Check API server metrics for reduced throttling events + 3. Railway: Verify pod deletion logs show successful operations + 4. Supabase: Confirm API traffic patterns stay within rate limits + 5. Metrics: Track pod deletion latency and success rate + +COMMIT MESSAGE TEMPLATE + + fix: implement exponential backoff for K8s pod deletions + + - Add retry logic with exponential backoff for 429 errors + - Configure client-side rate limiting (5 QPS, 10 burst) + - Add concurrency control with semaphore (max 10 concurrent) + - Reduce uvicorn workers from 50 to 20 per replica + + Fixes ORCHESTRATOR-AC + Resolves rate limiting issues affecting 16 users over 5 days + +SUCCESS CRITERIA + + - Zero 429 errors in Sentry for 7 consecutive days + - Pod deletion success rate > 99.9% + - Average deletion latency < 2 seconds + - No user-facing impact from pod lifecycle operations + - Supabase API calls stay within tier limits + +Investigation Status: Complete +Next Review: After fix deployment (monitor for 48 hours) +``` + +The entire investigation—from initial query to actionable recommendations—took about 5 minutes across the specialized subagents. + +## What We Learned + +1. **Environment design matters.** A focused toolset per domain outperforms a flat list of everything. + +2. **Scenarios are contracts.** They define what the orchestrator can ask and what the subagent returns. + +3. **Custom tools fill gaps.** When MCP servers don't fit your auth model, build direct API integrations. + +## See Also + +- [AgentTool Reference](/reference/tools#agenttool) +- [Building Environments](/build-environments) +- [Scenarios](/reference/environments#scenarios) diff --git a/docs/reference/tools.mdx b/docs/reference/tools.mdx index 3d12e0b93..bf5b208cb 100644 --- a/docs/reference/tools.mdx +++ b/docs/reference/tools.mdx @@ -69,6 +69,93 @@ async def url_match(url: str) -> EvaluationResult: # Agents call: evaluators(name="url_match", arguments={"url": "..."}) ``` +## Agent Tools + +### AgentTool + +```python +from hud.tools import AgentTool +``` + +Wraps a scenario as a tool that can be called by another agent. Essential for building **hierarchical agent systems** where an orchestrator delegates to specialized subagents. + +**Constructor Parameters:** +| Parameter | Type | Description | Default | +|-----------|------|-------------|---------| +| `task` | `Task` | Task template from `env("scenario_name")` | Required | +| `model` | `str` | Model for subagent (via gateway) | `None` | +| `agent` | `type[MCPAgent]` | Custom agent class | `None` | +| `agent_params` | `dict` | Additional agent parameters | `{}` | +| `name` | `str` | Tool name for orchestrator | From scenario | +| `description` | `str` | Tool description | Auto-generated | +| `trace` | `bool` | Enable tracing for standalone runs | `False` | + +Must provide either `model` or `agent`, not both. + +**Eval-Only Parameters:** + +Parameters with `| None = None` are hidden from the orchestrator but available for evaluation: + +```python +@env.scenario("investigate") +async def investigate( + query: str, # Visible - orchestrator passes this + expected_finding: str | None = None, # Hidden - only used in eval scoring +): + response = yield f"Investigate: {query}" + + # Scoring uses expected_finding but orchestrator never sees it + if expected_finding and response: + yield 1.0 if expected_finding in response else 0.5 + else: + yield 1.0 if response else 0.0 +``` + +**Usage:** +```python +from hud import Environment +from hud.tools import AgentTool + +# Subagent environment with scenario +sentry_env = Environment(name="sentry-agent") + +@sentry_env.scenario("investigate") +async def investigate_sentry(query: str): + yield f"Investigate Sentry: {query}" + +# Create orchestrator +orchestrator = Environment(name="orchestrator") + +# Wrap subagent scenario as tool +tool = AgentTool( + sentry_env("investigate"), # Task template + model="gpt-4o-mini", + name="investigate_sentry", + description="Investigate errors in Sentry", +) +orchestrator.add_tool(tool.mcp) + +# Now orchestrator agent can call investigate_sentry(query="...") +``` + +**Trace Continuity:** + +When called from within an eval context, AgentTool automatically: +1. Inherits the parent's trace_id +2. Skips duplicate trace registration +3. Routes all inference/tool calls to the parent trace + +```python +async with hud.eval(task) as ctx: + agent = create_agent("gpt-4o") + result = await agent.run(ctx) + # All subagent activity appears in this single trace +``` + +**See Also:** [Ops Diagnostics Cookbook](/cookbook/ops-diagnostics) for a complete hierarchical agent example. + +--- + ## Core Tools ### BashTool From 332f42da6d0ad71e884c1028ba10fbb754263691 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Wed, 7 Jan 2026 23:02:04 -0800 Subject: [PATCH 17/27] docs link --- docs/docs.json | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/docs/docs.json b/docs/docs.json index 71e69e115..69b9a671a 100644 --- a/docs/docs.json +++ b/docs/docs.json @@ -60,6 +60,12 @@ "migration" ] }, + { + "group": "Cookbook", + "pages": [ + "cookbook/ops-diagnostics" + ] + }, { "group": "Advanced", "pages": [ From 5325d2986598dd8d91befe07aa7a1c4d7d0a55c6 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 12:22:51 -0800 Subject: [PATCH 18/27] fix env connector --- hud/environment/environment.py | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/hud/environment/environment.py b/hud/environment/environment.py index 83924cd2e..79f57dffc 100644 --- a/hud/environment/environment.py +++ b/hud/environment/environment.py @@ -362,6 +362,22 @@ async def __aexit__( await asyncio.gather(*[c.disconnect() for c in self._connections.values()]) self._router.clear() + async def run_async( + self, + transport: Literal["stdio", "http", "sse"] | None = None, + show_banner: bool = True, + **transport_kwargs: Any, + ) -> None: + """Run the MCP server, auto-connecting all connectors first. + + This ensures that tools from external MCP servers (via connect_mcp_config) + are discovered and available when the server starts. + """ + async with self: # Connect all connectors via __aenter__ + await super().run_async( + transport=transport, show_banner=show_banner, **transport_kwargs + ) + async def _build_routing(self) -> None: """Build tool routing from local tools and connection caches.""" # Use get_tools() not list_tools() - it includes mounted servers without From f17a93b11d1350a6dc5a9f1715c39bd3093e1892 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 12:53:56 -0800 Subject: [PATCH 19/27] add routing and tools updates for remote --- hud/environment/environment.py | 15 +++++++++++++++ hud/tools/agent.py | 13 ++++++++----- 2 files changed, 23 insertions(+), 5 deletions(-) diff --git a/hud/environment/environment.py b/hud/environment/environment.py index 79f57dffc..bf9927d3f 100644 --- a/hud/environment/environment.py +++ b/hud/environment/environment.py @@ -392,6 +392,21 @@ async def _build_routing(self) -> None: # Populate mock schemas for auto-generated mock values self._populate_mock_schemas() + # ========================================================================= + # MCP Protocol Overrides - Include connector tools in MCP responses + # ========================================================================= + + def _mcp_list_tools(self) -> list[mcp_types.Tool]: + """Override FastMCP to return all tools including those from connectors.""" + return self._router.tools + + async def _mcp_call_tool( + self, key: str, arguments: dict[str, Any] + ) -> list[Any] | tuple[list[Any], dict[str, Any]]: + """Override FastMCP to route tool calls through our router.""" + result = await self._execute_tool(key, arguments) + return result.content or [] + # ========================================================================= # Tool Operations # ========================================================================= diff --git a/hud/tools/agent.py b/hud/tools/agent.py index 3932f5191..2f5ad3771 100644 --- a/hud/tools/agent.py +++ b/hud/tools/agent.py @@ -138,11 +138,14 @@ def _build_schema(self, params: dict[str, inspect.Parameter]) -> dict[str, Any]: try: annotation = eval(annotation) # noqa: S307 except Exception: - properties[name] = {"type": "string"} - continue - - adapter = TypeAdapter(annotation) - properties[name] = adapter.json_schema() + # Fall back to string type but don't skip required handling + annotation = None + + if annotation is not None: + adapter = TypeAdapter(annotation) + properties[name] = adapter.json_schema() + else: + properties[name] = {"type": "string"} except Exception: properties[name] = {"type": "string"} else: From c4188d2ee5de4574a728f7104dc071d5f2caece2 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 13:18:45 -0800 Subject: [PATCH 20/27] add tests to remote connectors and improve connection --- hud/environment/environment.py | 22 ++-- hud/environment/tests/test_environment.py | 147 ++++++++++++++++++++++ 2 files changed, 162 insertions(+), 7 deletions(-) diff --git a/hud/environment/environment.py b/hud/environment/environment.py index bf9927d3f..e14357ca0 100644 --- a/hud/environment/environment.py +++ b/hud/environment/environment.py @@ -396,15 +396,23 @@ async def _build_routing(self) -> None: # MCP Protocol Overrides - Include connector tools in MCP responses # ========================================================================= - def _mcp_list_tools(self) -> list[mcp_types.Tool]: - """Override FastMCP to return all tools including those from connectors.""" + def _setup_handlers(self) -> None: + """Override FastMCP to register our custom handlers for tools.""" + # Call parent to set up all standard handlers + super()._setup_handlers() + # Re-register our custom handlers (overwrites parent's registrations) + self._mcp_server.list_tools()(self._env_list_tools) + self._mcp_server.call_tool()(self._env_call_tool) + + async def _env_list_tools(self) -> list[mcp_types.Tool]: + """Return all tools including those from connectors.""" return self._router.tools - async def _mcp_call_tool( - self, key: str, arguments: dict[str, Any] - ) -> list[Any] | tuple[list[Any], dict[str, Any]]: - """Override FastMCP to route tool calls through our router.""" - result = await self._execute_tool(key, arguments) + async def _env_call_tool( + self, name: str, arguments: dict[str, Any] | None = None + ) -> list[Any]: + """Route tool calls through our router (handles both local and connector tools).""" + result = await self._execute_tool(name, arguments or {}) return result.content or [] # ========================================================================= diff --git a/hud/environment/tests/test_environment.py b/hud/environment/tests/test_environment.py index 44febe88d..04e2bc997 100644 --- a/hud/environment/tests/test_environment.py +++ b/hud/environment/tests/test_environment.py @@ -159,3 +159,150 @@ def test_chaining_multiple_setup_calls(self) -> None: ) assert len(env._setup_calls) == 2 + + +class TestEnvironmentMCPProtocol: + """Tests for MCP protocol overrides (_mcp_list_tools, _mcp_call_tool).""" + + @pytest.mark.asyncio + async def test_mcp_list_tools_includes_local_tools(self) -> None: + """_mcp_list_tools returns local tools.""" + from hud.environment import Environment + + env = Environment("test") + + @env.tool() + def my_tool(x: int) -> int: + """A test tool.""" + return x * 2 + + # Build routing manually without full context (avoids import issues) + await env._build_routing() + tools = env._mcp_list_tools() + + assert len(tools) == 1 + assert tools[0].name == "my_tool" + + @pytest.mark.asyncio + async def test_mcp_list_tools_includes_connector_tools(self) -> None: + """_mcp_list_tools returns tools from connectors.""" + from unittest.mock import AsyncMock, PropertyMock + + import mcp.types as mcp_types + + from hud.environment import Environment + + env = Environment("test") + + # Create a mock connector with cached tools + mock_tools = [ + mcp_types.Tool( + name="remote_tool", + description="A remote tool", + inputSchema={"type": "object"}, + ) + ] + + class MockConnector: + is_connected = True + _tools_cache = mock_tools + + @property + def cached_tools(self) -> list[mcp_types.Tool]: + return self._tools_cache + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def list_tools(self) -> list[mcp_types.Tool]: + return self._tools_cache + + # Add the mock connector + env._connections["mock"] = MockConnector() # type: ignore + + # Build routing manually + await env._build_routing() + tools = env._mcp_list_tools() + + # Should include the remote tool + tool_names = [t.name for t in tools] + assert "remote_tool" in tool_names + + @pytest.mark.asyncio + async def test_mcp_call_tool_routes_to_local(self) -> None: + """_mcp_call_tool routes local tool calls correctly.""" + from hud.environment import Environment + + env = Environment("test") + called_with: list[int] = [] + + @env.tool() + def my_tool(x: int) -> str: + """A test tool.""" + called_with.append(x) + return f"result: {x}" + + # Build routing manually without full context + await env._build_routing() + result = await env._mcp_call_tool("my_tool", {"x": 42}) + + assert called_with == [42] + assert len(result) == 1 + + @pytest.mark.asyncio + async def test_mcp_call_tool_routes_to_connector(self) -> None: + """_mcp_call_tool routes connector tool calls correctly.""" + from unittest.mock import AsyncMock + + import mcp.types as mcp_types + + from hud.environment import Environment + from hud.types import MCPToolResult + + env = Environment("test") + + # Create a mock connector + mock_tools = [ + mcp_types.Tool( + name="remote_tool", + description="A remote tool", + inputSchema={"type": "object"}, + ) + ] + + class MockConnector: + is_connected = True + _tools_cache = mock_tools + call_tool = AsyncMock( + return_value=MCPToolResult( + content=[mcp_types.TextContent(type="text", text="remote result")], + isError=False, + ) + ) + + @property + def cached_tools(self) -> list[mcp_types.Tool]: + return self._tools_cache + + async def connect(self) -> None: + pass + + async def disconnect(self) -> None: + pass + + async def list_tools(self) -> list[mcp_types.Tool]: + return self._tools_cache + + mock_conn = MockConnector() + env._connections["mock"] = mock_conn # type: ignore + + # Build routing manually without full context + await env._build_routing() + result = await env._mcp_call_tool("remote_tool", {"arg": "value"}) + + # Verify the connector was called + mock_conn.call_tool.assert_called_once_with("remote_tool", {"arg": "value"}) + assert len(result) == 1 \ No newline at end of file From 9f95e0fc5f3ee331248603e2493806e04b7c129f Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 13:20:17 -0800 Subject: [PATCH 21/27] more precise tests --- hud/environment/tests/test_environment.py | 61 +++++++++++++++-------- 1 file changed, 41 insertions(+), 20 deletions(-) diff --git a/hud/environment/tests/test_environment.py b/hud/environment/tests/test_environment.py index 04e2bc997..96d9cbb24 100644 --- a/hud/environment/tests/test_environment.py +++ b/hud/environment/tests/test_environment.py @@ -162,11 +162,14 @@ def test_chaining_multiple_setup_calls(self) -> None: class TestEnvironmentMCPProtocol: - """Tests for MCP protocol overrides (_mcp_list_tools, _mcp_call_tool).""" + """Tests for MCP protocol overrides - Environment._env_list_tools and _env_call_tool. + + These test that Environment properly exposes connector tools via MCP handlers. + """ @pytest.mark.asyncio - async def test_mcp_list_tools_includes_local_tools(self) -> None: - """_mcp_list_tools returns local tools.""" + async def test_env_list_tools_includes_local_tools(self) -> None: + """_env_list_tools returns local tools after routing is built.""" from hud.environment import Environment env = Environment("test") @@ -176,18 +179,18 @@ def my_tool(x: int) -> int: """A test tool.""" return x * 2 - # Build routing manually without full context (avoids import issues) + # Build routing (simulates what __aenter__ does) await env._build_routing() - tools = env._mcp_list_tools() + + # Call the handler that MCP will call + tools = await env._env_list_tools() assert len(tools) == 1 assert tools[0].name == "my_tool" @pytest.mark.asyncio - async def test_mcp_list_tools_includes_connector_tools(self) -> None: - """_mcp_list_tools returns tools from connectors.""" - from unittest.mock import AsyncMock, PropertyMock - + async def test_env_list_tools_includes_connector_tools(self) -> None: + """_env_list_tools returns tools from connectors (the key feature).""" import mcp.types as mcp_types from hud.environment import Environment @@ -223,17 +226,19 @@ async def list_tools(self) -> list[mcp_types.Tool]: # Add the mock connector env._connections["mock"] = MockConnector() # type: ignore - # Build routing manually + # Build routing await env._build_routing() - tools = env._mcp_list_tools() + + # Call the handler that MCP will call + tools = await env._env_list_tools() # Should include the remote tool tool_names = [t.name for t in tools] assert "remote_tool" in tool_names @pytest.mark.asyncio - async def test_mcp_call_tool_routes_to_local(self) -> None: - """_mcp_call_tool routes local tool calls correctly.""" + async def test_env_call_tool_routes_to_local(self) -> None: + """_env_call_tool routes local tool calls correctly.""" from hud.environment import Environment env = Environment("test") @@ -245,16 +250,18 @@ def my_tool(x: int) -> str: called_with.append(x) return f"result: {x}" - # Build routing manually without full context + # Build routing await env._build_routing() - result = await env._mcp_call_tool("my_tool", {"x": 42}) + + # Call the handler that MCP will call + result = await env._env_call_tool("my_tool", {"x": 42}) assert called_with == [42] assert len(result) == 1 @pytest.mark.asyncio - async def test_mcp_call_tool_routes_to_connector(self) -> None: - """_mcp_call_tool routes connector tool calls correctly.""" + async def test_env_call_tool_routes_to_connector(self) -> None: + """_env_call_tool routes connector tool calls correctly.""" from unittest.mock import AsyncMock import mcp.types as mcp_types @@ -299,10 +306,24 @@ async def list_tools(self) -> list[mcp_types.Tool]: mock_conn = MockConnector() env._connections["mock"] = mock_conn # type: ignore - # Build routing manually without full context + # Build routing await env._build_routing() - result = await env._mcp_call_tool("remote_tool", {"arg": "value"}) + + # Call the handler that MCP will call + result = await env._env_call_tool("remote_tool", {"arg": "value"}) # Verify the connector was called mock_conn.call_tool.assert_called_once_with("remote_tool", {"arg": "value"}) - assert len(result) == 1 \ No newline at end of file + assert len(result) == 1 + + def test_setup_handlers_registers_custom_handlers(self) -> None: + """Verify _setup_handlers registers our _env_list_tools and _env_call_tool.""" + from hud.environment import Environment + + env = Environment("test") + + # Verify the custom handlers exist + assert hasattr(env, "_env_list_tools") + assert hasattr(env, "_env_call_tool") + assert callable(env._env_list_tools) + assert callable(env._env_call_tool) \ No newline at end of file From f9e18eb8a6909d200e2ebad3f8569031550e95c0 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 15:23:20 -0800 Subject: [PATCH 22/27] fix: strip format field from JSON schemas for OpenAI strict mode --- hud/utils/strict_schema.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hud/utils/strict_schema.py b/hud/utils/strict_schema.py index 5d3fa0daa..263919b3e 100644 --- a/hud/utils/strict_schema.py +++ b/hud/utils/strict_schema.py @@ -118,7 +118,7 @@ def _ensure_strict_json_schema( if "default" in json_schema: json_schema.pop("default") - for keyword in ("title", "examples"): + for keyword in ("title", "examples", "format"): json_schema.pop(keyword, None) ref = json_schema.get("$ref") From f3c9e0c9baf0ae9e404665f46bfb2799e16a6b0e Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 16:35:03 -0800 Subject: [PATCH 23/27] move --- docs/{cookbook => cookbooks}/ops-diagnostics.mdx | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename docs/{cookbook => cookbooks}/ops-diagnostics.mdx (100%) diff --git a/docs/cookbook/ops-diagnostics.mdx b/docs/cookbooks/ops-diagnostics.mdx similarity index 100% rename from docs/cookbook/ops-diagnostics.mdx rename to docs/cookbooks/ops-diagnostics.mdx From 2f67cde08d13903072171fb9145468d50e9cb2c2 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 16:35:08 -0800 Subject: [PATCH 24/27] Merge main into feat/scenario-improvements, combine cookbooks --- SheetBench-50 copy.json | 4211 +++++++++++++++++++++++++++++++++++++++ SheetBench-50.json | 4211 +++++++++++++++++++++++++++++++++++++++ sheetbench_tasks.json | 2411 ++++++++++++++++++++++ 3 files changed, 10833 insertions(+) create mode 100644 SheetBench-50 copy.json create mode 100644 SheetBench-50.json create mode 100644 sheetbench_tasks.json diff --git a/SheetBench-50 copy.json b/SheetBench-50 copy.json new file mode 100644 index 000000000..fcaf77191 --- /dev/null +++ b/SheetBench-50 copy.json @@ -0,0 +1,4211 @@ +[ + { + "prompt": "Calculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1/12/2024", + "A2": "1/10/2024", + "A3": "1/15/2024", + "A4": "1/11/2024", + "A5": "1/17/2024" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Calculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B2": "CustomerID", + "B3": "476232", + "B4": "963589", + "B5": "1430820", + "B6": "2410043", + "B7": "3789483", + "B8": "4226444", + "B9": "4308096", + "C2": "2023", + "C5": "686.40", + "C8": "966.76", + "C9": "863.37", + "D2": "2024", + "D3": "912.70", + "D4": "397.62", + "D6": "383.38", + "D7": "619.56", + "E2": "Grand Total", + "E3": "912.70", + "E4": "397.62", + "E5": "686.40", + "E6": "383.38", + "E7": "619.56", + "E8": "966.76", + "E9": "863.37", + "B10": "5907496", + "B11": "6224271", + "B12": "6538101", + "B13": "6584993", + "B14": "6791810", + "B15": "7250781", + "B16": "8518187", + "B17": "8668097", + "B18": "8885050", + "B19": "9296053", + "B20": "10414682", + "B21": "10839621", + "B22": "12313143", + "B23": "13672423", + "B24": "14744105", + "B25": "14780608", + "B26": "15144539", + "B27": "15616289", + "B28": "15694782", + "B29": "15904102", + "B30": "16385696", + "B31": "17254618", + "B32": "17441704", + "B33": "17624107", + "B34": "18043433", + "B35": "18296075", + "B36": "18600797", + "B37": "19026954", + "B38": "19395319", + "B39": "19650142", + "B40": "19842318", + "B41": "20241148", + "B42": "20587659", + "B43": "21647994", + "B44": "21671104", + "B45": "21935183", + "B46": "22115836", + "B47": "22846037", + "B48": "23621954", + "B49": "24962520", + "B50": "25073201", + "B51": "25090031", + "B52": "25096194", + "B53": "25211207", + "B54": "26168579", + "B55": "26250755", + "B56": "26419282", + "B57": "26953175", + "B58": "27281086", + "B59": "27646724", + "B60": "30370166", + "B61": "30738270", + "B62": "30876644", + "B63": "32255126", + "B64": "32373769", + "B65": "33121017", + "B66": "33897455", + "B67": "34507001", + "B68": "35475491", + "B69": "36972111", + "B70": "37040802", + "B71": "37543198", + "B72": "37609900", + "B73": "37674649", + "B74": "38019388", + "B75": "38658747", + "B76": "38793189", + "B77": "39134950", + "B78": "40399799", + "B79": "41039038", + "B80": "41271414", + "B81": "41870334", + "B82": "42495130", + "B83": "43189090", + "B84": "43235421", + "B85": "43837144", + "B86": "44115166", + "B87": "44270797", + "B88": "45380457", + "B89": "45386282", + "B90": "46196026", + "B91": "46300157", + "B92": "48512428", + "B93": "49546866", + "B94": "49687477", + "B95": "50893874", + "B96": "51093532", + "B97": "51126698", + "B98": "51397982", + "B99": "52376160", + "C10": "714.36", + "C11": "369.01", + "C12": "525.20", + "C20": "120.72", + "C21": "269.30", + "C22": "422.36", + "C26": "971.83", + "C28": "110.65", + "C29": "496.66", + "C30": "841.89", + "C31": "886.06", + "C32": "536.09", + "C34": "16.13", + "C38": "90.40", + "C39": "76.23", + "C40": "185.33", + "C41": "65.32", + "C42": "111.51", + "C43": "493.89", + "C44": "787.04", + "C45": "578.58", + "C46": "204.48", + "C48": "985.49", + "C50": "268.33", + "C52": "34.51", + "C54": "61.19", + "C60": "62.83", + "C63": "326.11", + "C64": "898.66", + "C65": "251.17", + "C66": "902.46", + "C67": "442.33", + "C70": "561.10", + "C71": "469.05", + "C74": "477.14", + "C76": "284.69", + "C80": "266.21", + "C82": "699.49", + "C85": "82.73", + "C87": "542.97", + "C88": "446.42", + "C89": "380.14", + "C91": "683.73", + "C92": "566.99", + "C94": "646.75", + "C97": "748.88", + "C98": "253.40", + "D13": "174.59", + "D14": "374.65", + "D15": "339.03", + "D16": "140.66", + "D17": "862.92", + "D18": "282.39", + "D19": "847.67", + "D23": "616.14", + "D24": "376.83", + "D25": "523.78", + "D27": "701.88", + "D29": "496.66", + "D33": "805.83", + "D35": "579.30", + "D36": "577.84", + "D37": "673.27", + "D38": "90.40", + "D47": "47.03", + "D48": "985.49", + "D49": "779.72", + "D51": "277.24", + "D53": "702.84", + "D55": "334.42", + "D56": "875.84", + "D57": "570.46", + "D58": "60.52", + "D59": "29.15", + "D61": "822.61", + "D62": "931.36", + "D68": "728.49", + "D69": "454.44", + "D72": "866.51", + "D73": "968.06", + "D74": "477.14", + "D75": "154.14", + "D77": "69.76", + "D78": "323.27", + "D79": "862.51", + "D81": "335.73", + "D83": "555.46", + "D84": "685.07", + "D86": "942.65", + "D90": "813.06", + "D93": "671.93", + "D95": "995.04", + "D96": "896.64", + "D98": "253.40", + "D99": "66.10", + "E10": "714.36", + "E11": "369.01", + "E12": "525.20", + "E13": "174.59", + "E14": "374.65", + "E15": "339.03", + "E16": "140.66", + "E17": "862.92", + "E18": "282.39", + "E19": "847.67", + "E20": "120.72", + "E21": "269.30", + "E22": "422.36", + "E23": "616.14", + "E24": "376.83", + "E25": "523.78", + "E26": "971.83", + "E27": "701.88", + "E28": "110.65", + "E29": "993.32", + "E30": "841.89", + "E31": "886.06", + "E32": "536.09", + "E33": "805.83", + "E34": "16.13", + "E35": "579.30", + "E36": "577.84", + "E37": "673.27", + "E38": "180.80", + "E39": "76.23", + "E40": "185.33", + "E41": "65.32", + "E42": "111.51", + "E43": "493.89", + "E44": "787.04", + "E45": "578.58", + "E46": "204.48", + "E47": "47.03", + "E48": "1970.98", + "E49": "779.72", + "E50": "268.33", + "E51": "277.24", + "E52": "34.51", + "E53": "702.84", + "E54": "61.19", + "E55": "334.42", + "E56": "875.84", + "E57": "570.46", + "E58": "60.52", + "E59": "29.15", + "E60": "62.83", + "E61": "822.61", + "E62": "931.36", + "E63": "326.11", + "E64": "898.66", + "E65": "251.17", + "E66": "902.46", + "E67": "442.33", + "E68": "728.49", + "E69": "454.44", + "E70": "561.10", + "E71": "469.05", + "E72": "866.51", + "E73": "968.06", + "E74": "954.28", + "E75": "154.14", + "E76": "284.69", + "E77": "69.76", + "E78": "323.27", + "E79": "862.51", + "E80": "266.21", + "E81": "335.73", + "E82": "699.49", + "E83": "555.46", + "E84": "685.07", + "E85": "82.73", + "E86": "942.65", + "E87": "542.97", + "E88": "446.42", + "E89": "380.14", + "E90": "813.06", + "E91": "683.73", + "E92": "566.99", + "E93": "671.93", + "E94": "646.75", + "E95": "995.04", + "E96": "896.64", + "E97": "748.88", + "E98": "506.80", + "E99": "66.10", + "B100": "52697630", + "B101": "52870804", + "B102": "53239020", + "B103": "53335630", + "B104": "53858146", + "B105": "54040755", + "B106": "54479395", + "B107": "54528192", + "B108": "54730083", + "B109": "55444897", + "B110": "55450065", + "B111": "55853989", + "B112": "55859479", + "B113": "55978545", + "B114": "56702798", + "B115": "56940756", + "B116": "58152560", + "B117": "59169574", + "B118": "59736806", + "B119": "61602489", + "B120": "62665919", + "B121": "62758192", + "B122": "62943277", + "B123": "63051186", + "B124": "65657133", + "B125": "65899458", + "B126": "66761945", + "B127": "66846840", + "B128": "67283606", + "B129": "67548873", + "B130": "69613838", + "B131": "69765117", + "B132": "70098167", + "B133": "70534368", + "B134": "71078229", + "B135": "71319902", + "B136": "71369648", + "B137": "71376141", + "B138": "72217443", + "B139": "72659809", + "B140": "72883709", + "B141": "73191421", + "B142": "73684220", + "B143": "74014279", + "B144": "74088126", + "B145": "75410476", + "B146": "75817527", + "B147": "77140593", + "B148": "77558800", + "B149": "78916470", + "B150": "79031936", + "B151": "79751742", + "B152": "80286530", + "B153": "80765899", + "B154": "82060237", + "B155": "82306595", + "B156": "83101113", + "B157": "83211478", + "B158": "83713620", + "B159": "84770820", + "B160": "84800206", + "B161": "84952943", + "B162": "86407021", + "B163": "86619158", + "B164": "86663007", + "B165": "87144451", + "B166": "87254792", + "B167": "88194202", + "B168": "88266169", + "B169": "88761732", + "B170": "88882642", + "B171": "89421277", + "B172": "89565544", + "B173": "90841330", + "B174": "91483447", + "B175": "91590435", + "B176": "91939241", + "B177": "92335820", + "B178": "92422728", + "B179": "92676749", + "B180": "93202190", + "B181": "93479509", + "B182": "95353791", + "B183": "95696393", + "B184": "95804583", + "B185": "95860510", + "B186": "96051566", + "B187": "96263709", + "B188": "96456101", + "B189": "99519803", + "B190": "Grand Total", + "C100": "409.46", + "C101": "28.08", + "C106": "184.81", + "C107": "906.16", + "C108": "1094.00", + "C109": "818.87", + "C110": "192.68", + "C111": "385.72", + "C117": "976.44", + "C118": "310.80", + "C119": "354.99", + "C120": "898.04", + "C122": "397.81", + "C128": "537.90", + "C129": "886.96", + "C131": "420.66", + "C133": "359.50", + "C134": "60.29", + "C139": "740.19", + "C142": "554.52", + "C144": "759.21", + "C145": "116.62", + "C148": "302.90", + "C152": "187.16", + "C153": "847.55", + "C154": "700.62", + "C155": "190.59", + "C161": "432.99", + "C163": "886.00", + "C167": "715.20", + "C169": "101.82", + "C171": "122.95", + "C174": "609.90", + "C175": "867.14", + "C177": "675.11", + "C178": "623.53", + "C182": "40.39", + "C184": "554.62", + "C187": "839.28", + "C189": "717.61", + "C190": "43541.41", + "D102": "995.08", + "D103": "761.74", + "D104": "773.13", + "D105": "364.27", + "D112": "841.73", + "D113": "560.15", + "D114": "824.63", + "D115": "783.50", + "D116": "788.60", + "D117": "976.44", + "D118": "310.80", + "D121": "92.40", + "D123": "171.84", + "D124": "336.61", + "D125": "206.92", + "D126": "345.03", + "D127": "505.00", + "D130": "539.76", + "D132": "189.69", + "D135": "713.31", + "D136": "993.98", + "D137": "541.38", + "D138": "790.14", + "D139": "740.19", + "D140": "481.88", + "D141": "83.31", + "D143": "883.45", + "D146": "394.60", + "D147": "281.52", + "D149": "64.48", + "D150": "645.82", + "D151": "771.06", + "D156": "405.74", + "D157": "741.29", + "D158": "196.84", + "D159": "152.71", + "D160": "356.93", + "D162": "138.69", + "D164": "929.08", + "D165": "887.13", + "D166": "554.21", + "D168": "820.22", + "D170": "939.97", + "D171": "122.95", + "D172": "335.69", + "D173": "917.55", + "D176": "514.59", + "D179": "288.94", + "D180": "214.12", + "D181": "16.74", + "D183": "27.28", + "D185": "458.58", + "D186": "648.09", + "D188": "985.01", + "D190": "56717.97", + "E100": "409.46", + "E101": "28.08", + "E102": "995.08", + "E103": "761.74", + "E104": "773.13", + "E105": "364.27", + "E106": "184.81", + "E107": "906.16", + "E108": "1094.00", + "E109": "818.87", + "E110": "192.68", + "E111": "385.72", + "E112": "841.73", + "E113": "560.15", + "E114": "824.63", + "E115": "783.50", + "E116": "788.60", + "E117": "1952.88", + "E118": "621.60", + "E119": "354.99", + "E120": "898.04", + "E121": "92.40", + "E122": "397.81", + "E123": "171.84", + "E124": "336.61", + "E125": "206.92", + "E126": "345.03", + "E127": "505.00", + "E128": "537.90", + "E129": "886.96", + "E130": "539.76", + "E131": "420.66", + "E132": "189.69", + "E133": "359.50", + "E134": "60.29", + "E135": "713.31", + "E136": "993.98", + "E137": "541.38", + "E138": "790.14", + "E139": "1480.38", + "E140": "481.88", + "E141": "83.31", + "E142": "554.52", + "E143": "883.45", + "E144": "759.21", + "E145": "116.62", + "E146": "394.60", + "E147": "281.52", + "E148": "302.90", + "E149": "64.48", + "E150": "645.82", + "E151": "771.06", + "E152": "187.16", + "E153": "847.55", + "E154": "700.62", + "E155": "190.59", + "E156": "405.74", + "E157": "741.29", + "E158": "196.84", + "E159": "152.71", + "E160": "356.93", + "E161": "432.99", + "E162": "138.69", + "E163": "886.00", + "E164": "929.08", + "E165": "887.13", + "E166": "554.21", + "E167": "715.20", + "E168": "820.22", + "E169": "101.82", + "E170": "939.97", + "E171": "245.90", + "E172": "335.69", + "E173": "917.55", + "E174": "609.90", + "E175": "867.14", + "E176": "514.59", + "E177": "675.11", + "E178": "623.53", + "E179": "288.94", + "E180": "214.12", + "E181": "16.74", + "E182": "40.39", + "E183": "27.28", + "E184": "554.62", + "E185": "458.58", + "E186": "648.09", + "E187": "839.28", + "E188": "985.01", + "E189": "717.61", + "E190": "100259.38" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Find the GET request which most commonly results in an error. Place the URL in ANSWER!A1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "/api/users" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "For a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2026-04" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "For the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0963d367-f0ac-4be0-8f46-337bf335e68f", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "ABC", + "B1": "4999972", + "C1": "145%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$112,281.49" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "49ccea4d-4aaf-4faf-9659-b389686568a7", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$24,642.86" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "48d43d57-ed1d-4df0-8b36-62380bba7865", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1664934.45" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "ff114e80-196d-4a7a-99ca-152bac4fba90", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "MAY", + "A2": "OCT", + "B1": "95", + "B2": "73" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Category", + "A2": "Meals & Entertainment", + "A3": "Travel", + "A4": "Office Supplies", + "B1": "Sum of Amount", + "B2": "0", + "B3": "345", + "B4": "45760" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A3": "Sum of Amount (USD Millions)", + "A5": "Net Income", + "A6": "Revenue", + "A7": "Total Assets", + "A8": "Grand Total", + "B4": "FORD2024Q1", + "B5": "1.33", + "B6": "42.78", + "B7": "274.34", + "B8": "318.45", + "C4": "FORD2024Q2", + "C5": "1.83", + "C6": "44.81", + "C7": "276.59", + "C8": "323.23", + "D4": "FORD2024Q3", + "D5": "896.00", + "D6": "43.07", + "D7": "287.05", + "D8": "1226.12", + "E4": "FORD2025Q1", + "E5": "471.00", + "E6": "40.66", + "E7": "284.54", + "E8": "796.20", + "F4": "Grand Total", + "F5": "1370.17", + "F6": "171.32", + "F7": "1122.51", + "F8": "2664.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "C3": "Count of TicketID", + "D3": "Average of ResolutionTimeHours", + "A4": "Agent-101", + "B4": "Closed", + "C4": "73", + "D4": "13.90", + "B5": "In Progress", + "C5": "30", + "B6": "Open", + "C6": "31", + "B7": "Resolved", + "C7": "66", + "D7": "22.04", + "B8": "Waiting for Customer", + "C8": "12", + "C9": "212", + "D9": "17.76", + "A10": "Agent-102", + "B10": "Closed", + "C10": "70", + "D10": "28.53", + "B11": "In Progress", + "C11": "36", + "B12": "Open", + "C12": "32", + "B13": "Resolved", + "C13": "77", + "D13": "17.09", + "B14": "Waiting for Customer", + "C14": "10", + "C15": "225", + "D15": "22.54", + "A16": "Agent-103", + "B16": "Closed", + "C16": "73", + "D16": "23.11", + "B17": "In Progress", + "C17": "39", + "B18": "Open", + "C18": "14", + "B19": "Resolved", + "C19": "92", + "D19": "17.08", + "B20": "Waiting for Customer", + "C20": "11", + "C21": "229", + "D21": "19.75", + "A22": "Agent-104", + "B22": "Closed", + "C22": "63", + "D22": "15.48", + "B23": "In Progress", + "C23": "32", + "B24": "Open", + "C24": "18", + "B25": "Resolved", + "C25": "97", + "D25": "15.40", + "B26": "Waiting for Customer", + "C26": "13", + "C27": "223", + "D27": "15.43", + "A28": "Agent-105", + "B28": "Closed", + "C28": "70", + "D28": "22.20", + "B29": "In Progress", + "C29": "27", + "B30": "Open", + "C30": "16", + "B31": "Resolved", + "C31": "69", + "D31": "15.21", + "B32": "Waiting for Customer", + "C32": "10", + "C33": "192", + "D33": "18.73", + "A34": "Agent-106", + "B34": "Closed", + "C34": "65", + "D34": "26.15", + "B35": "In Progress", + "C35": "38", + "B36": "Open", + "C36": "20", + "B37": "Resolved", + "C37": "87", + "D37": "21.56", + "B38": "Waiting for Customer", + "C38": "10", + "C39": "220", + "D39": "23.52", + "A40": "Agent-107", + "B40": "Closed", + "C40": "68", + "D40": "18.41", + "B41": "In Progress", + "C41": "35", + "B42": "Open", + "C42": "17", + "B43": "Resolved", + "C43": "106", + "D43": "16.52", + "B44": "Waiting for Customer", + "C44": "11", + "C45": "237", + "D45": "17.26", + "A46": "Agent-108", + "B46": "Closed", + "C46": "65", + "D46": "16.81", + "B47": "In Progress", + "C47": "33", + "B48": "Open", + "C48": "20", + "B49": "Resolved", + "C49": "93", + "D49": "14.93", + "B50": "Waiting for Customer", + "C50": "8", + "C51": "219", + "D51": "15.70", + "B52": "(blank)", + "C52": "243", + "D52": "20.98", + "B53": "Closed", + "C53": "74", + "D53": "24.44", + "B54": "In Progress", + "C54": "32", + "B55": "Open", + "C55": "26", + "B56": "Resolved", + "C56": "99", + "D56": "18.39", + "B57": "Waiting for Customer", + "C57": "12", + "B58": "Grand Total", + "C58": "2000", + "D58": "19.05" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "7598db5d-0fd1-44db-ab1a-593cf026317b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B2": "1", + "B3": "2", + "B4": "3", + "B5": "4", + "B6": "5", + "B8": "1", + "B9": "2", + "C1": "Count of LastPromotionDate", + "C2": "12", + "C3": "13", + "C4": "10", + "C5": "15", + "C6": "5", + "C8": "1", + "C9": "5", + "D1": "Average of SalaryUSD", + "D2": "84965.91", + "D3": "87028.38", + "D4": "86094.90", + "D5": "78111.64", + "D6": "86015.00", + "D8": "82727.50", + "D9": "78876.00", + "A38": "Grand Total", + "B10": "3", + "B11": "4", + "B12": "5", + "B14": "1", + "B15": "2", + "B16": "3", + "B17": "4", + "B18": "5", + "B20": "1", + "B21": "2", + "B22": "3", + "B23": "4", + "B24": "5", + "B26": "1", + "B27": "2", + "B28": "3", + "B29": "4", + "B30": "5", + "B32": "1", + "B33": "2", + "B34": "3", + "B35": "4", + "B36": "5", + "C10": "4", + "C15": "3", + "C16": "2", + "C17": "2", + "C20": "1", + "C21": "4", + "C22": "6", + "C23": "3", + "C24": "5", + "C26": "10", + "C27": "9", + "C28": "4", + "C29": "6", + "C30": "4", + "C32": "5", + "C33": "2", + "C34": "4", + "C35": "4", + "C36": "1", + "C38": "140", + "D10": "76759.43", + "D11": "78318.00", + "D12": "80036.25", + "D14": "76599.00", + "D15": "93749.17", + "D16": "79929.00", + "D17": "88994.75", + "D18": "102902.00", + "D20": "77878.50", + "D21": "77480.38", + "D22": "84212.67", + "D23": "77508.83", + "D24": "77182.50", + "D26": "56034.08", + "D27": "52096.20", + "D28": "63190.71", + "D29": "60750.00", + "D30": "50299.25", + "D32": "71131.57", + "D33": "89669.50", + "D34": "81155.50", + "D35": "82037.80", + "D36": "61600.50", + "D38": "77770.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": " 27,301,058.62 " + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Year", + "A2": "2025", + "A3": "2026", + "A4": "2027", + "A5": "2028", + "A6": "2029", + "A9": "Year", + "B1": "Revenue", + "B2": "108,000,000.00", + "B3": "116,640,000.00", + "B4": "125,971,200.00", + "B5": "136,048,896.00", + "B6": "146,932,807.68", + "B9": "Revenue", + "C1": "COGS", + "C2": "43,200,000.00", + "C3": "46,656,000.00", + "C4": "50,388,480.00", + "C5": "54,419,558.40", + "C6": "58,773,123.07", + "C9": "COGS", + "D1": "Operating Expenses", + "D2": "21,600,000.00", + "D3": "23,328,000.00", + "D4": "25,194,240.00", + "D5": "27,209,779.20", + "D6": "29,386,561.54", + "D9": "Operating Expenses", + "E1": "EBITDA", + "E2": "43,200,000.00", + "E3": "46,656,000.00", + "E4": "50,388,480.00", + "E5": "54,419,558.40", + "E6": "58,773,123.07", + "E9": "EBITDA", + "F1": "Net Income", + "F2": "34,560,000.00", + "F3": "37,324,800.00", + "F4": "40,310,784.00", + "F5": "43,535,646.72", + "F6": "47,018,498.46", + "F9": "Net Income", + "A10": "2025", + "A11": "2026", + "A12": "2027", + "A13": "2028", + "A14": "2029", + "A17": "Year", + "A18": "2025", + "A19": "2026", + "A20": "2027", + "A21": "2028", + "A22": "2029", + "B10": "112,000,000.00", + "B11": "125,440,000.00", + "B12": "140,492,800.00", + "B13": "157,351,936.00", + "B14": "176,234,168.32", + "B17": "Revenue", + "B18": "103,000,000.00", + "B19": "106,090,000.00", + "B20": "109,272,700.00", + "B21": "112,550,881.00", + "B22": "115,927,407.43", + "C10": "39,200,000.00", + "C11": "43,904,000.00", + "C12": "49,172,480.00", + "C13": "55,073,177.60", + "C14": "61,681,958.91", + "C17": "COGS", + "C18": "46,350,000.00", + "C19": "47,740,500.00", + "C20": "49,172,715.00", + "C21": "50,647,896.45", + "C22": "52,167,333.34", + "D10": "33,600,000.00", + "D11": "37,632,000.00", + "D12": "42,147,840.00", + "D13": "47,205,580.80", + "D14": "52,870,250.50", + "D17": "Operating Expenses", + "D18": "20,600,000.00", + "D19": "21,218,000.00", + "D20": "21,854,540.00", + "D21": "22,510,176.20", + "D22": "23,185,481.49", + "E10": "39,200,000.00", + "E11": "43,904,000.00", + "E12": "49,172,480.00", + "E13": "55,073,177.60", + "E14": "61,681,958.91", + "E17": "EBITDA", + "E18": "36,050,000.00", + "E19": "37,131,500.00", + "E20": "38,245,445.00", + "E21": "39,392,808.35", + "E22": "40,574,592.60", + "F10": "31,360,000.00", + "F11": "35,123,200.00", + "F12": "39,337,984.00", + "F13": "44,058,542.08", + "F14": "49,345,567.13", + "F17": "Net Income", + "F18": "28,840,000.00", + "F19": "29,705,200.00", + "F20": "30,596,356.00", + "F21": "31,514,246.68", + "F22": "32,459,674.08" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "cea3b19a-6855-45f0-863e-42694346d487", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "2024-04-01", + "A5": "2024-04-02", + "A6": "2024-04-03", + "A7": "2024-04-04", + "A8": "2024-04-05", + "A9": "2024-04-08", + "B4": "1.020", + "B5": "0.953", + "B6": "0.989", + "B7": "0.046", + "B8": "0.046", + "B9": "0.046", + "A10": "2024-04-09", + "A11": "2024-04-10", + "A12": "2024-04-11", + "A13": "2024-04-12", + "A14": "Grand Total", + "B10": "0.046", + "B11": "0.046", + "B12": "0.046", + "B13": "0.046", + "B14": "0.328" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1,758,109,357.43" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A2": "Kressa", + "A3": "Saumya", + "A4": "Jeane", + "A5": "Sarah", + "A6": "Anaiya", + "A7": "Varun", + "A8": "Sofie", + "B2": "$27.00", + "B3": "$30.50", + "B4": "$27.00", + "B5": "$18.00", + "B6": "$50.00", + "B7": "$52.00", + "B8": "$19.00", + "C2": "40", + "C3": "50", + "C4": "20", + "C5": "20", + "C6": "80", + "C7": "80", + "C8": "20", + "D2": "$1,080.00", + "D3": "$1,525.00", + "D4": "$540.00", + "D5": "$360.00", + "D6": "$4,000.00", + "D7": "$4,160.00", + "D8": "$380.00", + "E2": "$66.96", + "E3": "$94.55", + "E4": "$33.48", + "E5": "$22.32", + "E6": "$248.00", + "E7": "$257.92", + "E8": "$23.56", + "F2": "$15.66", + "F3": "$22.11", + "F4": "$7.83", + "F5": "$5.22", + "F6": "$58.00", + "F7": "$60.32", + "F8": "$5.51", + "G2": "$2.24", + "G3": "$2.80", + "G4": "$1.12", + "G5": "$1.12", + "G6": "$4.48", + "G7": "$4.48", + "G8": "$1.12", + "H2": "$0.32", + "H3": "$0.46", + "H4": "$0.16", + "H5": "$0.11", + "H6": "$1.20", + "H7": "$1.25", + "H8": "$0.11", + "I2": "$5.71", + "I3": "$8.06", + "I4": "$2.85", + "I5": "$1.90", + "I6": "$21.14", + "I7": "$21.99", + "I8": "$2.01", + "J2": "$6.26", + "J3": "$8.85", + "J4": "$3.13", + "J5": "$2.09", + "J6": "$23.20", + "J7": "$24.13", + "J8": "$2.20", + "K2": "$97.16", + "K3": "$136.82", + "K4": "$48.58", + "K5": "$32.76", + "K6": "$356.02", + "K7": "$370.08", + "K8": "$34.52" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B3": "Maturity", + "A4": "CurveDate", + "B4": "10Y", + "C4": "1Y", + "D4": "2Y", + "E4": "30Y", + "F4": "5Y", + "G4": "Grand Total", + "A5": "2024-01-01", + "B5": "$3.52", + "C5": "$3.73", + "D5": "$3.71", + "E5": "$3.86", + "F5": "$4.71", + "G5": "$19.53", + "A6": "2024-01-02", + "B6": "$3.73", + "C6": "$3.62", + "D6": "$3.79", + "E6": "$4.81", + "F6": "$4.54", + "G6": "$20.49", + "A7": "2024-01-03", + "B7": "$4.28", + "C7": "$4.98", + "D7": "$4.63", + "E7": "$3.70", + "F7": "$4.79", + "G7": "$22.38", + "A8": "2024-01-04", + "B8": "$4.03", + "C8": "$4.84", + "D8": "$3.94", + "E8": "$4.84", + "F8": "$4.49", + "G8": "$22.14", + "A9": "Grand Total", + "B9": "$15.56", + "C9": "$17.17", + "D9": "$16.07", + "E9": "$17.21", + "F9": "$18.53", + "G9": "$84.54" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "ABC", + "B1": "-0.08" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Carla White", + "A2": "$155.00", + "B1": "$8,758.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1e4ede56-163f-4815-b634-7946cf9e60c0", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Content Marketing", + "A2": "Email", + "A3": "PPC", + "A4": "SEO", + "A5": "Social Media", + "B1": "Harness Frictionless Users", + "B2": "Transform Out-Of-The-Box Schemas", + "B3": "Morph Back-End E-Business", + "B4": "Facilitate Dynamic Channels", + "B5": "Re-Intermediate Cutting-Edge Web-Readiness", + "C1": "6.61", + "C2": "5.99", + "C3": "5.02", + "C4": "5.49", + "C5": "5.68" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "8499e399-48e6-4603-bd96-9b45f5aabed0", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Omar Donovan", + "A2": "Omar Donovan" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1876764.43" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "991105", + "A2": "1044283", + "A3": "1095308", + "A4": "1169204", + "B1": "4299900" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "764ca583-7091-4b7f-8663-5e996db26a2c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "16.74%", + "B1": "27.80%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9f71aa71-07f7-4421-8757-bbccbeab0984", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/gold_solution_3.xlsx" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Star City", + "B1": "39.11%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Gotham", + "B1": "4" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Finance", + "B1": "1,000" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "21680.086" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d2c56ee0-7863-45be-b503-eb1639454629", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Scenario 3", + "A2": "Scenario 2", + "A3": "Scenario 1" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "077926f8-6e93-406b-9eb9-b44907173c74", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Vendor", + "A2": "vendor 18", + "A3": "vendor 15", + "A4": "vendor 19", + "A5": "vendor 1", + "A6": "vendor 14", + "A7": "vendor 9", + "A8": "vendor 8", + "A9": "vendor 11", + "B1": "1099 Report amount", + "B2": "$4.00", + "B3": "$48.00", + "B4": "$81.00", + "B5": "$92.60", + "B6": "$177.00", + "B7": "$866.00", + "B8": "$1,661.00", + "B9": "$2,615.00", + "A10": "vendor 4", + "A11": "vendor 5", + "B10": "$6,446.00", + "B11": "$6,519.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "f1afee7f-df70-4a11-a65e-767a718f2117", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2025-02" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "36e43b84-e583-4f02-a160-74049bcab901", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "Platform", + "A5": "Facebook", + "A9": "Instagram", + "B4": "Values", + "B5": "Sum of Posts", + "B6": "Sum of Likes", + "B7": "Sum of Shares", + "B8": "Sum of Comments", + "B9": "Sum of Posts", + "C4": "August", + "C5": "66", + "C6": "3899", + "C7": "618", + "C8": "241", + "C9": "54", + "D4": "July", + "D5": "65", + "D6": "4008", + "D7": "749", + "D8": "260", + "D9": "58", + "E4": "June", + "E5": "54", + "E6": "3236", + "E7": "560", + "E8": "296", + "E9": "70", + "F4": "November", + "F5": "39", + "F6": "2066", + "F7": "430", + "F8": "209", + "F9": "52", + "G4": "October", + "G5": "68", + "G6": "3048", + "G7": "773", + "G8": "271", + "G9": "54", + "H4": "September", + "H5": "64", + "H6": "2918", + "H7": "769", + "H8": "262", + "H9": "57", + "I4": "Grand Total", + "I5": "356", + "I6": "19175", + "I7": "3899", + "I8": "1539", + "I9": "345", + "A13": "LinkedIn", + "A17": "Twitter", + "A21": "Grand Total", + "B10": "Sum of Likes", + "B11": "Sum of Shares", + "B12": "Sum of Comments", + "B13": "Sum of Posts", + "B14": "Sum of Likes", + "B15": "Sum of Shares", + "B16": "Sum of Comments", + "B17": "Sum of Posts", + "B18": "Sum of Likes", + "B19": "Sum of Shares", + "B20": "Sum of Comments", + "B21": "Sum of Posts", + "B22": "Sum of Likes", + "B23": "Sum of Shares", + "B24": "Sum of Comments", + "C10": "3231", + "C11": "541", + "C12": "226", + "C13": "61", + "C14": "2923", + "C15": "668", + "C16": "295", + "C17": "60", + "C18": "3290", + "C19": "671", + "C20": "295", + "C21": "241", + "C22": "13343", + "C23": "2498", + "C24": "1057", + "D10": "3025", + "D11": "700", + "D12": "232", + "D13": "60", + "D14": "3122", + "D15": "589", + "D16": "314", + "D17": "54", + "D18": "2950", + "D19": "454", + "D20": "271", + "D21": "237", + "D22": "13105", + "D23": "2492", + "D24": "1077", + "E10": "4195", + "E11": "834", + "E12": "317", + "E13": "45", + "E14": "2291", + "E15": "451", + "E16": "185", + "E17": "63", + "E18": "3366", + "E19": "797", + "E20": "278", + "E21": "232", + "E22": "13088", + "E23": "2642", + "E24": "1076", + "F10": "2860", + "F11": "514", + "F12": "249", + "F13": "61", + "F14": "3116", + "F15": "708", + "F16": "246", + "F17": "63", + "F18": "2714", + "F19": "581", + "F20": "243", + "F21": "215", + "F22": "10756", + "F23": "2233", + "F24": "947", + "G10": "2993", + "G11": "623", + "G12": "265", + "G13": "70", + "G14": "3596", + "G15": "716", + "G16": "336", + "G17": "58", + "G18": "2382", + "G19": "578", + "G20": "293", + "G21": "250", + "G22": "12019", + "G23": "2690", + "G24": "1165", + "H10": "3409", + "H11": "401", + "H12": "313", + "H13": "65", + "H14": "3271", + "H15": "771", + "H16": "278", + "H17": "51", + "H18": "2156", + "H19": "431", + "H20": "312", + "H21": "237", + "H22": "11754", + "H23": "2372", + "H24": "1165", + "I10": "19713", + "I11": "3613", + "I12": "1602", + "I13": "362", + "I14": "18319", + "I15": "3903", + "I16": "1654", + "I17": "349", + "I18": "16858", + "I19": "3512", + "I20": "1692", + "I21": "1412", + "I22": "74065", + "I23": "14927", + "I24": "6487" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Platform", + "A2": "Twitter", + "A3": "Facebook", + "A4": "Instagram", + "A5": "LinkedIn", + "B1": "RATIO OF HIGH / LOW", + "B2": "1.88", + "B3": "2.64", + "B4": "3.17", + "B5": "2.05" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "c437f78d-6382-43b2-b857-153db6efa1c9", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "Jun", + "A5": "Jul", + "A6": "Aug", + "A7": "Sep", + "A8": "Oct", + "A9": "Nov", + "B3": "Sum of Posts", + "B4": "124", + "B5": "123", + "B6": "120", + "B7": "121", + "B8": "122", + "B9": "91", + "C3": "Sum of Likes", + "C4": "7431", + "C5": "7033", + "C6": "7130", + "C7": "6327", + "C8": "6041", + "C9": "4926", + "D3": "Sum of Comments", + "D4": "613", + "D5": "492", + "D6": "467", + "D7": "575", + "D8": "536", + "D9": "458", + "E3": "Sum of Shares", + "E4": "1394", + "E5": "1449", + "E6": "1159", + "E7": "1170", + "E8": "1396", + "E9": "944", + "A10": "Grand Total", + "B10": "701", + "C10": "38888", + "D10": "3141", + "E10": "7512" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.68%", + "E3": "43.14%", + "E4": "43.77%", + "E5": "43.27%", + "E6": "44.90%", + "E7": "44.03%", + "E8": "44.06%", + "E9": "43.97%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "E10": "44.37%", + "E11": "44.10%", + "E12": "46.70%", + "E13": "48.06%", + "E14": "45.94%", + "E15": "45.48%", + "E16": "44.06%", + "E17": "43.93%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9ba9631e-8560-4dc4-a285-8d186715a542", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.7%", + "E3": "43.1%", + "E4": "43.8%", + "E5": "43.3%", + "E6": "44.9%", + "E7": "44.0%", + "E8": "44.1%", + "E9": "44.0%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "A18": "5", + "A19": "6", + "A20": "7", + "A21": "8", + "A22": "9", + "A23": "10", + "A24": "11", + "A25": "12", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "B18": "2024", + "B19": "2024", + "B20": "2024", + "B21": "2024", + "B22": "2024", + "B23": "2024", + "B24": "2024", + "B25": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "C18": "21095", + "C19": "21015", + "C20": "21141", + "C21": "21311", + "C22": "21064", + "C23": "21182", + "C24": "21209", + "C25": "21238", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "D18": "34237", + "D19": "33819", + "D20": "33547", + "D21": "33832", + "D22": "33424", + "D23": "33162", + "D24": "33042", + "D25": "32929", + "E10": "44.4%", + "E11": "44.1%", + "E12": "46.7%", + "E13": "48.1%", + "E14": "45.9%", + "E15": "45.5%", + "E16": "44.1%", + "E17": "43.9%", + "E18": "43.1%", + "E19": "41.8%", + "E20": "41.3%", + "E21": "40.4%", + "E22": "39.7%", + "E23": "38.7%", + "E24": "37.9%", + "E25": "37.2%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2023-04-30", + "A2": "0.48", + "A3": "0.00296" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "D2": "Rank Based on 2024 Sales", + "D3": "4", + "D4": "3", + "D5": "7", + "D6": "17", + "D7": "12", + "D8": "1", + "D9": "18", + "D10": "8", + "D11": "6", + "D12": "14", + "D13": "5", + "D14": "13", + "D15": "9", + "D16": "20", + "D17": "11", + "D18": "10", + "D19": "19", + "D20": "15", + "D21": "2", + "D22": "16" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "20507108-4188-402e-8cfe-2354309bad6c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A2": "Region", + "A3": "Central", + "A4": "East", + "A5": "North", + "A6": "South", + "A7": "West", + "B2": "2023", + "B3": "351351.87", + "B4": "364129.72", + "B5": "377252.87", + "B6": "396259.93", + "B7": "395672.78", + "C2": "2024", + "C3": "368644.54", + "C4": "399862.83", + "C5": "364113.70", + "C6": "345896.80", + "C7": "393456.82", + "D2": "YoY Growth", + "D3": "4.92%", + "D4": "9.81%", + "D5": "-3.48%", + "D6": "-12.71%", + "D7": "-0.56%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Shopping", + "B1": "$ (16,401.91)" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "6.17%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "24.20%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "371507f7-721a-457e-9618-bc8fba1b909b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$316,309.56" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "4a081ed2-532c-4895-a034-ab0076927c7c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2026-03", + "A2": "2025-09", + "A3": "2026-12" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "JPY", + "A2": "USD", + "A3": "EUR", + "A4": "GBP", + "B1": "3500000.00", + "B2": "64351.25", + "B3": "43501.00", + "B4": "15000.25" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "This spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "31508ec6-c993-4e00-b70c-093dba016fcc", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1.07%", + "A2": "1.73%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Which city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "133ef005-207c-490f-b55c-7734fd1678da", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Central City" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Work in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header “Date” plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "eb410896-3e1e-4491-9460-061579b65c6f", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Date", + "A2": "2025-01-05", + "A3": "2025-01-12", + "A4": "2025-01-15", + "A5": "2025-01-20", + "A6": "2025-01-25", + "A7": "2025-01-30", + "A8": "2025-02-05", + "A9": "2025-02-08", + "B1": "Description", + "B2": "Membership Fee", + "B3": "Project Income", + "B4": "Interest", + "B5": "Event Revenue", + "B6": "Refund", + "B7": "Consulting Fee", + "B8": "Website Hosting", + "B9": "Maintenance", + "C1": "Amount", + "C2": "-250.00", + "C3": "5600.00", + "C4": "350.00", + "C5": "7000.00", + "C6": "-5000.00", + "C7": "7800.00", + "C8": "-99.99", + "C9": "-750.25", + "D1": "Currency", + "D2": "USD", + "D3": "USD", + "D4": "USD", + "D5": "USD", + "D6": "USD", + "D7": "USD", + "D8": "USD", + "D9": "USD", + "A10": "2025-02-15", + "A11": "2025-02-18", + "A12": "2025-02-20", + "A13": "2025-02-25", + "A14": "2025-02-28", + "A15": "2025-03-01", + "A16": "2025-03-05", + "A17": "2025-03-10", + "A18": "2025-03-15", + "A19": "2025-03-20", + "A20": "2025-03-25", + "B10": "Invoice Payment", + "B11": "Equipment Purchase", + "B12": "Software License", + "B13": "Bonus", + "B14": "Travel Expenses", + "B15": "Subscription", + "B16": "Advertising", + "B17": "Office Supplies", + "B18": "Utilities", + "B19": "Legal Fees", + "B20": "Marketing", + "C10": "2500.00", + "C11": "-3600.00", + "C12": "-3000.00", + "C13": "4800.00", + "C14": "-1750.00", + "C15": "-1200.00", + "C16": "-2450.50", + "C17": "-450.75", + "C18": "-899.00", + "C19": "-4000.00", + "C20": "-2200.00", + "D10": "USD", + "D11": "USD", + "D12": "USD", + "D13": "USD", + "D14": "USD", + "D15": "USD", + "D16": "USD", + "D17": "USD", + "D18": "USD", + "D19": "USD", + "D20": "USD" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + } +] \ No newline at end of file diff --git a/SheetBench-50.json b/SheetBench-50.json new file mode 100644 index 000000000..67f759656 --- /dev/null +++ b/SheetBench-50.json @@ -0,0 +1,4211 @@ +[ + { + "prompt": "Calculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", + "mcp_config": { + "hud": { + "url": "https://orcstaging.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.3" + } + } + }, + "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1/12/2024", + "A2": "1/10/2024", + "A3": "1/15/2024", + "A4": "1/11/2024", + "A5": "1/17/2024" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Calculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B2": "CustomerID", + "B3": "476232", + "B4": "963589", + "B5": "1430820", + "B6": "2410043", + "B7": "3789483", + "B8": "4226444", + "B9": "4308096", + "C2": "2023", + "C5": "686.40", + "C8": "966.76", + "C9": "863.37", + "D2": "2024", + "D3": "912.70", + "D4": "397.62", + "D6": "383.38", + "D7": "619.56", + "E2": "Grand Total", + "E3": "912.70", + "E4": "397.62", + "E5": "686.40", + "E6": "383.38", + "E7": "619.56", + "E8": "966.76", + "E9": "863.37", + "B10": "5907496", + "B11": "6224271", + "B12": "6538101", + "B13": "6584993", + "B14": "6791810", + "B15": "7250781", + "B16": "8518187", + "B17": "8668097", + "B18": "8885050", + "B19": "9296053", + "B20": "10414682", + "B21": "10839621", + "B22": "12313143", + "B23": "13672423", + "B24": "14744105", + "B25": "14780608", + "B26": "15144539", + "B27": "15616289", + "B28": "15694782", + "B29": "15904102", + "B30": "16385696", + "B31": "17254618", + "B32": "17441704", + "B33": "17624107", + "B34": "18043433", + "B35": "18296075", + "B36": "18600797", + "B37": "19026954", + "B38": "19395319", + "B39": "19650142", + "B40": "19842318", + "B41": "20241148", + "B42": "20587659", + "B43": "21647994", + "B44": "21671104", + "B45": "21935183", + "B46": "22115836", + "B47": "22846037", + "B48": "23621954", + "B49": "24962520", + "B50": "25073201", + "B51": "25090031", + "B52": "25096194", + "B53": "25211207", + "B54": "26168579", + "B55": "26250755", + "B56": "26419282", + "B57": "26953175", + "B58": "27281086", + "B59": "27646724", + "B60": "30370166", + "B61": "30738270", + "B62": "30876644", + "B63": "32255126", + "B64": "32373769", + "B65": "33121017", + "B66": "33897455", + "B67": "34507001", + "B68": "35475491", + "B69": "36972111", + "B70": "37040802", + "B71": "37543198", + "B72": "37609900", + "B73": "37674649", + "B74": "38019388", + "B75": "38658747", + "B76": "38793189", + "B77": "39134950", + "B78": "40399799", + "B79": "41039038", + "B80": "41271414", + "B81": "41870334", + "B82": "42495130", + "B83": "43189090", + "B84": "43235421", + "B85": "43837144", + "B86": "44115166", + "B87": "44270797", + "B88": "45380457", + "B89": "45386282", + "B90": "46196026", + "B91": "46300157", + "B92": "48512428", + "B93": "49546866", + "B94": "49687477", + "B95": "50893874", + "B96": "51093532", + "B97": "51126698", + "B98": "51397982", + "B99": "52376160", + "C10": "714.36", + "C11": "369.01", + "C12": "525.20", + "C20": "120.72", + "C21": "269.30", + "C22": "422.36", + "C26": "971.83", + "C28": "110.65", + "C29": "496.66", + "C30": "841.89", + "C31": "886.06", + "C32": "536.09", + "C34": "16.13", + "C38": "90.40", + "C39": "76.23", + "C40": "185.33", + "C41": "65.32", + "C42": "111.51", + "C43": "493.89", + "C44": "787.04", + "C45": "578.58", + "C46": "204.48", + "C48": "985.49", + "C50": "268.33", + "C52": "34.51", + "C54": "61.19", + "C60": "62.83", + "C63": "326.11", + "C64": "898.66", + "C65": "251.17", + "C66": "902.46", + "C67": "442.33", + "C70": "561.10", + "C71": "469.05", + "C74": "477.14", + "C76": "284.69", + "C80": "266.21", + "C82": "699.49", + "C85": "82.73", + "C87": "542.97", + "C88": "446.42", + "C89": "380.14", + "C91": "683.73", + "C92": "566.99", + "C94": "646.75", + "C97": "748.88", + "C98": "253.40", + "D13": "174.59", + "D14": "374.65", + "D15": "339.03", + "D16": "140.66", + "D17": "862.92", + "D18": "282.39", + "D19": "847.67", + "D23": "616.14", + "D24": "376.83", + "D25": "523.78", + "D27": "701.88", + "D29": "496.66", + "D33": "805.83", + "D35": "579.30", + "D36": "577.84", + "D37": "673.27", + "D38": "90.40", + "D47": "47.03", + "D48": "985.49", + "D49": "779.72", + "D51": "277.24", + "D53": "702.84", + "D55": "334.42", + "D56": "875.84", + "D57": "570.46", + "D58": "60.52", + "D59": "29.15", + "D61": "822.61", + "D62": "931.36", + "D68": "728.49", + "D69": "454.44", + "D72": "866.51", + "D73": "968.06", + "D74": "477.14", + "D75": "154.14", + "D77": "69.76", + "D78": "323.27", + "D79": "862.51", + "D81": "335.73", + "D83": "555.46", + "D84": "685.07", + "D86": "942.65", + "D90": "813.06", + "D93": "671.93", + "D95": "995.04", + "D96": "896.64", + "D98": "253.40", + "D99": "66.10", + "E10": "714.36", + "E11": "369.01", + "E12": "525.20", + "E13": "174.59", + "E14": "374.65", + "E15": "339.03", + "E16": "140.66", + "E17": "862.92", + "E18": "282.39", + "E19": "847.67", + "E20": "120.72", + "E21": "269.30", + "E22": "422.36", + "E23": "616.14", + "E24": "376.83", + "E25": "523.78", + "E26": "971.83", + "E27": "701.88", + "E28": "110.65", + "E29": "993.32", + "E30": "841.89", + "E31": "886.06", + "E32": "536.09", + "E33": "805.83", + "E34": "16.13", + "E35": "579.30", + "E36": "577.84", + "E37": "673.27", + "E38": "180.80", + "E39": "76.23", + "E40": "185.33", + "E41": "65.32", + "E42": "111.51", + "E43": "493.89", + "E44": "787.04", + "E45": "578.58", + "E46": "204.48", + "E47": "47.03", + "E48": "1970.98", + "E49": "779.72", + "E50": "268.33", + "E51": "277.24", + "E52": "34.51", + "E53": "702.84", + "E54": "61.19", + "E55": "334.42", + "E56": "875.84", + "E57": "570.46", + "E58": "60.52", + "E59": "29.15", + "E60": "62.83", + "E61": "822.61", + "E62": "931.36", + "E63": "326.11", + "E64": "898.66", + "E65": "251.17", + "E66": "902.46", + "E67": "442.33", + "E68": "728.49", + "E69": "454.44", + "E70": "561.10", + "E71": "469.05", + "E72": "866.51", + "E73": "968.06", + "E74": "954.28", + "E75": "154.14", + "E76": "284.69", + "E77": "69.76", + "E78": "323.27", + "E79": "862.51", + "E80": "266.21", + "E81": "335.73", + "E82": "699.49", + "E83": "555.46", + "E84": "685.07", + "E85": "82.73", + "E86": "942.65", + "E87": "542.97", + "E88": "446.42", + "E89": "380.14", + "E90": "813.06", + "E91": "683.73", + "E92": "566.99", + "E93": "671.93", + "E94": "646.75", + "E95": "995.04", + "E96": "896.64", + "E97": "748.88", + "E98": "506.80", + "E99": "66.10", + "B100": "52697630", + "B101": "52870804", + "B102": "53239020", + "B103": "53335630", + "B104": "53858146", + "B105": "54040755", + "B106": "54479395", + "B107": "54528192", + "B108": "54730083", + "B109": "55444897", + "B110": "55450065", + "B111": "55853989", + "B112": "55859479", + "B113": "55978545", + "B114": "56702798", + "B115": "56940756", + "B116": "58152560", + "B117": "59169574", + "B118": "59736806", + "B119": "61602489", + "B120": "62665919", + "B121": "62758192", + "B122": "62943277", + "B123": "63051186", + "B124": "65657133", + "B125": "65899458", + "B126": "66761945", + "B127": "66846840", + "B128": "67283606", + "B129": "67548873", + "B130": "69613838", + "B131": "69765117", + "B132": "70098167", + "B133": "70534368", + "B134": "71078229", + "B135": "71319902", + "B136": "71369648", + "B137": "71376141", + "B138": "72217443", + "B139": "72659809", + "B140": "72883709", + "B141": "73191421", + "B142": "73684220", + "B143": "74014279", + "B144": "74088126", + "B145": "75410476", + "B146": "75817527", + "B147": "77140593", + "B148": "77558800", + "B149": "78916470", + "B150": "79031936", + "B151": "79751742", + "B152": "80286530", + "B153": "80765899", + "B154": "82060237", + "B155": "82306595", + "B156": "83101113", + "B157": "83211478", + "B158": "83713620", + "B159": "84770820", + "B160": "84800206", + "B161": "84952943", + "B162": "86407021", + "B163": "86619158", + "B164": "86663007", + "B165": "87144451", + "B166": "87254792", + "B167": "88194202", + "B168": "88266169", + "B169": "88761732", + "B170": "88882642", + "B171": "89421277", + "B172": "89565544", + "B173": "90841330", + "B174": "91483447", + "B175": "91590435", + "B176": "91939241", + "B177": "92335820", + "B178": "92422728", + "B179": "92676749", + "B180": "93202190", + "B181": "93479509", + "B182": "95353791", + "B183": "95696393", + "B184": "95804583", + "B185": "95860510", + "B186": "96051566", + "B187": "96263709", + "B188": "96456101", + "B189": "99519803", + "B190": "Grand Total", + "C100": "409.46", + "C101": "28.08", + "C106": "184.81", + "C107": "906.16", + "C108": "1094.00", + "C109": "818.87", + "C110": "192.68", + "C111": "385.72", + "C117": "976.44", + "C118": "310.80", + "C119": "354.99", + "C120": "898.04", + "C122": "397.81", + "C128": "537.90", + "C129": "886.96", + "C131": "420.66", + "C133": "359.50", + "C134": "60.29", + "C139": "740.19", + "C142": "554.52", + "C144": "759.21", + "C145": "116.62", + "C148": "302.90", + "C152": "187.16", + "C153": "847.55", + "C154": "700.62", + "C155": "190.59", + "C161": "432.99", + "C163": "886.00", + "C167": "715.20", + "C169": "101.82", + "C171": "122.95", + "C174": "609.90", + "C175": "867.14", + "C177": "675.11", + "C178": "623.53", + "C182": "40.39", + "C184": "554.62", + "C187": "839.28", + "C189": "717.61", + "C190": "43541.41", + "D102": "995.08", + "D103": "761.74", + "D104": "773.13", + "D105": "364.27", + "D112": "841.73", + "D113": "560.15", + "D114": "824.63", + "D115": "783.50", + "D116": "788.60", + "D117": "976.44", + "D118": "310.80", + "D121": "92.40", + "D123": "171.84", + "D124": "336.61", + "D125": "206.92", + "D126": "345.03", + "D127": "505.00", + "D130": "539.76", + "D132": "189.69", + "D135": "713.31", + "D136": "993.98", + "D137": "541.38", + "D138": "790.14", + "D139": "740.19", + "D140": "481.88", + "D141": "83.31", + "D143": "883.45", + "D146": "394.60", + "D147": "281.52", + "D149": "64.48", + "D150": "645.82", + "D151": "771.06", + "D156": "405.74", + "D157": "741.29", + "D158": "196.84", + "D159": "152.71", + "D160": "356.93", + "D162": "138.69", + "D164": "929.08", + "D165": "887.13", + "D166": "554.21", + "D168": "820.22", + "D170": "939.97", + "D171": "122.95", + "D172": "335.69", + "D173": "917.55", + "D176": "514.59", + "D179": "288.94", + "D180": "214.12", + "D181": "16.74", + "D183": "27.28", + "D185": "458.58", + "D186": "648.09", + "D188": "985.01", + "D190": "56717.97", + "E100": "409.46", + "E101": "28.08", + "E102": "995.08", + "E103": "761.74", + "E104": "773.13", + "E105": "364.27", + "E106": "184.81", + "E107": "906.16", + "E108": "1094.00", + "E109": "818.87", + "E110": "192.68", + "E111": "385.72", + "E112": "841.73", + "E113": "560.15", + "E114": "824.63", + "E115": "783.50", + "E116": "788.60", + "E117": "1952.88", + "E118": "621.60", + "E119": "354.99", + "E120": "898.04", + "E121": "92.40", + "E122": "397.81", + "E123": "171.84", + "E124": "336.61", + "E125": "206.92", + "E126": "345.03", + "E127": "505.00", + "E128": "537.90", + "E129": "886.96", + "E130": "539.76", + "E131": "420.66", + "E132": "189.69", + "E133": "359.50", + "E134": "60.29", + "E135": "713.31", + "E136": "993.98", + "E137": "541.38", + "E138": "790.14", + "E139": "1480.38", + "E140": "481.88", + "E141": "83.31", + "E142": "554.52", + "E143": "883.45", + "E144": "759.21", + "E145": "116.62", + "E146": "394.60", + "E147": "281.52", + "E148": "302.90", + "E149": "64.48", + "E150": "645.82", + "E151": "771.06", + "E152": "187.16", + "E153": "847.55", + "E154": "700.62", + "E155": "190.59", + "E156": "405.74", + "E157": "741.29", + "E158": "196.84", + "E159": "152.71", + "E160": "356.93", + "E161": "432.99", + "E162": "138.69", + "E163": "886.00", + "E164": "929.08", + "E165": "887.13", + "E166": "554.21", + "E167": "715.20", + "E168": "820.22", + "E169": "101.82", + "E170": "939.97", + "E171": "245.90", + "E172": "335.69", + "E173": "917.55", + "E174": "609.90", + "E175": "867.14", + "E176": "514.59", + "E177": "675.11", + "E178": "623.53", + "E179": "288.94", + "E180": "214.12", + "E181": "16.74", + "E182": "40.39", + "E183": "27.28", + "E184": "554.62", + "E185": "458.58", + "E186": "648.09", + "E187": "839.28", + "E188": "985.01", + "E189": "717.61", + "E190": "100259.38" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Find the GET request which most commonly results in an error. Place the URL in ANSWER!A1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "/api/users" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "For a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2026-04" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "For the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0963d367-f0ac-4be0-8f46-337bf335e68f", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "ABC", + "B1": "4999972", + "C1": "145%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$112,281.49" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "49ccea4d-4aaf-4faf-9659-b389686568a7", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$24,642.86" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "48d43d57-ed1d-4df0-8b36-62380bba7865", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1664934.45" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "ff114e80-196d-4a7a-99ca-152bac4fba90", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "MAY", + "A2": "OCT", + "B1": "95", + "B2": "73" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Category", + "A2": "Meals & Entertainment", + "A3": "Travel", + "A4": "Office Supplies", + "B1": "Sum of Amount", + "B2": "0", + "B3": "345", + "B4": "45760" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A3": "Sum of Amount (USD Millions)", + "A5": "Net Income", + "A6": "Revenue", + "A7": "Total Assets", + "A8": "Grand Total", + "B4": "FORD2024Q1", + "B5": "1.33", + "B6": "42.78", + "B7": "274.34", + "B8": "318.45", + "C4": "FORD2024Q2", + "C5": "1.83", + "C6": "44.81", + "C7": "276.59", + "C8": "323.23", + "D4": "FORD2024Q3", + "D5": "896.00", + "D6": "43.07", + "D7": "287.05", + "D8": "1226.12", + "E4": "FORD2025Q1", + "E5": "471.00", + "E6": "40.66", + "E7": "284.54", + "E8": "796.20", + "F4": "Grand Total", + "F5": "1370.17", + "F6": "171.32", + "F7": "1122.51", + "F8": "2664.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "C3": "Count of TicketID", + "D3": "Average of ResolutionTimeHours", + "A4": "Agent-101", + "B4": "Closed", + "C4": "73", + "D4": "13.90", + "B5": "In Progress", + "C5": "30", + "B6": "Open", + "C6": "31", + "B7": "Resolved", + "C7": "66", + "D7": "22.04", + "B8": "Waiting for Customer", + "C8": "12", + "C9": "212", + "D9": "17.76", + "A10": "Agent-102", + "B10": "Closed", + "C10": "70", + "D10": "28.53", + "B11": "In Progress", + "C11": "36", + "B12": "Open", + "C12": "32", + "B13": "Resolved", + "C13": "77", + "D13": "17.09", + "B14": "Waiting for Customer", + "C14": "10", + "C15": "225", + "D15": "22.54", + "A16": "Agent-103", + "B16": "Closed", + "C16": "73", + "D16": "23.11", + "B17": "In Progress", + "C17": "39", + "B18": "Open", + "C18": "14", + "B19": "Resolved", + "C19": "92", + "D19": "17.08", + "B20": "Waiting for Customer", + "C20": "11", + "C21": "229", + "D21": "19.75", + "A22": "Agent-104", + "B22": "Closed", + "C22": "63", + "D22": "15.48", + "B23": "In Progress", + "C23": "32", + "B24": "Open", + "C24": "18", + "B25": "Resolved", + "C25": "97", + "D25": "15.40", + "B26": "Waiting for Customer", + "C26": "13", + "C27": "223", + "D27": "15.43", + "A28": "Agent-105", + "B28": "Closed", + "C28": "70", + "D28": "22.20", + "B29": "In Progress", + "C29": "27", + "B30": "Open", + "C30": "16", + "B31": "Resolved", + "C31": "69", + "D31": "15.21", + "B32": "Waiting for Customer", + "C32": "10", + "C33": "192", + "D33": "18.73", + "A34": "Agent-106", + "B34": "Closed", + "C34": "65", + "D34": "26.15", + "B35": "In Progress", + "C35": "38", + "B36": "Open", + "C36": "20", + "B37": "Resolved", + "C37": "87", + "D37": "21.56", + "B38": "Waiting for Customer", + "C38": "10", + "C39": "220", + "D39": "23.52", + "A40": "Agent-107", + "B40": "Closed", + "C40": "68", + "D40": "18.41", + "B41": "In Progress", + "C41": "35", + "B42": "Open", + "C42": "17", + "B43": "Resolved", + "C43": "106", + "D43": "16.52", + "B44": "Waiting for Customer", + "C44": "11", + "C45": "237", + "D45": "17.26", + "A46": "Agent-108", + "B46": "Closed", + "C46": "65", + "D46": "16.81", + "B47": "In Progress", + "C47": "33", + "B48": "Open", + "C48": "20", + "B49": "Resolved", + "C49": "93", + "D49": "14.93", + "B50": "Waiting for Customer", + "C50": "8", + "C51": "219", + "D51": "15.70", + "B52": "(blank)", + "C52": "243", + "D52": "20.98", + "B53": "Closed", + "C53": "74", + "D53": "24.44", + "B54": "In Progress", + "C54": "32", + "B55": "Open", + "C55": "26", + "B56": "Resolved", + "C56": "99", + "D56": "18.39", + "B57": "Waiting for Customer", + "C57": "12", + "B58": "Grand Total", + "C58": "2000", + "D58": "19.05" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "7598db5d-0fd1-44db-ab1a-593cf026317b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B2": "1", + "B3": "2", + "B4": "3", + "B5": "4", + "B6": "5", + "B8": "1", + "B9": "2", + "C1": "Count of LastPromotionDate", + "C2": "12", + "C3": "13", + "C4": "10", + "C5": "15", + "C6": "5", + "C8": "1", + "C9": "5", + "D1": "Average of SalaryUSD", + "D2": "84965.91", + "D3": "87028.38", + "D4": "86094.90", + "D5": "78111.64", + "D6": "86015.00", + "D8": "82727.50", + "D9": "78876.00", + "A38": "Grand Total", + "B10": "3", + "B11": "4", + "B12": "5", + "B14": "1", + "B15": "2", + "B16": "3", + "B17": "4", + "B18": "5", + "B20": "1", + "B21": "2", + "B22": "3", + "B23": "4", + "B24": "5", + "B26": "1", + "B27": "2", + "B28": "3", + "B29": "4", + "B30": "5", + "B32": "1", + "B33": "2", + "B34": "3", + "B35": "4", + "B36": "5", + "C10": "4", + "C15": "3", + "C16": "2", + "C17": "2", + "C20": "1", + "C21": "4", + "C22": "6", + "C23": "3", + "C24": "5", + "C26": "10", + "C27": "9", + "C28": "4", + "C29": "6", + "C30": "4", + "C32": "5", + "C33": "2", + "C34": "4", + "C35": "4", + "C36": "1", + "C38": "140", + "D10": "76759.43", + "D11": "78318.00", + "D12": "80036.25", + "D14": "76599.00", + "D15": "93749.17", + "D16": "79929.00", + "D17": "88994.75", + "D18": "102902.00", + "D20": "77878.50", + "D21": "77480.38", + "D22": "84212.67", + "D23": "77508.83", + "D24": "77182.50", + "D26": "56034.08", + "D27": "52096.20", + "D28": "63190.71", + "D29": "60750.00", + "D30": "50299.25", + "D32": "71131.57", + "D33": "89669.50", + "D34": "81155.50", + "D35": "82037.80", + "D36": "61600.50", + "D38": "77770.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": " 27,301,058.62 " + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Year", + "A2": "2025", + "A3": "2026", + "A4": "2027", + "A5": "2028", + "A6": "2029", + "A9": "Year", + "B1": "Revenue", + "B2": "108,000,000.00", + "B3": "116,640,000.00", + "B4": "125,971,200.00", + "B5": "136,048,896.00", + "B6": "146,932,807.68", + "B9": "Revenue", + "C1": "COGS", + "C2": "43,200,000.00", + "C3": "46,656,000.00", + "C4": "50,388,480.00", + "C5": "54,419,558.40", + "C6": "58,773,123.07", + "C9": "COGS", + "D1": "Operating Expenses", + "D2": "21,600,000.00", + "D3": "23,328,000.00", + "D4": "25,194,240.00", + "D5": "27,209,779.20", + "D6": "29,386,561.54", + "D9": "Operating Expenses", + "E1": "EBITDA", + "E2": "43,200,000.00", + "E3": "46,656,000.00", + "E4": "50,388,480.00", + "E5": "54,419,558.40", + "E6": "58,773,123.07", + "E9": "EBITDA", + "F1": "Net Income", + "F2": "34,560,000.00", + "F3": "37,324,800.00", + "F4": "40,310,784.00", + "F5": "43,535,646.72", + "F6": "47,018,498.46", + "F9": "Net Income", + "A10": "2025", + "A11": "2026", + "A12": "2027", + "A13": "2028", + "A14": "2029", + "A17": "Year", + "A18": "2025", + "A19": "2026", + "A20": "2027", + "A21": "2028", + "A22": "2029", + "B10": "112,000,000.00", + "B11": "125,440,000.00", + "B12": "140,492,800.00", + "B13": "157,351,936.00", + "B14": "176,234,168.32", + "B17": "Revenue", + "B18": "103,000,000.00", + "B19": "106,090,000.00", + "B20": "109,272,700.00", + "B21": "112,550,881.00", + "B22": "115,927,407.43", + "C10": "39,200,000.00", + "C11": "43,904,000.00", + "C12": "49,172,480.00", + "C13": "55,073,177.60", + "C14": "61,681,958.91", + "C17": "COGS", + "C18": "46,350,000.00", + "C19": "47,740,500.00", + "C20": "49,172,715.00", + "C21": "50,647,896.45", + "C22": "52,167,333.34", + "D10": "33,600,000.00", + "D11": "37,632,000.00", + "D12": "42,147,840.00", + "D13": "47,205,580.80", + "D14": "52,870,250.50", + "D17": "Operating Expenses", + "D18": "20,600,000.00", + "D19": "21,218,000.00", + "D20": "21,854,540.00", + "D21": "22,510,176.20", + "D22": "23,185,481.49", + "E10": "39,200,000.00", + "E11": "43,904,000.00", + "E12": "49,172,480.00", + "E13": "55,073,177.60", + "E14": "61,681,958.91", + "E17": "EBITDA", + "E18": "36,050,000.00", + "E19": "37,131,500.00", + "E20": "38,245,445.00", + "E21": "39,392,808.35", + "E22": "40,574,592.60", + "F10": "31,360,000.00", + "F11": "35,123,200.00", + "F12": "39,337,984.00", + "F13": "44,058,542.08", + "F14": "49,345,567.13", + "F17": "Net Income", + "F18": "28,840,000.00", + "F19": "29,705,200.00", + "F20": "30,596,356.00", + "F21": "31,514,246.68", + "F22": "32,459,674.08" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "cea3b19a-6855-45f0-863e-42694346d487", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "2024-04-01", + "A5": "2024-04-02", + "A6": "2024-04-03", + "A7": "2024-04-04", + "A8": "2024-04-05", + "A9": "2024-04-08", + "B4": "1.020", + "B5": "0.953", + "B6": "0.989", + "B7": "0.046", + "B8": "0.046", + "B9": "0.046", + "A10": "2024-04-09", + "A11": "2024-04-10", + "A12": "2024-04-11", + "A13": "2024-04-12", + "A14": "Grand Total", + "B10": "0.046", + "B11": "0.046", + "B12": "0.046", + "B13": "0.046", + "B14": "0.328" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1,758,109,357.43" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A2": "Kressa", + "A3": "Saumya", + "A4": "Jeane", + "A5": "Sarah", + "A6": "Anaiya", + "A7": "Varun", + "A8": "Sofie", + "B2": "$27.00", + "B3": "$30.50", + "B4": "$27.00", + "B5": "$18.00", + "B6": "$50.00", + "B7": "$52.00", + "B8": "$19.00", + "C2": "40", + "C3": "50", + "C4": "20", + "C5": "20", + "C6": "80", + "C7": "80", + "C8": "20", + "D2": "$1,080.00", + "D3": "$1,525.00", + "D4": "$540.00", + "D5": "$360.00", + "D6": "$4,000.00", + "D7": "$4,160.00", + "D8": "$380.00", + "E2": "$66.96", + "E3": "$94.55", + "E4": "$33.48", + "E5": "$22.32", + "E6": "$248.00", + "E7": "$257.92", + "E8": "$23.56", + "F2": "$15.66", + "F3": "$22.11", + "F4": "$7.83", + "F5": "$5.22", + "F6": "$58.00", + "F7": "$60.32", + "F8": "$5.51", + "G2": "$2.24", + "G3": "$2.80", + "G4": "$1.12", + "G5": "$1.12", + "G6": "$4.48", + "G7": "$4.48", + "G8": "$1.12", + "H2": "$0.32", + "H3": "$0.46", + "H4": "$0.16", + "H5": "$0.11", + "H6": "$1.20", + "H7": "$1.25", + "H8": "$0.11", + "I2": "$5.71", + "I3": "$8.06", + "I4": "$2.85", + "I5": "$1.90", + "I6": "$21.14", + "I7": "$21.99", + "I8": "$2.01", + "J2": "$6.26", + "J3": "$8.85", + "J4": "$3.13", + "J5": "$2.09", + "J6": "$23.20", + "J7": "$24.13", + "J8": "$2.20", + "K2": "$97.16", + "K3": "$136.82", + "K4": "$48.58", + "K5": "$32.76", + "K6": "$356.02", + "K7": "$370.08", + "K8": "$34.52" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "B3": "Maturity", + "A4": "CurveDate", + "B4": "10Y", + "C4": "1Y", + "D4": "2Y", + "E4": "30Y", + "F4": "5Y", + "G4": "Grand Total", + "A5": "2024-01-01", + "B5": "$3.52", + "C5": "$3.73", + "D5": "$3.71", + "E5": "$3.86", + "F5": "$4.71", + "G5": "$19.53", + "A6": "2024-01-02", + "B6": "$3.73", + "C6": "$3.62", + "D6": "$3.79", + "E6": "$4.81", + "F6": "$4.54", + "G6": "$20.49", + "A7": "2024-01-03", + "B7": "$4.28", + "C7": "$4.98", + "D7": "$4.63", + "E7": "$3.70", + "F7": "$4.79", + "G7": "$22.38", + "A8": "2024-01-04", + "B8": "$4.03", + "C8": "$4.84", + "D8": "$3.94", + "E8": "$4.84", + "F8": "$4.49", + "G8": "$22.14", + "A9": "Grand Total", + "B9": "$15.56", + "C9": "$17.17", + "D9": "$16.07", + "E9": "$17.21", + "F9": "$18.53", + "G9": "$84.54" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "ABC", + "B1": "-0.08" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Carla White", + "A2": "$155.00", + "B1": "$8,758.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1e4ede56-163f-4815-b634-7946cf9e60c0", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Content Marketing", + "A2": "Email", + "A3": "PPC", + "A4": "SEO", + "A5": "Social Media", + "B1": "Harness Frictionless Users", + "B2": "Transform Out-Of-The-Box Schemas", + "B3": "Morph Back-End E-Business", + "B4": "Facilitate Dynamic Channels", + "B5": "Re-Intermediate Cutting-Edge Web-Readiness", + "C1": "6.61", + "C2": "5.99", + "C3": "5.02", + "C4": "5.49", + "C5": "5.68" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "8499e399-48e6-4603-bd96-9b45f5aabed0", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Omar Donovan", + "A2": "Omar Donovan" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1876764.43" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "991105", + "A2": "1044283", + "A3": "1095308", + "A4": "1169204", + "B1": "4299900" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "764ca583-7091-4b7f-8663-5e996db26a2c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "16.74%", + "B1": "27.80%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9f71aa71-07f7-4421-8757-bbccbeab0984", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/gold_solution_3.xlsx" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Star City", + "B1": "39.11%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Gotham", + "B1": "4" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Finance", + "B1": "1,000" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "21680.086" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "d2c56ee0-7863-45be-b503-eb1639454629", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Scenario 3", + "A2": "Scenario 2", + "A3": "Scenario 1" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "077926f8-6e93-406b-9eb9-b44907173c74", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Vendor", + "A2": "vendor 18", + "A3": "vendor 15", + "A4": "vendor 19", + "A5": "vendor 1", + "A6": "vendor 14", + "A7": "vendor 9", + "A8": "vendor 8", + "A9": "vendor 11", + "B1": "1099 Report amount", + "B2": "$4.00", + "B3": "$48.00", + "B4": "$81.00", + "B5": "$92.60", + "B6": "$177.00", + "B7": "$866.00", + "B8": "$1,661.00", + "B9": "$2,615.00", + "A10": "vendor 4", + "A11": "vendor 5", + "B10": "$6,446.00", + "B11": "$6,519.00" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "f1afee7f-df70-4a11-a65e-767a718f2117", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2025-02" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "36e43b84-e583-4f02-a160-74049bcab901", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "Platform", + "A5": "Facebook", + "A9": "Instagram", + "B4": "Values", + "B5": "Sum of Posts", + "B6": "Sum of Likes", + "B7": "Sum of Shares", + "B8": "Sum of Comments", + "B9": "Sum of Posts", + "C4": "August", + "C5": "66", + "C6": "3899", + "C7": "618", + "C8": "241", + "C9": "54", + "D4": "July", + "D5": "65", + "D6": "4008", + "D7": "749", + "D8": "260", + "D9": "58", + "E4": "June", + "E5": "54", + "E6": "3236", + "E7": "560", + "E8": "296", + "E9": "70", + "F4": "November", + "F5": "39", + "F6": "2066", + "F7": "430", + "F8": "209", + "F9": "52", + "G4": "October", + "G5": "68", + "G6": "3048", + "G7": "773", + "G8": "271", + "G9": "54", + "H4": "September", + "H5": "64", + "H6": "2918", + "H7": "769", + "H8": "262", + "H9": "57", + "I4": "Grand Total", + "I5": "356", + "I6": "19175", + "I7": "3899", + "I8": "1539", + "I9": "345", + "A13": "LinkedIn", + "A17": "Twitter", + "A21": "Grand Total", + "B10": "Sum of Likes", + "B11": "Sum of Shares", + "B12": "Sum of Comments", + "B13": "Sum of Posts", + "B14": "Sum of Likes", + "B15": "Sum of Shares", + "B16": "Sum of Comments", + "B17": "Sum of Posts", + "B18": "Sum of Likes", + "B19": "Sum of Shares", + "B20": "Sum of Comments", + "B21": "Sum of Posts", + "B22": "Sum of Likes", + "B23": "Sum of Shares", + "B24": "Sum of Comments", + "C10": "3231", + "C11": "541", + "C12": "226", + "C13": "61", + "C14": "2923", + "C15": "668", + "C16": "295", + "C17": "60", + "C18": "3290", + "C19": "671", + "C20": "295", + "C21": "241", + "C22": "13343", + "C23": "2498", + "C24": "1057", + "D10": "3025", + "D11": "700", + "D12": "232", + "D13": "60", + "D14": "3122", + "D15": "589", + "D16": "314", + "D17": "54", + "D18": "2950", + "D19": "454", + "D20": "271", + "D21": "237", + "D22": "13105", + "D23": "2492", + "D24": "1077", + "E10": "4195", + "E11": "834", + "E12": "317", + "E13": "45", + "E14": "2291", + "E15": "451", + "E16": "185", + "E17": "63", + "E18": "3366", + "E19": "797", + "E20": "278", + "E21": "232", + "E22": "13088", + "E23": "2642", + "E24": "1076", + "F10": "2860", + "F11": "514", + "F12": "249", + "F13": "61", + "F14": "3116", + "F15": "708", + "F16": "246", + "F17": "63", + "F18": "2714", + "F19": "581", + "F20": "243", + "F21": "215", + "F22": "10756", + "F23": "2233", + "F24": "947", + "G10": "2993", + "G11": "623", + "G12": "265", + "G13": "70", + "G14": "3596", + "G15": "716", + "G16": "336", + "G17": "58", + "G18": "2382", + "G19": "578", + "G20": "293", + "G21": "250", + "G22": "12019", + "G23": "2690", + "G24": "1165", + "H10": "3409", + "H11": "401", + "H12": "313", + "H13": "65", + "H14": "3271", + "H15": "771", + "H16": "278", + "H17": "51", + "H18": "2156", + "H19": "431", + "H20": "312", + "H21": "237", + "H22": "11754", + "H23": "2372", + "H24": "1165", + "I10": "19713", + "I11": "3613", + "I12": "1602", + "I13": "362", + "I14": "18319", + "I15": "3903", + "I16": "1654", + "I17": "349", + "I18": "16858", + "I19": "3512", + "I20": "1692", + "I21": "1412", + "I22": "74065", + "I23": "14927", + "I24": "6487" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Platform", + "A2": "Twitter", + "A3": "Facebook", + "A4": "Instagram", + "A5": "LinkedIn", + "B1": "RATIO OF HIGH / LOW", + "B2": "1.88", + "B3": "2.64", + "B4": "3.17", + "B5": "2.05" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "c437f78d-6382-43b2-b857-153db6efa1c9", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A4": "Jun", + "A5": "Jul", + "A6": "Aug", + "A7": "Sep", + "A8": "Oct", + "A9": "Nov", + "B3": "Sum of Posts", + "B4": "124", + "B5": "123", + "B6": "120", + "B7": "121", + "B8": "122", + "B9": "91", + "C3": "Sum of Likes", + "C4": "7431", + "C5": "7033", + "C6": "7130", + "C7": "6327", + "C8": "6041", + "C9": "4926", + "D3": "Sum of Comments", + "D4": "613", + "D5": "492", + "D6": "467", + "D7": "575", + "D8": "536", + "D9": "458", + "E3": "Sum of Shares", + "E4": "1394", + "E5": "1449", + "E6": "1159", + "E7": "1170", + "E8": "1396", + "E9": "944", + "A10": "Grand Total", + "B10": "701", + "C10": "38888", + "D10": "3141", + "E10": "7512" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.68%", + "E3": "43.14%", + "E4": "43.77%", + "E5": "43.27%", + "E6": "44.90%", + "E7": "44.03%", + "E8": "44.06%", + "E9": "43.97%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "E10": "44.37%", + "E11": "44.10%", + "E12": "46.70%", + "E13": "48.06%", + "E14": "45.94%", + "E15": "45.48%", + "E16": "44.06%", + "E17": "43.93%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "9ba9631e-8560-4dc4-a285-8d186715a542", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.7%", + "E3": "43.1%", + "E4": "43.8%", + "E5": "43.3%", + "E6": "44.9%", + "E7": "44.0%", + "E8": "44.1%", + "E9": "44.0%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "A18": "5", + "A19": "6", + "A20": "7", + "A21": "8", + "A22": "9", + "A23": "10", + "A24": "11", + "A25": "12", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "B18": "2024", + "B19": "2024", + "B20": "2024", + "B21": "2024", + "B22": "2024", + "B23": "2024", + "B24": "2024", + "B25": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "C18": "21095", + "C19": "21015", + "C20": "21141", + "C21": "21311", + "C22": "21064", + "C23": "21182", + "C24": "21209", + "C25": "21238", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "D18": "34237", + "D19": "33819", + "D20": "33547", + "D21": "33832", + "D22": "33424", + "D23": "33162", + "D24": "33042", + "D25": "32929", + "E10": "44.4%", + "E11": "44.1%", + "E12": "46.7%", + "E13": "48.1%", + "E14": "45.9%", + "E15": "45.5%", + "E16": "44.1%", + "E17": "43.9%", + "E18": "43.1%", + "E19": "41.8%", + "E20": "41.3%", + "E21": "40.4%", + "E22": "39.7%", + "E23": "38.7%", + "E24": "37.9%", + "E25": "37.2%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Given the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2023-04-30", + "A2": "0.48", + "A3": "0.00296" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "D2": "Rank Based on 2024 Sales", + "D3": "4", + "D4": "3", + "D5": "7", + "D6": "17", + "D7": "12", + "D8": "1", + "D9": "18", + "D10": "8", + "D11": "6", + "D12": "14", + "D13": "5", + "D14": "13", + "D15": "9", + "D16": "20", + "D17": "11", + "D18": "10", + "D19": "19", + "D20": "15", + "D21": "2", + "D22": "16" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "20507108-4188-402e-8cfe-2354309bad6c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A2": "Region", + "A3": "Central", + "A4": "East", + "A5": "North", + "A6": "South", + "A7": "West", + "B2": "2023", + "B3": "351351.87", + "B4": "364129.72", + "B5": "377252.87", + "B6": "396259.93", + "B7": "395672.78", + "C2": "2024", + "C3": "368644.54", + "C4": "399862.83", + "C5": "364113.70", + "C6": "345896.80", + "C7": "393456.82", + "D2": "YoY Growth", + "D3": "4.92%", + "D4": "9.81%", + "D5": "-3.48%", + "D6": "-12.71%", + "D7": "-0.56%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "In your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Shopping", + "B1": "$ (16,401.91)" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "6.17%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "24.20%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "371507f7-721a-457e-9618-bc8fba1b909b", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "$316,309.56" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "4a081ed2-532c-4895-a034-ab0076927c7c", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "2026-03", + "A2": "2025-09", + "A3": "2026-12" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Sum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "JPY", + "A2": "USD", + "A3": "EUR", + "A4": "GBP", + "B1": "3500000.00", + "B2": "64351.25", + "B3": "43501.00", + "B4": "15000.25" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "This spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "31508ec6-c993-4e00-b70c-093dba016fcc", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "1.07%", + "A2": "1.73%" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Which city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "133ef005-207c-490f-b55c-7734fd1678da", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Central City" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + }, + { + "prompt": "Work in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header “Date” plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", + "mcp_config": { + "hud": { + "url": "https://mcp.hud.so/v3/mcp", + "headers": { + "Authorization": "Bearer ${HUD_API_KEY}", + "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" + } + } + }, + "id": "eb410896-3e1e-4491-9460-061579b65c6f", + "metadata": { + "partial": true, + "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/gold_solution_2.xlsx?" + }, + "setup_tool": { + "name": "setup", + "arguments": { + "name": "sheets_from_xlsx", + "arguments": { + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?" + } + } + }, + "evaluate_tool": { + "name": "evaluate", + "arguments": { + "name": "sheets_cell_values", + "arguments": { + "args": { + "A1": "Date", + "A2": "2025-01-05", + "A3": "2025-01-12", + "A4": "2025-01-15", + "A5": "2025-01-20", + "A6": "2025-01-25", + "A7": "2025-01-30", + "A8": "2025-02-05", + "A9": "2025-02-08", + "B1": "Description", + "B2": "Membership Fee", + "B3": "Project Income", + "B4": "Interest", + "B5": "Event Revenue", + "B6": "Refund", + "B7": "Consulting Fee", + "B8": "Website Hosting", + "B9": "Maintenance", + "C1": "Amount", + "C2": "-250.00", + "C3": "5600.00", + "C4": "350.00", + "C5": "7000.00", + "C6": "-5000.00", + "C7": "7800.00", + "C8": "-99.99", + "C9": "-750.25", + "D1": "Currency", + "D2": "USD", + "D3": "USD", + "D4": "USD", + "D5": "USD", + "D6": "USD", + "D7": "USD", + "D8": "USD", + "D9": "USD", + "A10": "2025-02-15", + "A11": "2025-02-18", + "A12": "2025-02-20", + "A13": "2025-02-25", + "A14": "2025-02-28", + "A15": "2025-03-01", + "A16": "2025-03-05", + "A17": "2025-03-10", + "A18": "2025-03-15", + "A19": "2025-03-20", + "A20": "2025-03-25", + "B10": "Invoice Payment", + "B11": "Equipment Purchase", + "B12": "Software License", + "B13": "Bonus", + "B14": "Travel Expenses", + "B15": "Subscription", + "B16": "Advertising", + "B17": "Office Supplies", + "B18": "Utilities", + "B19": "Legal Fees", + "B20": "Marketing", + "C10": "2500.00", + "C11": "-3600.00", + "C12": "-3000.00", + "C13": "4800.00", + "C14": "-1750.00", + "C15": "-1200.00", + "C16": "-2450.50", + "C17": "-450.75", + "C18": "-899.00", + "C19": "-4000.00", + "C20": "-2200.00", + "D10": "USD", + "D11": "USD", + "D12": "USD", + "D13": "USD", + "D14": "USD", + "D15": "USD", + "D16": "USD", + "D17": "USD", + "D18": "USD", + "D19": "USD", + "D20": "USD" + } + } + } + }, + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "agent_config": { + "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", + "allowed_tools": [ + "*" + ], + "disallowed_tools": [ + "setup", + "evaluate" + ], + "append_setup_tool": true, + "initial_screenshot": true + } + } +] \ No newline at end of file diff --git a/sheetbench_tasks.json b/sheetbench_tasks.json new file mode 100644 index 000000000..7e7671577 --- /dev/null +++ b/sheetbench_tasks.json @@ -0,0 +1,2411 @@ +[ + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nCalculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?", + "expected_cells": { + "A1": "1/12/2024", + "A2": "1/10/2024", + "A3": "1/15/2024", + "A4": "1/11/2024", + "A5": "1/17/2024" + } + }, + "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nCalculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?", + "expected_cells": { + "B2": "CustomerID", + "B3": "476232", + "B4": "963589", + "B5": "1430820", + "B6": "2410043", + "B7": "3789483", + "B8": "4226444", + "B9": "4308096", + "C2": "2023", + "C5": "686.40", + "C8": "966.76", + "C9": "863.37", + "D2": "2024", + "D3": "912.70", + "D4": "397.62", + "D6": "383.38", + "D7": "619.56", + "E2": "Grand Total", + "E3": "912.70", + "E4": "397.62", + "E5": "686.40", + "E6": "383.38", + "E7": "619.56", + "E8": "966.76", + "E9": "863.37", + "B10": "5907496", + "B11": "6224271", + "B12": "6538101", + "B13": "6584993", + "B14": "6791810", + "B15": "7250781", + "B16": "8518187", + "B17": "8668097", + "B18": "8885050", + "B19": "9296053", + "B20": "10414682", + "B21": "10839621", + "B22": "12313143", + "B23": "13672423", + "B24": "14744105", + "B25": "14780608", + "B26": "15144539", + "B27": "15616289", + "B28": "15694782", + "B29": "15904102", + "B30": "16385696", + "B31": "17254618", + "B32": "17441704", + "B33": "17624107", + "B34": "18043433", + "B35": "18296075", + "B36": "18600797", + "B37": "19026954", + "B38": "19395319", + "B39": "19650142", + "B40": "19842318", + "B41": "20241148", + "B42": "20587659", + "B43": "21647994", + "B44": "21671104", + "B45": "21935183", + "B46": "22115836", + "B47": "22846037", + "B48": "23621954", + "B49": "24962520", + "B50": "25073201", + "B51": "25090031", + "B52": "25096194", + "B53": "25211207", + "B54": "26168579", + "B55": "26250755", + "B56": "26419282", + "B57": "26953175", + "B58": "27281086", + "B59": "27646724", + "B60": "30370166", + "B61": "30738270", + "B62": "30876644", + "B63": "32255126", + "B64": "32373769", + "B65": "33121017", + "B66": "33897455", + "B67": "34507001", + "B68": "35475491", + "B69": "36972111", + "B70": "37040802", + "B71": "37543198", + "B72": "37609900", + "B73": "37674649", + "B74": "38019388", + "B75": "38658747", + "B76": "38793189", + "B77": "39134950", + "B78": "40399799", + "B79": "41039038", + "B80": "41271414", + "B81": "41870334", + "B82": "42495130", + "B83": "43189090", + "B84": "43235421", + "B85": "43837144", + "B86": "44115166", + "B87": "44270797", + "B88": "45380457", + "B89": "45386282", + "B90": "46196026", + "B91": "46300157", + "B92": "48512428", + "B93": "49546866", + "B94": "49687477", + "B95": "50893874", + "B96": "51093532", + "B97": "51126698", + "B98": "51397982", + "B99": "52376160", + "C10": "714.36", + "C11": "369.01", + "C12": "525.20", + "C20": "120.72", + "C21": "269.30", + "C22": "422.36", + "C26": "971.83", + "C28": "110.65", + "C29": "496.66", + "C30": "841.89", + "C31": "886.06", + "C32": "536.09", + "C34": "16.13", + "C38": "90.40", + "C39": "76.23", + "C40": "185.33", + "C41": "65.32", + "C42": "111.51", + "C43": "493.89", + "C44": "787.04", + "C45": "578.58", + "C46": "204.48", + "C48": "985.49", + "C50": "268.33", + "C52": "34.51", + "C54": "61.19", + "C60": "62.83", + "C63": "326.11", + "C64": "898.66", + "C65": "251.17", + "C66": "902.46", + "C67": "442.33", + "C70": "561.10", + "C71": "469.05", + "C74": "477.14", + "C76": "284.69", + "C80": "266.21", + "C82": "699.49", + "C85": "82.73", + "C87": "542.97", + "C88": "446.42", + "C89": "380.14", + "C91": "683.73", + "C92": "566.99", + "C94": "646.75", + "C97": "748.88", + "C98": "253.40", + "D13": "174.59", + "D14": "374.65", + "D15": "339.03", + "D16": "140.66", + "D17": "862.92", + "D18": "282.39", + "D19": "847.67", + "D23": "616.14", + "D24": "376.83", + "D25": "523.78", + "D27": "701.88", + "D29": "496.66", + "D33": "805.83", + "D35": "579.30", + "D36": "577.84", + "D37": "673.27", + "D38": "90.40", + "D47": "47.03", + "D48": "985.49", + "D49": "779.72", + "D51": "277.24", + "D53": "702.84", + "D55": "334.42", + "D56": "875.84", + "D57": "570.46", + "D58": "60.52", + "D59": "29.15", + "D61": "822.61", + "D62": "931.36", + "D68": "728.49", + "D69": "454.44", + "D72": "866.51", + "D73": "968.06", + "D74": "477.14", + "D75": "154.14", + "D77": "69.76", + "D78": "323.27", + "D79": "862.51", + "D81": "335.73", + "D83": "555.46", + "D84": "685.07", + "D86": "942.65", + "D90": "813.06", + "D93": "671.93", + "D95": "995.04", + "D96": "896.64", + "D98": "253.40", + "D99": "66.10", + "E10": "714.36", + "E11": "369.01", + "E12": "525.20", + "E13": "174.59", + "E14": "374.65", + "E15": "339.03", + "E16": "140.66", + "E17": "862.92", + "E18": "282.39", + "E19": "847.67", + "E20": "120.72", + "E21": "269.30", + "E22": "422.36", + "E23": "616.14", + "E24": "376.83", + "E25": "523.78", + "E26": "971.83", + "E27": "701.88", + "E28": "110.65", + "E29": "993.32", + "E30": "841.89", + "E31": "886.06", + "E32": "536.09", + "E33": "805.83", + "E34": "16.13", + "E35": "579.30", + "E36": "577.84", + "E37": "673.27", + "E38": "180.80", + "E39": "76.23", + "E40": "185.33", + "E41": "65.32", + "E42": "111.51", + "E43": "493.89", + "E44": "787.04", + "E45": "578.58", + "E46": "204.48", + "E47": "47.03", + "E48": "1970.98", + "E49": "779.72", + "E50": "268.33", + "E51": "277.24", + "E52": "34.51", + "E53": "702.84", + "E54": "61.19", + "E55": "334.42", + "E56": "875.84", + "E57": "570.46", + "E58": "60.52", + "E59": "29.15", + "E60": "62.83", + "E61": "822.61", + "E62": "931.36", + "E63": "326.11", + "E64": "898.66", + "E65": "251.17", + "E66": "902.46", + "E67": "442.33", + "E68": "728.49", + "E69": "454.44", + "E70": "561.10", + "E71": "469.05", + "E72": "866.51", + "E73": "968.06", + "E74": "954.28", + "E75": "154.14", + "E76": "284.69", + "E77": "69.76", + "E78": "323.27", + "E79": "862.51", + "E80": "266.21", + "E81": "335.73", + "E82": "699.49", + "E83": "555.46", + "E84": "685.07", + "E85": "82.73", + "E86": "942.65", + "E87": "542.97", + "E88": "446.42", + "E89": "380.14", + "E90": "813.06", + "E91": "683.73", + "E92": "566.99", + "E93": "671.93", + "E94": "646.75", + "E95": "995.04", + "E96": "896.64", + "E97": "748.88", + "E98": "506.80", + "E99": "66.10", + "B100": "52697630", + "B101": "52870804", + "B102": "53239020", + "B103": "53335630", + "B104": "53858146", + "B105": "54040755", + "B106": "54479395", + "B107": "54528192", + "B108": "54730083", + "B109": "55444897", + "B110": "55450065", + "B111": "55853989", + "B112": "55859479", + "B113": "55978545", + "B114": "56702798", + "B115": "56940756", + "B116": "58152560", + "B117": "59169574", + "B118": "59736806", + "B119": "61602489", + "B120": "62665919", + "B121": "62758192", + "B122": "62943277", + "B123": "63051186", + "B124": "65657133", + "B125": "65899458", + "B126": "66761945", + "B127": "66846840", + "B128": "67283606", + "B129": "67548873", + "B130": "69613838", + "B131": "69765117", + "B132": "70098167", + "B133": "70534368", + "B134": "71078229", + "B135": "71319902", + "B136": "71369648", + "B137": "71376141", + "B138": "72217443", + "B139": "72659809", + "B140": "72883709", + "B141": "73191421", + "B142": "73684220", + "B143": "74014279", + "B144": "74088126", + "B145": "75410476", + "B146": "75817527", + "B147": "77140593", + "B148": "77558800", + "B149": "78916470", + "B150": "79031936", + "B151": "79751742", + "B152": "80286530", + "B153": "80765899", + "B154": "82060237", + "B155": "82306595", + "B156": "83101113", + "B157": "83211478", + "B158": "83713620", + "B159": "84770820", + "B160": "84800206", + "B161": "84952943", + "B162": "86407021", + "B163": "86619158", + "B164": "86663007", + "B165": "87144451", + "B166": "87254792", + "B167": "88194202", + "B168": "88266169", + "B169": "88761732", + "B170": "88882642", + "B171": "89421277", + "B172": "89565544", + "B173": "90841330", + "B174": "91483447", + "B175": "91590435", + "B176": "91939241", + "B177": "92335820", + "B178": "92422728", + "B179": "92676749", + "B180": "93202190", + "B181": "93479509", + "B182": "95353791", + "B183": "95696393", + "B184": "95804583", + "B185": "95860510", + "B186": "96051566", + "B187": "96263709", + "B188": "96456101", + "B189": "99519803", + "B190": "Grand Total", + "C100": "409.46", + "C101": "28.08", + "C106": "184.81", + "C107": "906.16", + "C108": "1094.00", + "C109": "818.87", + "C110": "192.68", + "C111": "385.72", + "C117": "976.44", + "C118": "310.80", + "C119": "354.99", + "C120": "898.04", + "C122": "397.81", + "C128": "537.90", + "C129": "886.96", + "C131": "420.66", + "C133": "359.50", + "C134": "60.29", + "C139": "740.19", + "C142": "554.52", + "C144": "759.21", + "C145": "116.62", + "C148": "302.90", + "C152": "187.16", + "C153": "847.55", + "C154": "700.62", + "C155": "190.59", + "C161": "432.99", + "C163": "886.00", + "C167": "715.20", + "C169": "101.82", + "C171": "122.95", + "C174": "609.90", + "C175": "867.14", + "C177": "675.11", + "C178": "623.53", + "C182": "40.39", + "C184": "554.62", + "C187": "839.28", + "C189": "717.61", + "C190": "43541.41", + "D102": "995.08", + "D103": "761.74", + "D104": "773.13", + "D105": "364.27", + "D112": "841.73", + "D113": "560.15", + "D114": "824.63", + "D115": "783.50", + "D116": "788.60", + "D117": "976.44", + "D118": "310.80", + "D121": "92.40", + "D123": "171.84", + "D124": "336.61", + "D125": "206.92", + "D126": "345.03", + "D127": "505.00", + "D130": "539.76", + "D132": "189.69", + "D135": "713.31", + "D136": "993.98", + "D137": "541.38", + "D138": "790.14", + "D139": "740.19", + "D140": "481.88", + "D141": "83.31", + "D143": "883.45", + "D146": "394.60", + "D147": "281.52", + "D149": "64.48", + "D150": "645.82", + "D151": "771.06", + "D156": "405.74", + "D157": "741.29", + "D158": "196.84", + "D159": "152.71", + "D160": "356.93", + "D162": "138.69", + "D164": "929.08", + "D165": "887.13", + "D166": "554.21", + "D168": "820.22", + "D170": "939.97", + "D171": "122.95", + "D172": "335.69", + "D173": "917.55", + "D176": "514.59", + "D179": "288.94", + "D180": "214.12", + "D181": "16.74", + "D183": "27.28", + "D185": "458.58", + "D186": "648.09", + "D188": "985.01", + "D190": "56717.97", + "E100": "409.46", + "E101": "28.08", + "E102": "995.08", + "E103": "761.74", + "E104": "773.13", + "E105": "364.27", + "E106": "184.81", + "E107": "906.16", + "E108": "1094.00", + "E109": "818.87", + "E110": "192.68", + "E111": "385.72", + "E112": "841.73", + "E113": "560.15", + "E114": "824.63", + "E115": "783.50", + "E116": "788.60", + "E117": "1952.88", + "E118": "621.60", + "E119": "354.99", + "E120": "898.04", + "E121": "92.40", + "E122": "397.81", + "E123": "171.84", + "E124": "336.61", + "E125": "206.92", + "E126": "345.03", + "E127": "505.00", + "E128": "537.90", + "E129": "886.96", + "E130": "539.76", + "E131": "420.66", + "E132": "189.69", + "E133": "359.50", + "E134": "60.29", + "E135": "713.31", + "E136": "993.98", + "E137": "541.38", + "E138": "790.14", + "E139": "1480.38", + "E140": "481.88", + "E141": "83.31", + "E142": "554.52", + "E143": "883.45", + "E144": "759.21", + "E145": "116.62", + "E146": "394.60", + "E147": "281.52", + "E148": "302.90", + "E149": "64.48", + "E150": "645.82", + "E151": "771.06", + "E152": "187.16", + "E153": "847.55", + "E154": "700.62", + "E155": "190.59", + "E156": "405.74", + "E157": "741.29", + "E158": "196.84", + "E159": "152.71", + "E160": "356.93", + "E161": "432.99", + "E162": "138.69", + "E163": "886.00", + "E164": "929.08", + "E165": "887.13", + "E166": "554.21", + "E167": "715.20", + "E168": "820.22", + "E169": "101.82", + "E170": "939.97", + "E171": "245.90", + "E172": "335.69", + "E173": "917.55", + "E174": "609.90", + "E175": "867.14", + "E176": "514.59", + "E177": "675.11", + "E178": "623.53", + "E179": "288.94", + "E180": "214.12", + "E181": "16.74", + "E182": "40.39", + "E183": "27.28", + "E184": "554.62", + "E185": "458.58", + "E186": "648.09", + "E187": "839.28", + "E188": "985.01", + "E189": "717.61", + "E190": "100259.38" + } + }, + "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFind the GET request which most commonly results in an error. Place the URL in ANSWER!A1", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?", + "expected_cells": { + "A1": "/api/users" + } + }, + "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFor a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?", + "expected_cells": { + "A1": "2026-04" + } + }, + "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFor the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?", + "expected_cells": { + "A1": "ABC", + "B1": "4999972", + "C1": "145%" + } + }, + "id": "0963d367-f0ac-4be0-8f46-337bf335e68f" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?", + "expected_cells": { + "A1": "$112,281.49" + } + }, + "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?", + "expected_cells": { + "A1": "$24,642.86" + } + }, + "id": "49ccea4d-4aaf-4faf-9659-b389686568a7" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?", + "expected_cells": { + "A1": "1664934.45" + } + }, + "id": "48d43d57-ed1d-4df0-8b36-62380bba7865" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?", + "expected_cells": { + "A1": "MAY", + "A2": "OCT", + "B1": "95", + "B2": "73" + } + }, + "id": "ff114e80-196d-4a7a-99ca-152bac4fba90" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Category", + "A2": "Meals & Entertainment", + "A3": "Travel", + "A4": "Office Supplies", + "B1": "Sum of Amount", + "B2": "0", + "B3": "345", + "B4": "45760" + } + }, + "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?", + "expected_cells": { + "A3": "Sum of Amount (USD Millions)", + "A5": "Net Income", + "A6": "Revenue", + "A7": "Total Assets", + "A8": "Grand Total", + "B4": "FORD2024Q1", + "B5": "1.33", + "B6": "42.78", + "B7": "274.34", + "B8": "318.45", + "C4": "FORD2024Q2", + "C5": "1.83", + "C6": "44.81", + "C7": "276.59", + "C8": "323.23", + "D4": "FORD2024Q3", + "D5": "896.00", + "D6": "43.07", + "D7": "287.05", + "D8": "1226.12", + "E4": "FORD2025Q1", + "E5": "471.00", + "E6": "40.66", + "E7": "284.54", + "E8": "796.20", + "F4": "Grand Total", + "F5": "1370.17", + "F6": "171.32", + "F7": "1122.51", + "F8": "2664.00" + } + }, + "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?", + "expected_cells": { + "C3": "Count of TicketID", + "D3": "Average of ResolutionTimeHours", + "A4": "Agent-101", + "B4": "Closed", + "C4": "73", + "D4": "13.90", + "B5": "In Progress", + "C5": "30", + "B6": "Open", + "C6": "31", + "B7": "Resolved", + "C7": "66", + "D7": "22.04", + "B8": "Waiting for Customer", + "C8": "12", + "C9": "212", + "D9": "17.76", + "A10": "Agent-102", + "B10": "Closed", + "C10": "70", + "D10": "28.53", + "B11": "In Progress", + "C11": "36", + "B12": "Open", + "C12": "32", + "B13": "Resolved", + "C13": "77", + "D13": "17.09", + "B14": "Waiting for Customer", + "C14": "10", + "C15": "225", + "D15": "22.54", + "A16": "Agent-103", + "B16": "Closed", + "C16": "73", + "D16": "23.11", + "B17": "In Progress", + "C17": "39", + "B18": "Open", + "C18": "14", + "B19": "Resolved", + "C19": "92", + "D19": "17.08", + "B20": "Waiting for Customer", + "C20": "11", + "C21": "229", + "D21": "19.75", + "A22": "Agent-104", + "B22": "Closed", + "C22": "63", + "D22": "15.48", + "B23": "In Progress", + "C23": "32", + "B24": "Open", + "C24": "18", + "B25": "Resolved", + "C25": "97", + "D25": "15.40", + "B26": "Waiting for Customer", + "C26": "13", + "C27": "223", + "D27": "15.43", + "A28": "Agent-105", + "B28": "Closed", + "C28": "70", + "D28": "22.20", + "B29": "In Progress", + "C29": "27", + "B30": "Open", + "C30": "16", + "B31": "Resolved", + "C31": "69", + "D31": "15.21", + "B32": "Waiting for Customer", + "C32": "10", + "C33": "192", + "D33": "18.73", + "A34": "Agent-106", + "B34": "Closed", + "C34": "65", + "D34": "26.15", + "B35": "In Progress", + "C35": "38", + "B36": "Open", + "C36": "20", + "B37": "Resolved", + "C37": "87", + "D37": "21.56", + "B38": "Waiting for Customer", + "C38": "10", + "C39": "220", + "D39": "23.52", + "A40": "Agent-107", + "B40": "Closed", + "C40": "68", + "D40": "18.41", + "B41": "In Progress", + "C41": "35", + "B42": "Open", + "C42": "17", + "B43": "Resolved", + "C43": "106", + "D43": "16.52", + "B44": "Waiting for Customer", + "C44": "11", + "C45": "237", + "D45": "17.26", + "A46": "Agent-108", + "B46": "Closed", + "C46": "65", + "D46": "16.81", + "B47": "In Progress", + "C47": "33", + "B48": "Open", + "C48": "20", + "B49": "Resolved", + "C49": "93", + "D49": "14.93", + "B50": "Waiting for Customer", + "C50": "8", + "C51": "219", + "D51": "15.70", + "B52": "(blank)", + "C52": "243", + "D52": "20.98", + "B53": "Closed", + "C53": "74", + "D53": "24.44", + "B54": "In Progress", + "C54": "32", + "B55": "Open", + "C55": "26", + "B56": "Resolved", + "C56": "99", + "D56": "18.39", + "B57": "Waiting for Customer", + "C57": "12", + "B58": "Grand Total", + "C58": "2000", + "D58": "19.05" + } + }, + "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?", + "expected_cells": { + "B2": "1", + "B3": "2", + "B4": "3", + "B5": "4", + "B6": "5", + "B8": "1", + "B9": "2", + "C1": "Count of LastPromotionDate", + "C2": "12", + "C3": "13", + "C4": "10", + "C5": "15", + "C6": "5", + "C8": "1", + "C9": "5", + "D1": "Average of SalaryUSD", + "D2": "84965.91", + "D3": "87028.38", + "D4": "86094.90", + "D5": "78111.64", + "D6": "86015.00", + "D8": "82727.50", + "D9": "78876.00", + "A38": "Grand Total", + "B10": "3", + "B11": "4", + "B12": "5", + "B14": "1", + "B15": "2", + "B16": "3", + "B17": "4", + "B18": "5", + "B20": "1", + "B21": "2", + "B22": "3", + "B23": "4", + "B24": "5", + "B26": "1", + "B27": "2", + "B28": "3", + "B29": "4", + "B30": "5", + "B32": "1", + "B33": "2", + "B34": "3", + "B35": "4", + "B36": "5", + "C10": "4", + "C15": "3", + "C16": "2", + "C17": "2", + "C20": "1", + "C21": "4", + "C22": "6", + "C23": "3", + "C24": "5", + "C26": "10", + "C27": "9", + "C28": "4", + "C29": "6", + "C30": "4", + "C32": "5", + "C33": "2", + "C34": "4", + "C35": "4", + "C36": "1", + "C38": "140", + "D10": "76759.43", + "D11": "78318.00", + "D12": "80036.25", + "D14": "76599.00", + "D15": "93749.17", + "D16": "79929.00", + "D17": "88994.75", + "D18": "102902.00", + "D20": "77878.50", + "D21": "77480.38", + "D22": "84212.67", + "D23": "77508.83", + "D24": "77182.50", + "D26": "56034.08", + "D27": "52096.20", + "D28": "63190.71", + "D29": "60750.00", + "D30": "50299.25", + "D32": "71131.57", + "D33": "89669.50", + "D34": "81155.50", + "D35": "82037.80", + "D36": "61600.50", + "D38": "77770.00" + } + }, + "id": "7598db5d-0fd1-44db-ab1a-593cf026317b" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?", + "expected_cells": { + "A1": " 27,301,058.62 " + } + }, + "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Year", + "A2": "2025", + "A3": "2026", + "A4": "2027", + "A5": "2028", + "A6": "2029", + "A9": "Year", + "B1": "Revenue", + "B2": "108,000,000.00", + "B3": "116,640,000.00", + "B4": "125,971,200.00", + "B5": "136,048,896.00", + "B6": "146,932,807.68", + "B9": "Revenue", + "C1": "COGS", + "C2": "43,200,000.00", + "C3": "46,656,000.00", + "C4": "50,388,480.00", + "C5": "54,419,558.40", + "C6": "58,773,123.07", + "C9": "COGS", + "D1": "Operating Expenses", + "D2": "21,600,000.00", + "D3": "23,328,000.00", + "D4": "25,194,240.00", + "D5": "27,209,779.20", + "D6": "29,386,561.54", + "D9": "Operating Expenses", + "E1": "EBITDA", + "E2": "43,200,000.00", + "E3": "46,656,000.00", + "E4": "50,388,480.00", + "E5": "54,419,558.40", + "E6": "58,773,123.07", + "E9": "EBITDA", + "F1": "Net Income", + "F2": "34,560,000.00", + "F3": "37,324,800.00", + "F4": "40,310,784.00", + "F5": "43,535,646.72", + "F6": "47,018,498.46", + "F9": "Net Income", + "A10": "2025", + "A11": "2026", + "A12": "2027", + "A13": "2028", + "A14": "2029", + "A17": "Year", + "A18": "2025", + "A19": "2026", + "A20": "2027", + "A21": "2028", + "A22": "2029", + "B10": "112,000,000.00", + "B11": "125,440,000.00", + "B12": "140,492,800.00", + "B13": "157,351,936.00", + "B14": "176,234,168.32", + "B17": "Revenue", + "B18": "103,000,000.00", + "B19": "106,090,000.00", + "B20": "109,272,700.00", + "B21": "112,550,881.00", + "B22": "115,927,407.43", + "C10": "39,200,000.00", + "C11": "43,904,000.00", + "C12": "49,172,480.00", + "C13": "55,073,177.60", + "C14": "61,681,958.91", + "C17": "COGS", + "C18": "46,350,000.00", + "C19": "47,740,500.00", + "C20": "49,172,715.00", + "C21": "50,647,896.45", + "C22": "52,167,333.34", + "D10": "33,600,000.00", + "D11": "37,632,000.00", + "D12": "42,147,840.00", + "D13": "47,205,580.80", + "D14": "52,870,250.50", + "D17": "Operating Expenses", + "D18": "20,600,000.00", + "D19": "21,218,000.00", + "D20": "21,854,540.00", + "D21": "22,510,176.20", + "D22": "23,185,481.49", + "E10": "39,200,000.00", + "E11": "43,904,000.00", + "E12": "49,172,480.00", + "E13": "55,073,177.60", + "E14": "61,681,958.91", + "E17": "EBITDA", + "E18": "36,050,000.00", + "E19": "37,131,500.00", + "E20": "38,245,445.00", + "E21": "39,392,808.35", + "E22": "40,574,592.60", + "F10": "31,360,000.00", + "F11": "35,123,200.00", + "F12": "39,337,984.00", + "F13": "44,058,542.08", + "F14": "49,345,567.13", + "F17": "Net Income", + "F18": "28,840,000.00", + "F19": "29,705,200.00", + "F20": "30,596,356.00", + "F21": "31,514,246.68", + "F22": "32,459,674.08" + } + }, + "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?", + "expected_cells": { + "A4": "2024-04-01", + "A5": "2024-04-02", + "A6": "2024-04-03", + "A7": "2024-04-04", + "A8": "2024-04-05", + "A9": "2024-04-08", + "B4": "1.020", + "B5": "0.953", + "B6": "0.989", + "B7": "0.046", + "B8": "0.046", + "B9": "0.046", + "A10": "2024-04-09", + "A11": "2024-04-10", + "A12": "2024-04-11", + "A13": "2024-04-12", + "A14": "Grand Total", + "B10": "0.046", + "B11": "0.046", + "B12": "0.046", + "B13": "0.046", + "B14": "0.328" + } + }, + "id": "cea3b19a-6855-45f0-863e-42694346d487" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?", + "expected_cells": { + "A1": "1,758,109,357.43" + } + }, + "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?", + "expected_cells": { + "A2": "Kressa", + "A3": "Saumya", + "A4": "Jeane", + "A5": "Sarah", + "A6": "Anaiya", + "A7": "Varun", + "A8": "Sofie", + "B2": "$27.00", + "B3": "$30.50", + "B4": "$27.00", + "B5": "$18.00", + "B6": "$50.00", + "B7": "$52.00", + "B8": "$19.00", + "C2": "40", + "C3": "50", + "C4": "20", + "C5": "20", + "C6": "80", + "C7": "80", + "C8": "20", + "D2": "$1,080.00", + "D3": "$1,525.00", + "D4": "$540.00", + "D5": "$360.00", + "D6": "$4,000.00", + "D7": "$4,160.00", + "D8": "$380.00", + "E2": "$66.96", + "E3": "$94.55", + "E4": "$33.48", + "E5": "$22.32", + "E6": "$248.00", + "E7": "$257.92", + "E8": "$23.56", + "F2": "$15.66", + "F3": "$22.11", + "F4": "$7.83", + "F5": "$5.22", + "F6": "$58.00", + "F7": "$60.32", + "F8": "$5.51", + "G2": "$2.24", + "G3": "$2.80", + "G4": "$1.12", + "G5": "$1.12", + "G6": "$4.48", + "G7": "$4.48", + "G8": "$1.12", + "H2": "$0.32", + "H3": "$0.46", + "H4": "$0.16", + "H5": "$0.11", + "H6": "$1.20", + "H7": "$1.25", + "H8": "$0.11", + "I2": "$5.71", + "I3": "$8.06", + "I4": "$2.85", + "I5": "$1.90", + "I6": "$21.14", + "I7": "$21.99", + "I8": "$2.01", + "J2": "$6.26", + "J3": "$8.85", + "J4": "$3.13", + "J5": "$2.09", + "J6": "$23.20", + "J7": "$24.13", + "J8": "$2.20", + "K2": "$97.16", + "K3": "$136.82", + "K4": "$48.58", + "K5": "$32.76", + "K6": "$356.02", + "K7": "$370.08", + "K8": "$34.52" + } + }, + "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?", + "expected_cells": { + "B3": "Maturity", + "A4": "CurveDate", + "B4": "10Y", + "C4": "1Y", + "D4": "2Y", + "E4": "30Y", + "F4": "5Y", + "G4": "Grand Total", + "A5": "2024-01-01", + "B5": "$3.52", + "C5": "$3.73", + "D5": "$3.71", + "E5": "$3.86", + "F5": "$4.71", + "G5": "$19.53", + "A6": "2024-01-02", + "B6": "$3.73", + "C6": "$3.62", + "D6": "$3.79", + "E6": "$4.81", + "F6": "$4.54", + "G6": "$20.49", + "A7": "2024-01-03", + "B7": "$4.28", + "C7": "$4.98", + "D7": "$4.63", + "E7": "$3.70", + "F7": "$4.79", + "G7": "$22.38", + "A8": "2024-01-04", + "B8": "$4.03", + "C8": "$4.84", + "D8": "$3.94", + "E8": "$4.84", + "F8": "$4.49", + "G8": "$22.14", + "A9": "Grand Total", + "B9": "$15.56", + "C9": "$17.17", + "D9": "$16.07", + "E9": "$17.21", + "F9": "$18.53", + "G9": "$84.54" + } + }, + "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?", + "expected_cells": { + "A1": "ABC", + "B1": "-0.08" + } + }, + "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Carla White", + "A2": "$155.00", + "B1": "$8,758.00" + } + }, + "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Content Marketing", + "A2": "Email", + "A3": "PPC", + "A4": "SEO", + "A5": "Social Media", + "B1": "Harness Frictionless Users", + "B2": "Transform Out-Of-The-Box Schemas", + "B3": "Morph Back-End E-Business", + "B4": "Facilitate Dynamic Channels", + "B5": "Re-Intermediate Cutting-Edge Web-Readiness", + "C1": "6.61", + "C2": "5.99", + "C3": "5.02", + "C4": "5.49", + "C5": "5.68" + } + }, + "id": "1e4ede56-163f-4815-b634-7946cf9e60c0" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Omar Donovan", + "A2": "Omar Donovan" + } + }, + "id": "8499e399-48e6-4603-bd96-9b45f5aabed0" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?", + "expected_cells": { + "A1": "1876764.43" + } + }, + "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?", + "expected_cells": { + "A1": "991105", + "A2": "1044283", + "A3": "1095308", + "A4": "1169204", + "B1": "4299900" + } + }, + "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?", + "expected_cells": { + "A1": "16.74%", + "B1": "27.80%" + } + }, + "id": "764ca583-7091-4b7f-8663-5e996db26a2c" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Star City", + "B1": "39.11%" + } + }, + "id": "9f71aa71-07f7-4421-8757-bbccbeab0984" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Gotham", + "B1": "4" + } + }, + "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Finance", + "B1": "1,000" + } + }, + "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?", + "expected_cells": { + "A1": "21680.086" + } + }, + "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Scenario 3", + "A2": "Scenario 2", + "A3": "Scenario 1" + } + }, + "id": "d2c56ee0-7863-45be-b503-eb1639454629" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Vendor", + "A2": "vendor 18", + "A3": "vendor 15", + "A4": "vendor 19", + "A5": "vendor 1", + "A6": "vendor 14", + "A7": "vendor 9", + "A8": "vendor 8", + "A9": "vendor 11", + "B1": "1099 Report amount", + "B2": "$4.00", + "B3": "$48.00", + "B4": "$81.00", + "B5": "$92.60", + "B6": "$177.00", + "B7": "$866.00", + "B8": "$1,661.00", + "B9": "$2,615.00", + "A10": "vendor 4", + "A11": "vendor 5", + "B10": "$6,446.00", + "B11": "$6,519.00" + } + }, + "id": "077926f8-6e93-406b-9eb9-b44907173c74" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?", + "expected_cells": { + "A1": "2025-02" + } + }, + "id": "f1afee7f-df70-4a11-a65e-767a718f2117" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?", + "expected_cells": { + "A4": "Platform", + "A5": "Facebook", + "A9": "Instagram", + "B4": "Values", + "B5": "Sum of Posts", + "B6": "Sum of Likes", + "B7": "Sum of Shares", + "B8": "Sum of Comments", + "B9": "Sum of Posts", + "C4": "August", + "C5": "66", + "C6": "3899", + "C7": "618", + "C8": "241", + "C9": "54", + "D4": "July", + "D5": "65", + "D6": "4008", + "D7": "749", + "D8": "260", + "D9": "58", + "E4": "June", + "E5": "54", + "E6": "3236", + "E7": "560", + "E8": "296", + "E9": "70", + "F4": "November", + "F5": "39", + "F6": "2066", + "F7": "430", + "F8": "209", + "F9": "52", + "G4": "October", + "G5": "68", + "G6": "3048", + "G7": "773", + "G8": "271", + "G9": "54", + "H4": "September", + "H5": "64", + "H6": "2918", + "H7": "769", + "H8": "262", + "H9": "57", + "I4": "Grand Total", + "I5": "356", + "I6": "19175", + "I7": "3899", + "I8": "1539", + "I9": "345", + "A13": "LinkedIn", + "A17": "Twitter", + "A21": "Grand Total", + "B10": "Sum of Likes", + "B11": "Sum of Shares", + "B12": "Sum of Comments", + "B13": "Sum of Posts", + "B14": "Sum of Likes", + "B15": "Sum of Shares", + "B16": "Sum of Comments", + "B17": "Sum of Posts", + "B18": "Sum of Likes", + "B19": "Sum of Shares", + "B20": "Sum of Comments", + "B21": "Sum of Posts", + "B22": "Sum of Likes", + "B23": "Sum of Shares", + "B24": "Sum of Comments", + "C10": "3231", + "C11": "541", + "C12": "226", + "C13": "61", + "C14": "2923", + "C15": "668", + "C16": "295", + "C17": "60", + "C18": "3290", + "C19": "671", + "C20": "295", + "C21": "241", + "C22": "13343", + "C23": "2498", + "C24": "1057", + "D10": "3025", + "D11": "700", + "D12": "232", + "D13": "60", + "D14": "3122", + "D15": "589", + "D16": "314", + "D17": "54", + "D18": "2950", + "D19": "454", + "D20": "271", + "D21": "237", + "D22": "13105", + "D23": "2492", + "D24": "1077", + "E10": "4195", + "E11": "834", + "E12": "317", + "E13": "45", + "E14": "2291", + "E15": "451", + "E16": "185", + "E17": "63", + "E18": "3366", + "E19": "797", + "E20": "278", + "E21": "232", + "E22": "13088", + "E23": "2642", + "E24": "1076", + "F10": "2860", + "F11": "514", + "F12": "249", + "F13": "61", + "F14": "3116", + "F15": "708", + "F16": "246", + "F17": "63", + "F18": "2714", + "F19": "581", + "F20": "243", + "F21": "215", + "F22": "10756", + "F23": "2233", + "F24": "947", + "G10": "2993", + "G11": "623", + "G12": "265", + "G13": "70", + "G14": "3596", + "G15": "716", + "G16": "336", + "G17": "58", + "G18": "2382", + "G19": "578", + "G20": "293", + "G21": "250", + "G22": "12019", + "G23": "2690", + "G24": "1165", + "H10": "3409", + "H11": "401", + "H12": "313", + "H13": "65", + "H14": "3271", + "H15": "771", + "H16": "278", + "H17": "51", + "H18": "2156", + "H19": "431", + "H20": "312", + "H21": "237", + "H22": "11754", + "H23": "2372", + "H24": "1165", + "I10": "19713", + "I11": "3613", + "I12": "1602", + "I13": "362", + "I14": "18319", + "I15": "3903", + "I16": "1654", + "I17": "349", + "I18": "16858", + "I19": "3512", + "I20": "1692", + "I21": "1412", + "I22": "74065", + "I23": "14927", + "I24": "6487" + } + }, + "id": "36e43b84-e583-4f02-a160-74049bcab901" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Platform", + "A2": "Twitter", + "A3": "Facebook", + "A4": "Instagram", + "A5": "LinkedIn", + "B1": "RATIO OF HIGH / LOW", + "B2": "1.88", + "B3": "2.64", + "B4": "3.17", + "B5": "2.05" + } + }, + "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?", + "expected_cells": { + "A4": "Jun", + "A5": "Jul", + "A6": "Aug", + "A7": "Sep", + "A8": "Oct", + "A9": "Nov", + "B3": "Sum of Posts", + "B4": "124", + "B5": "123", + "B6": "120", + "B7": "121", + "B8": "122", + "B9": "91", + "C3": "Sum of Likes", + "C4": "7431", + "C5": "7033", + "C6": "7130", + "C7": "6327", + "C8": "6041", + "C9": "4926", + "D3": "Sum of Comments", + "D4": "613", + "D5": "492", + "D6": "467", + "D7": "575", + "D8": "536", + "D9": "458", + "E3": "Sum of Shares", + "E4": "1394", + "E5": "1449", + "E6": "1159", + "E7": "1170", + "E8": "1396", + "E9": "944", + "A10": "Grand Total", + "B10": "701", + "C10": "38888", + "D10": "3141", + "E10": "7512" + } + }, + "id": "c437f78d-6382-43b2-b857-153db6efa1c9" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.68%", + "E3": "43.14%", + "E4": "43.77%", + "E5": "43.27%", + "E6": "44.90%", + "E7": "44.03%", + "E8": "44.06%", + "E9": "43.97%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "E10": "44.37%", + "E11": "44.10%", + "E12": "46.70%", + "E13": "48.06%", + "E14": "45.94%", + "E15": "45.48%", + "E16": "44.06%", + "E17": "43.93%" + } + }, + "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Month", + "A2": "1", + "A3": "2", + "A4": "3", + "A5": "4", + "A6": "5", + "A7": "6", + "A8": "7", + "A9": "8", + "B1": "Year", + "B2": "2023", + "B3": "2023", + "B4": "2023", + "B5": "2023", + "B6": "2023", + "B7": "2023", + "B8": "2023", + "B9": "2023", + "C1": "Total Monthly Unique Users", + "C2": "21928", + "C3": "19296", + "C4": "21453", + "C5": "20987", + "C6": "21944", + "C7": "21024", + "C8": "20875", + "C9": "21495", + "D1": "Total Monthly Page Views", + "D2": "34560", + "D3": "32162", + "D4": "35497", + "D5": "34263", + "D6": "35875", + "D7": "34896", + "D8": "36121", + "D9": "35382", + "E1": "Avg Monthly Bounce Rate (%)", + "E2": "44.7%", + "E3": "43.1%", + "E4": "43.8%", + "E5": "43.3%", + "E6": "44.9%", + "E7": "44.0%", + "E8": "44.1%", + "E9": "44.0%", + "A10": "9", + "A11": "10", + "A12": "11", + "A13": "12", + "A14": "1", + "A15": "2", + "A16": "3", + "A17": "4", + "A18": "5", + "A19": "6", + "A20": "7", + "A21": "8", + "A22": "9", + "A23": "10", + "A24": "11", + "A25": "12", + "B10": "2023", + "B11": "2023", + "B12": "2023", + "B13": "2023", + "B14": "2024", + "B15": "2024", + "B16": "2024", + "B17": "2024", + "B18": "2024", + "B19": "2024", + "B20": "2024", + "B21": "2024", + "B22": "2024", + "B23": "2024", + "B24": "2024", + "B25": "2024", + "C10": "21054", + "C11": "21153", + "C12": "20957", + "C13": "21493", + "C14": "21238", + "C15": "20205", + "C16": "21617", + "C17": "21094", + "C18": "21095", + "C19": "21015", + "C20": "21141", + "C21": "21311", + "C22": "21064", + "C23": "21182", + "C24": "21209", + "C25": "21238", + "D10": "34959", + "D11": "36723", + "D12": "34626", + "D13": "35262", + "D14": "36110", + "D15": "33484", + "D16": "34397", + "D17": "34643", + "D18": "34237", + "D19": "33819", + "D20": "33547", + "D21": "33832", + "D22": "33424", + "D23": "33162", + "D24": "33042", + "D25": "32929", + "E10": "44.4%", + "E11": "44.1%", + "E12": "46.7%", + "E13": "48.1%", + "E14": "45.9%", + "E15": "45.5%", + "E16": "44.1%", + "E17": "43.9%", + "E18": "43.1%", + "E19": "41.8%", + "E20": "41.3%", + "E21": "40.4%", + "E22": "39.7%", + "E23": "38.7%", + "E24": "37.9%", + "E25": "37.2%" + } + }, + "id": "9ba9631e-8560-4dc4-a285-8d186715a542" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?", + "expected_cells": { + "A1": "2023-04-30", + "A2": "0.48", + "A3": "0.00296" + } + }, + "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?", + "expected_cells": { + "D2": "Rank Based on 2024 Sales", + "D3": "4", + "D4": "3", + "D5": "7", + "D6": "17", + "D7": "12", + "D8": "1", + "D9": "18", + "D10": "8", + "D11": "6", + "D12": "14", + "D13": "5", + "D14": "13", + "D15": "9", + "D16": "20", + "D17": "11", + "D18": "10", + "D19": "19", + "D20": "15", + "D21": "2", + "D22": "16" + } + }, + "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?", + "expected_cells": { + "A2": "Region", + "A3": "Central", + "A4": "East", + "A5": "North", + "A6": "South", + "A7": "West", + "B2": "2023", + "B3": "351351.87", + "B4": "364129.72", + "B5": "377252.87", + "B6": "396259.93", + "B7": "395672.78", + "C2": "2024", + "C3": "368644.54", + "C4": "399862.83", + "C5": "364113.70", + "C6": "345896.80", + "C7": "393456.82", + "D2": "YoY Growth", + "D3": "4.92%", + "D4": "9.81%", + "D5": "-3.48%", + "D6": "-12.71%", + "D7": "-0.56%" + } + }, + "id": "20507108-4188-402e-8cfe-2354309bad6c" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Shopping", + "B1": "$ (16,401.91)" + } + }, + "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?", + "expected_cells": { + "A1": "6.17%" + } + }, + "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?", + "expected_cells": { + "A1": "24.20%" + } + }, + "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?", + "expected_cells": { + "A1": "$316,309.56" + } + }, + "id": "371507f7-721a-457e-9618-bc8fba1b909b" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?", + "expected_cells": { + "A1": "2026-03", + "A2": "2025-09", + "A3": "2026-12" + } + }, + "id": "4a081ed2-532c-4895-a034-ab0076927c7c" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?", + "expected_cells": { + "A1": "JPY", + "A2": "USD", + "A3": "EUR", + "A4": "GBP", + "B1": "3500000.00", + "B2": "64351.25", + "B3": "43501.00", + "B4": "15000.25" + } + }, + "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nThis spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?", + "expected_cells": { + "A1": "1.07%", + "A2": "1.73%" + } + }, + "id": "31508ec6-c993-4e00-b70c-093dba016fcc" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nWhich city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Central City" + } + }, + "id": "133ef005-207c-490f-b55c-7734fd1678da" + }, + { + "env": { + "name": "hud-remote-browser" + }, + "scenario": "remote-browser:sheet-from-file", + "args": { + "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nWork in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header \u201cDate\u201d plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", + "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?", + "expected_cells": { + "A1": "Date", + "A2": "2025-01-05", + "A3": "2025-01-12", + "A4": "2025-01-15", + "A5": "2025-01-20", + "A6": "2025-01-25", + "A7": "2025-01-30", + "A8": "2025-02-05", + "A9": "2025-02-08", + "B1": "Description", + "B2": "Membership Fee", + "B3": "Project Income", + "B4": "Interest", + "B5": "Event Revenue", + "B6": "Refund", + "B7": "Consulting Fee", + "B8": "Website Hosting", + "B9": "Maintenance", + "C1": "Amount", + "C2": "-250.00", + "C3": "5600.00", + "C4": "350.00", + "C5": "7000.00", + "C6": "-5000.00", + "C7": "7800.00", + "C8": "-99.99", + "C9": "-750.25", + "D1": "Currency", + "D2": "USD", + "D3": "USD", + "D4": "USD", + "D5": "USD", + "D6": "USD", + "D7": "USD", + "D8": "USD", + "D9": "USD", + "A10": "2025-02-15", + "A11": "2025-02-18", + "A12": "2025-02-20", + "A13": "2025-02-25", + "A14": "2025-02-28", + "A15": "2025-03-01", + "A16": "2025-03-05", + "A17": "2025-03-10", + "A18": "2025-03-15", + "A19": "2025-03-20", + "A20": "2025-03-25", + "B10": "Invoice Payment", + "B11": "Equipment Purchase", + "B12": "Software License", + "B13": "Bonus", + "B14": "Travel Expenses", + "B15": "Subscription", + "B16": "Advertising", + "B17": "Office Supplies", + "B18": "Utilities", + "B19": "Legal Fees", + "B20": "Marketing", + "C10": "2500.00", + "C11": "-3600.00", + "C12": "-3000.00", + "C13": "4800.00", + "C14": "-1750.00", + "C15": "-1200.00", + "C16": "-2450.50", + "C17": "-450.75", + "C18": "-899.00", + "C19": "-4000.00", + "C20": "-2200.00", + "D10": "USD", + "D11": "USD", + "D12": "USD", + "D13": "USD", + "D14": "USD", + "D15": "USD", + "D16": "USD", + "D17": "USD", + "D18": "USD", + "D19": "USD", + "D20": "USD" + } + }, + "id": "eb410896-3e1e-4491-9460-061579b65c6f" + } +] \ No newline at end of file From ff91f24a0fce0b0d89b10e2cd2d1383ea2ee94a9 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 16:35:59 -0800 Subject: [PATCH 25/27] rm commit --- SheetBench-50 copy.json | 4211 --------------------------------------- SheetBench-50.json | 4211 --------------------------------------- sheetbench_tasks.json | 2411 ---------------------- 3 files changed, 10833 deletions(-) delete mode 100644 SheetBench-50 copy.json delete mode 100644 SheetBench-50.json delete mode 100644 sheetbench_tasks.json diff --git a/SheetBench-50 copy.json b/SheetBench-50 copy.json deleted file mode 100644 index fcaf77191..000000000 --- a/SheetBench-50 copy.json +++ /dev/null @@ -1,4211 +0,0 @@ -[ - { - "prompt": "Calculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1/12/2024", - "A2": "1/10/2024", - "A3": "1/15/2024", - "A4": "1/11/2024", - "A5": "1/17/2024" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Calculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B2": "CustomerID", - "B3": "476232", - "B4": "963589", - "B5": "1430820", - "B6": "2410043", - "B7": "3789483", - "B8": "4226444", - "B9": "4308096", - "C2": "2023", - "C5": "686.40", - "C8": "966.76", - "C9": "863.37", - "D2": "2024", - "D3": "912.70", - "D4": "397.62", - "D6": "383.38", - "D7": "619.56", - "E2": "Grand Total", - "E3": "912.70", - "E4": "397.62", - "E5": "686.40", - "E6": "383.38", - "E7": "619.56", - "E8": "966.76", - "E9": "863.37", - "B10": "5907496", - "B11": "6224271", - "B12": "6538101", - "B13": "6584993", - "B14": "6791810", - "B15": "7250781", - "B16": "8518187", - "B17": "8668097", - "B18": "8885050", - "B19": "9296053", - "B20": "10414682", - "B21": "10839621", - "B22": "12313143", - "B23": "13672423", - "B24": "14744105", - "B25": "14780608", - "B26": "15144539", - "B27": "15616289", - "B28": "15694782", - "B29": "15904102", - "B30": "16385696", - "B31": "17254618", - "B32": "17441704", - "B33": "17624107", - "B34": "18043433", - "B35": "18296075", - "B36": "18600797", - "B37": "19026954", - "B38": "19395319", - "B39": "19650142", - "B40": "19842318", - "B41": "20241148", - "B42": "20587659", - "B43": "21647994", - "B44": "21671104", - "B45": "21935183", - "B46": "22115836", - "B47": "22846037", - "B48": "23621954", - "B49": "24962520", - "B50": "25073201", - "B51": "25090031", - "B52": "25096194", - "B53": "25211207", - "B54": "26168579", - "B55": "26250755", - "B56": "26419282", - "B57": "26953175", - "B58": "27281086", - "B59": "27646724", - "B60": "30370166", - "B61": "30738270", - "B62": "30876644", - "B63": "32255126", - "B64": "32373769", - "B65": "33121017", - "B66": "33897455", - "B67": "34507001", - "B68": "35475491", - "B69": "36972111", - "B70": "37040802", - "B71": "37543198", - "B72": "37609900", - "B73": "37674649", - "B74": "38019388", - "B75": "38658747", - "B76": "38793189", - "B77": "39134950", - "B78": "40399799", - "B79": "41039038", - "B80": "41271414", - "B81": "41870334", - "B82": "42495130", - "B83": "43189090", - "B84": "43235421", - "B85": "43837144", - "B86": "44115166", - "B87": "44270797", - "B88": "45380457", - "B89": "45386282", - "B90": "46196026", - "B91": "46300157", - "B92": "48512428", - "B93": "49546866", - "B94": "49687477", - "B95": "50893874", - "B96": "51093532", - "B97": "51126698", - "B98": "51397982", - "B99": "52376160", - "C10": "714.36", - "C11": "369.01", - "C12": "525.20", - "C20": "120.72", - "C21": "269.30", - "C22": "422.36", - "C26": "971.83", - "C28": "110.65", - "C29": "496.66", - "C30": "841.89", - "C31": "886.06", - "C32": "536.09", - "C34": "16.13", - "C38": "90.40", - "C39": "76.23", - "C40": "185.33", - "C41": "65.32", - "C42": "111.51", - "C43": "493.89", - "C44": "787.04", - "C45": "578.58", - "C46": "204.48", - "C48": "985.49", - "C50": "268.33", - "C52": "34.51", - "C54": "61.19", - "C60": "62.83", - "C63": "326.11", - "C64": "898.66", - "C65": "251.17", - "C66": "902.46", - "C67": "442.33", - "C70": "561.10", - "C71": "469.05", - "C74": "477.14", - "C76": "284.69", - "C80": "266.21", - "C82": "699.49", - "C85": "82.73", - "C87": "542.97", - "C88": "446.42", - "C89": "380.14", - "C91": "683.73", - "C92": "566.99", - "C94": "646.75", - "C97": "748.88", - "C98": "253.40", - "D13": "174.59", - "D14": "374.65", - "D15": "339.03", - "D16": "140.66", - "D17": "862.92", - "D18": "282.39", - "D19": "847.67", - "D23": "616.14", - "D24": "376.83", - "D25": "523.78", - "D27": "701.88", - "D29": "496.66", - "D33": "805.83", - "D35": "579.30", - "D36": "577.84", - "D37": "673.27", - "D38": "90.40", - "D47": "47.03", - "D48": "985.49", - "D49": "779.72", - "D51": "277.24", - "D53": "702.84", - "D55": "334.42", - "D56": "875.84", - "D57": "570.46", - "D58": "60.52", - "D59": "29.15", - "D61": "822.61", - "D62": "931.36", - "D68": "728.49", - "D69": "454.44", - "D72": "866.51", - "D73": "968.06", - "D74": "477.14", - "D75": "154.14", - "D77": "69.76", - "D78": "323.27", - "D79": "862.51", - "D81": "335.73", - "D83": "555.46", - "D84": "685.07", - "D86": "942.65", - "D90": "813.06", - "D93": "671.93", - "D95": "995.04", - "D96": "896.64", - "D98": "253.40", - "D99": "66.10", - "E10": "714.36", - "E11": "369.01", - "E12": "525.20", - "E13": "174.59", - "E14": "374.65", - "E15": "339.03", - "E16": "140.66", - "E17": "862.92", - "E18": "282.39", - "E19": "847.67", - "E20": "120.72", - "E21": "269.30", - "E22": "422.36", - "E23": "616.14", - "E24": "376.83", - "E25": "523.78", - "E26": "971.83", - "E27": "701.88", - "E28": "110.65", - "E29": "993.32", - "E30": "841.89", - "E31": "886.06", - "E32": "536.09", - "E33": "805.83", - "E34": "16.13", - "E35": "579.30", - "E36": "577.84", - "E37": "673.27", - "E38": "180.80", - "E39": "76.23", - "E40": "185.33", - "E41": "65.32", - "E42": "111.51", - "E43": "493.89", - "E44": "787.04", - "E45": "578.58", - "E46": "204.48", - "E47": "47.03", - "E48": "1970.98", - "E49": "779.72", - "E50": "268.33", - "E51": "277.24", - "E52": "34.51", - "E53": "702.84", - "E54": "61.19", - "E55": "334.42", - "E56": "875.84", - "E57": "570.46", - "E58": "60.52", - "E59": "29.15", - "E60": "62.83", - "E61": "822.61", - "E62": "931.36", - "E63": "326.11", - "E64": "898.66", - "E65": "251.17", - "E66": "902.46", - "E67": "442.33", - "E68": "728.49", - "E69": "454.44", - "E70": "561.10", - "E71": "469.05", - "E72": "866.51", - "E73": "968.06", - "E74": "954.28", - "E75": "154.14", - "E76": "284.69", - "E77": "69.76", - "E78": "323.27", - "E79": "862.51", - "E80": "266.21", - "E81": "335.73", - "E82": "699.49", - "E83": "555.46", - "E84": "685.07", - "E85": "82.73", - "E86": "942.65", - "E87": "542.97", - "E88": "446.42", - "E89": "380.14", - "E90": "813.06", - "E91": "683.73", - "E92": "566.99", - "E93": "671.93", - "E94": "646.75", - "E95": "995.04", - "E96": "896.64", - "E97": "748.88", - "E98": "506.80", - "E99": "66.10", - "B100": "52697630", - "B101": "52870804", - "B102": "53239020", - "B103": "53335630", - "B104": "53858146", - "B105": "54040755", - "B106": "54479395", - "B107": "54528192", - "B108": "54730083", - "B109": "55444897", - "B110": "55450065", - "B111": "55853989", - "B112": "55859479", - "B113": "55978545", - "B114": "56702798", - "B115": "56940756", - "B116": "58152560", - "B117": "59169574", - "B118": "59736806", - "B119": "61602489", - "B120": "62665919", - "B121": "62758192", - "B122": "62943277", - "B123": "63051186", - "B124": "65657133", - "B125": "65899458", - "B126": "66761945", - "B127": "66846840", - "B128": "67283606", - "B129": "67548873", - "B130": "69613838", - "B131": "69765117", - "B132": "70098167", - "B133": "70534368", - "B134": "71078229", - "B135": "71319902", - "B136": "71369648", - "B137": "71376141", - "B138": "72217443", - "B139": "72659809", - "B140": "72883709", - "B141": "73191421", - "B142": "73684220", - "B143": "74014279", - "B144": "74088126", - "B145": "75410476", - "B146": "75817527", - "B147": "77140593", - "B148": "77558800", - "B149": "78916470", - "B150": "79031936", - "B151": "79751742", - "B152": "80286530", - "B153": "80765899", - "B154": "82060237", - "B155": "82306595", - "B156": "83101113", - "B157": "83211478", - "B158": "83713620", - "B159": "84770820", - "B160": "84800206", - "B161": "84952943", - "B162": "86407021", - "B163": "86619158", - "B164": "86663007", - "B165": "87144451", - "B166": "87254792", - "B167": "88194202", - "B168": "88266169", - "B169": "88761732", - "B170": "88882642", - "B171": "89421277", - "B172": "89565544", - "B173": "90841330", - "B174": "91483447", - "B175": "91590435", - "B176": "91939241", - "B177": "92335820", - "B178": "92422728", - "B179": "92676749", - "B180": "93202190", - "B181": "93479509", - "B182": "95353791", - "B183": "95696393", - "B184": "95804583", - "B185": "95860510", - "B186": "96051566", - "B187": "96263709", - "B188": "96456101", - "B189": "99519803", - "B190": "Grand Total", - "C100": "409.46", - "C101": "28.08", - "C106": "184.81", - "C107": "906.16", - "C108": "1094.00", - "C109": "818.87", - "C110": "192.68", - "C111": "385.72", - "C117": "976.44", - "C118": "310.80", - "C119": "354.99", - "C120": "898.04", - "C122": "397.81", - "C128": "537.90", - "C129": "886.96", - "C131": "420.66", - "C133": "359.50", - "C134": "60.29", - "C139": "740.19", - "C142": "554.52", - "C144": "759.21", - "C145": "116.62", - "C148": "302.90", - "C152": "187.16", - "C153": "847.55", - "C154": "700.62", - "C155": "190.59", - "C161": "432.99", - "C163": "886.00", - "C167": "715.20", - "C169": "101.82", - "C171": "122.95", - "C174": "609.90", - "C175": "867.14", - "C177": "675.11", - "C178": "623.53", - "C182": "40.39", - "C184": "554.62", - "C187": "839.28", - "C189": "717.61", - "C190": "43541.41", - "D102": "995.08", - "D103": "761.74", - "D104": "773.13", - "D105": "364.27", - "D112": "841.73", - "D113": "560.15", - "D114": "824.63", - "D115": "783.50", - "D116": "788.60", - "D117": "976.44", - "D118": "310.80", - "D121": "92.40", - "D123": "171.84", - "D124": "336.61", - "D125": "206.92", - "D126": "345.03", - "D127": "505.00", - "D130": "539.76", - "D132": "189.69", - "D135": "713.31", - "D136": "993.98", - "D137": "541.38", - "D138": "790.14", - "D139": "740.19", - "D140": "481.88", - "D141": "83.31", - "D143": "883.45", - "D146": "394.60", - "D147": "281.52", - "D149": "64.48", - "D150": "645.82", - "D151": "771.06", - "D156": "405.74", - "D157": "741.29", - "D158": "196.84", - "D159": "152.71", - "D160": "356.93", - "D162": "138.69", - "D164": "929.08", - "D165": "887.13", - "D166": "554.21", - "D168": "820.22", - "D170": "939.97", - "D171": "122.95", - "D172": "335.69", - "D173": "917.55", - "D176": "514.59", - "D179": "288.94", - "D180": "214.12", - "D181": "16.74", - "D183": "27.28", - "D185": "458.58", - "D186": "648.09", - "D188": "985.01", - "D190": "56717.97", - "E100": "409.46", - "E101": "28.08", - "E102": "995.08", - "E103": "761.74", - "E104": "773.13", - "E105": "364.27", - "E106": "184.81", - "E107": "906.16", - "E108": "1094.00", - "E109": "818.87", - "E110": "192.68", - "E111": "385.72", - "E112": "841.73", - "E113": "560.15", - "E114": "824.63", - "E115": "783.50", - "E116": "788.60", - "E117": "1952.88", - "E118": "621.60", - "E119": "354.99", - "E120": "898.04", - "E121": "92.40", - "E122": "397.81", - "E123": "171.84", - "E124": "336.61", - "E125": "206.92", - "E126": "345.03", - "E127": "505.00", - "E128": "537.90", - "E129": "886.96", - "E130": "539.76", - "E131": "420.66", - "E132": "189.69", - "E133": "359.50", - "E134": "60.29", - "E135": "713.31", - "E136": "993.98", - "E137": "541.38", - "E138": "790.14", - "E139": "1480.38", - "E140": "481.88", - "E141": "83.31", - "E142": "554.52", - "E143": "883.45", - "E144": "759.21", - "E145": "116.62", - "E146": "394.60", - "E147": "281.52", - "E148": "302.90", - "E149": "64.48", - "E150": "645.82", - "E151": "771.06", - "E152": "187.16", - "E153": "847.55", - "E154": "700.62", - "E155": "190.59", - "E156": "405.74", - "E157": "741.29", - "E158": "196.84", - "E159": "152.71", - "E160": "356.93", - "E161": "432.99", - "E162": "138.69", - "E163": "886.00", - "E164": "929.08", - "E165": "887.13", - "E166": "554.21", - "E167": "715.20", - "E168": "820.22", - "E169": "101.82", - "E170": "939.97", - "E171": "245.90", - "E172": "335.69", - "E173": "917.55", - "E174": "609.90", - "E175": "867.14", - "E176": "514.59", - "E177": "675.11", - "E178": "623.53", - "E179": "288.94", - "E180": "214.12", - "E181": "16.74", - "E182": "40.39", - "E183": "27.28", - "E184": "554.62", - "E185": "458.58", - "E186": "648.09", - "E187": "839.28", - "E188": "985.01", - "E189": "717.61", - "E190": "100259.38" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Find the GET request which most commonly results in an error. Place the URL in ANSWER!A1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "/api/users" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "For a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2026-04" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "For the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0963d367-f0ac-4be0-8f46-337bf335e68f", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "ABC", - "B1": "4999972", - "C1": "145%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$112,281.49" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "49ccea4d-4aaf-4faf-9659-b389686568a7", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$24,642.86" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "48d43d57-ed1d-4df0-8b36-62380bba7865", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1664934.45" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "ff114e80-196d-4a7a-99ca-152bac4fba90", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "MAY", - "A2": "OCT", - "B1": "95", - "B2": "73" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Category", - "A2": "Meals & Entertainment", - "A3": "Travel", - "A4": "Office Supplies", - "B1": "Sum of Amount", - "B2": "0", - "B3": "345", - "B4": "45760" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A3": "Sum of Amount (USD Millions)", - "A5": "Net Income", - "A6": "Revenue", - "A7": "Total Assets", - "A8": "Grand Total", - "B4": "FORD2024Q1", - "B5": "1.33", - "B6": "42.78", - "B7": "274.34", - "B8": "318.45", - "C4": "FORD2024Q2", - "C5": "1.83", - "C6": "44.81", - "C7": "276.59", - "C8": "323.23", - "D4": "FORD2024Q3", - "D5": "896.00", - "D6": "43.07", - "D7": "287.05", - "D8": "1226.12", - "E4": "FORD2025Q1", - "E5": "471.00", - "E6": "40.66", - "E7": "284.54", - "E8": "796.20", - "F4": "Grand Total", - "F5": "1370.17", - "F6": "171.32", - "F7": "1122.51", - "F8": "2664.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "C3": "Count of TicketID", - "D3": "Average of ResolutionTimeHours", - "A4": "Agent-101", - "B4": "Closed", - "C4": "73", - "D4": "13.90", - "B5": "In Progress", - "C5": "30", - "B6": "Open", - "C6": "31", - "B7": "Resolved", - "C7": "66", - "D7": "22.04", - "B8": "Waiting for Customer", - "C8": "12", - "C9": "212", - "D9": "17.76", - "A10": "Agent-102", - "B10": "Closed", - "C10": "70", - "D10": "28.53", - "B11": "In Progress", - "C11": "36", - "B12": "Open", - "C12": "32", - "B13": "Resolved", - "C13": "77", - "D13": "17.09", - "B14": "Waiting for Customer", - "C14": "10", - "C15": "225", - "D15": "22.54", - "A16": "Agent-103", - "B16": "Closed", - "C16": "73", - "D16": "23.11", - "B17": "In Progress", - "C17": "39", - "B18": "Open", - "C18": "14", - "B19": "Resolved", - "C19": "92", - "D19": "17.08", - "B20": "Waiting for Customer", - "C20": "11", - "C21": "229", - "D21": "19.75", - "A22": "Agent-104", - "B22": "Closed", - "C22": "63", - "D22": "15.48", - "B23": "In Progress", - "C23": "32", - "B24": "Open", - "C24": "18", - "B25": "Resolved", - "C25": "97", - "D25": "15.40", - "B26": "Waiting for Customer", - "C26": "13", - "C27": "223", - "D27": "15.43", - "A28": "Agent-105", - "B28": "Closed", - "C28": "70", - "D28": "22.20", - "B29": "In Progress", - "C29": "27", - "B30": "Open", - "C30": "16", - "B31": "Resolved", - "C31": "69", - "D31": "15.21", - "B32": "Waiting for Customer", - "C32": "10", - "C33": "192", - "D33": "18.73", - "A34": "Agent-106", - "B34": "Closed", - "C34": "65", - "D34": "26.15", - "B35": "In Progress", - "C35": "38", - "B36": "Open", - "C36": "20", - "B37": "Resolved", - "C37": "87", - "D37": "21.56", - "B38": "Waiting for Customer", - "C38": "10", - "C39": "220", - "D39": "23.52", - "A40": "Agent-107", - "B40": "Closed", - "C40": "68", - "D40": "18.41", - "B41": "In Progress", - "C41": "35", - "B42": "Open", - "C42": "17", - "B43": "Resolved", - "C43": "106", - "D43": "16.52", - "B44": "Waiting for Customer", - "C44": "11", - "C45": "237", - "D45": "17.26", - "A46": "Agent-108", - "B46": "Closed", - "C46": "65", - "D46": "16.81", - "B47": "In Progress", - "C47": "33", - "B48": "Open", - "C48": "20", - "B49": "Resolved", - "C49": "93", - "D49": "14.93", - "B50": "Waiting for Customer", - "C50": "8", - "C51": "219", - "D51": "15.70", - "B52": "(blank)", - "C52": "243", - "D52": "20.98", - "B53": "Closed", - "C53": "74", - "D53": "24.44", - "B54": "In Progress", - "C54": "32", - "B55": "Open", - "C55": "26", - "B56": "Resolved", - "C56": "99", - "D56": "18.39", - "B57": "Waiting for Customer", - "C57": "12", - "B58": "Grand Total", - "C58": "2000", - "D58": "19.05" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "7598db5d-0fd1-44db-ab1a-593cf026317b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B2": "1", - "B3": "2", - "B4": "3", - "B5": "4", - "B6": "5", - "B8": "1", - "B9": "2", - "C1": "Count of LastPromotionDate", - "C2": "12", - "C3": "13", - "C4": "10", - "C5": "15", - "C6": "5", - "C8": "1", - "C9": "5", - "D1": "Average of SalaryUSD", - "D2": "84965.91", - "D3": "87028.38", - "D4": "86094.90", - "D5": "78111.64", - "D6": "86015.00", - "D8": "82727.50", - "D9": "78876.00", - "A38": "Grand Total", - "B10": "3", - "B11": "4", - "B12": "5", - "B14": "1", - "B15": "2", - "B16": "3", - "B17": "4", - "B18": "5", - "B20": "1", - "B21": "2", - "B22": "3", - "B23": "4", - "B24": "5", - "B26": "1", - "B27": "2", - "B28": "3", - "B29": "4", - "B30": "5", - "B32": "1", - "B33": "2", - "B34": "3", - "B35": "4", - "B36": "5", - "C10": "4", - "C15": "3", - "C16": "2", - "C17": "2", - "C20": "1", - "C21": "4", - "C22": "6", - "C23": "3", - "C24": "5", - "C26": "10", - "C27": "9", - "C28": "4", - "C29": "6", - "C30": "4", - "C32": "5", - "C33": "2", - "C34": "4", - "C35": "4", - "C36": "1", - "C38": "140", - "D10": "76759.43", - "D11": "78318.00", - "D12": "80036.25", - "D14": "76599.00", - "D15": "93749.17", - "D16": "79929.00", - "D17": "88994.75", - "D18": "102902.00", - "D20": "77878.50", - "D21": "77480.38", - "D22": "84212.67", - "D23": "77508.83", - "D24": "77182.50", - "D26": "56034.08", - "D27": "52096.20", - "D28": "63190.71", - "D29": "60750.00", - "D30": "50299.25", - "D32": "71131.57", - "D33": "89669.50", - "D34": "81155.50", - "D35": "82037.80", - "D36": "61600.50", - "D38": "77770.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": " 27,301,058.62 " - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Year", - "A2": "2025", - "A3": "2026", - "A4": "2027", - "A5": "2028", - "A6": "2029", - "A9": "Year", - "B1": "Revenue", - "B2": "108,000,000.00", - "B3": "116,640,000.00", - "B4": "125,971,200.00", - "B5": "136,048,896.00", - "B6": "146,932,807.68", - "B9": "Revenue", - "C1": "COGS", - "C2": "43,200,000.00", - "C3": "46,656,000.00", - "C4": "50,388,480.00", - "C5": "54,419,558.40", - "C6": "58,773,123.07", - "C9": "COGS", - "D1": "Operating Expenses", - "D2": "21,600,000.00", - "D3": "23,328,000.00", - "D4": "25,194,240.00", - "D5": "27,209,779.20", - "D6": "29,386,561.54", - "D9": "Operating Expenses", - "E1": "EBITDA", - "E2": "43,200,000.00", - "E3": "46,656,000.00", - "E4": "50,388,480.00", - "E5": "54,419,558.40", - "E6": "58,773,123.07", - "E9": "EBITDA", - "F1": "Net Income", - "F2": "34,560,000.00", - "F3": "37,324,800.00", - "F4": "40,310,784.00", - "F5": "43,535,646.72", - "F6": "47,018,498.46", - "F9": "Net Income", - "A10": "2025", - "A11": "2026", - "A12": "2027", - "A13": "2028", - "A14": "2029", - "A17": "Year", - "A18": "2025", - "A19": "2026", - "A20": "2027", - "A21": "2028", - "A22": "2029", - "B10": "112,000,000.00", - "B11": "125,440,000.00", - "B12": "140,492,800.00", - "B13": "157,351,936.00", - "B14": "176,234,168.32", - "B17": "Revenue", - "B18": "103,000,000.00", - "B19": "106,090,000.00", - "B20": "109,272,700.00", - "B21": "112,550,881.00", - "B22": "115,927,407.43", - "C10": "39,200,000.00", - "C11": "43,904,000.00", - "C12": "49,172,480.00", - "C13": "55,073,177.60", - "C14": "61,681,958.91", - "C17": "COGS", - "C18": "46,350,000.00", - "C19": "47,740,500.00", - "C20": "49,172,715.00", - "C21": "50,647,896.45", - "C22": "52,167,333.34", - "D10": "33,600,000.00", - "D11": "37,632,000.00", - "D12": "42,147,840.00", - "D13": "47,205,580.80", - "D14": "52,870,250.50", - "D17": "Operating Expenses", - "D18": "20,600,000.00", - "D19": "21,218,000.00", - "D20": "21,854,540.00", - "D21": "22,510,176.20", - "D22": "23,185,481.49", - "E10": "39,200,000.00", - "E11": "43,904,000.00", - "E12": "49,172,480.00", - "E13": "55,073,177.60", - "E14": "61,681,958.91", - "E17": "EBITDA", - "E18": "36,050,000.00", - "E19": "37,131,500.00", - "E20": "38,245,445.00", - "E21": "39,392,808.35", - "E22": "40,574,592.60", - "F10": "31,360,000.00", - "F11": "35,123,200.00", - "F12": "39,337,984.00", - "F13": "44,058,542.08", - "F14": "49,345,567.13", - "F17": "Net Income", - "F18": "28,840,000.00", - "F19": "29,705,200.00", - "F20": "30,596,356.00", - "F21": "31,514,246.68", - "F22": "32,459,674.08" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "cea3b19a-6855-45f0-863e-42694346d487", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "2024-04-01", - "A5": "2024-04-02", - "A6": "2024-04-03", - "A7": "2024-04-04", - "A8": "2024-04-05", - "A9": "2024-04-08", - "B4": "1.020", - "B5": "0.953", - "B6": "0.989", - "B7": "0.046", - "B8": "0.046", - "B9": "0.046", - "A10": "2024-04-09", - "A11": "2024-04-10", - "A12": "2024-04-11", - "A13": "2024-04-12", - "A14": "Grand Total", - "B10": "0.046", - "B11": "0.046", - "B12": "0.046", - "B13": "0.046", - "B14": "0.328" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1,758,109,357.43" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A2": "Kressa", - "A3": "Saumya", - "A4": "Jeane", - "A5": "Sarah", - "A6": "Anaiya", - "A7": "Varun", - "A8": "Sofie", - "B2": "$27.00", - "B3": "$30.50", - "B4": "$27.00", - "B5": "$18.00", - "B6": "$50.00", - "B7": "$52.00", - "B8": "$19.00", - "C2": "40", - "C3": "50", - "C4": "20", - "C5": "20", - "C6": "80", - "C7": "80", - "C8": "20", - "D2": "$1,080.00", - "D3": "$1,525.00", - "D4": "$540.00", - "D5": "$360.00", - "D6": "$4,000.00", - "D7": "$4,160.00", - "D8": "$380.00", - "E2": "$66.96", - "E3": "$94.55", - "E4": "$33.48", - "E5": "$22.32", - "E6": "$248.00", - "E7": "$257.92", - "E8": "$23.56", - "F2": "$15.66", - "F3": "$22.11", - "F4": "$7.83", - "F5": "$5.22", - "F6": "$58.00", - "F7": "$60.32", - "F8": "$5.51", - "G2": "$2.24", - "G3": "$2.80", - "G4": "$1.12", - "G5": "$1.12", - "G6": "$4.48", - "G7": "$4.48", - "G8": "$1.12", - "H2": "$0.32", - "H3": "$0.46", - "H4": "$0.16", - "H5": "$0.11", - "H6": "$1.20", - "H7": "$1.25", - "H8": "$0.11", - "I2": "$5.71", - "I3": "$8.06", - "I4": "$2.85", - "I5": "$1.90", - "I6": "$21.14", - "I7": "$21.99", - "I8": "$2.01", - "J2": "$6.26", - "J3": "$8.85", - "J4": "$3.13", - "J5": "$2.09", - "J6": "$23.20", - "J7": "$24.13", - "J8": "$2.20", - "K2": "$97.16", - "K3": "$136.82", - "K4": "$48.58", - "K5": "$32.76", - "K6": "$356.02", - "K7": "$370.08", - "K8": "$34.52" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B3": "Maturity", - "A4": "CurveDate", - "B4": "10Y", - "C4": "1Y", - "D4": "2Y", - "E4": "30Y", - "F4": "5Y", - "G4": "Grand Total", - "A5": "2024-01-01", - "B5": "$3.52", - "C5": "$3.73", - "D5": "$3.71", - "E5": "$3.86", - "F5": "$4.71", - "G5": "$19.53", - "A6": "2024-01-02", - "B6": "$3.73", - "C6": "$3.62", - "D6": "$3.79", - "E6": "$4.81", - "F6": "$4.54", - "G6": "$20.49", - "A7": "2024-01-03", - "B7": "$4.28", - "C7": "$4.98", - "D7": "$4.63", - "E7": "$3.70", - "F7": "$4.79", - "G7": "$22.38", - "A8": "2024-01-04", - "B8": "$4.03", - "C8": "$4.84", - "D8": "$3.94", - "E8": "$4.84", - "F8": "$4.49", - "G8": "$22.14", - "A9": "Grand Total", - "B9": "$15.56", - "C9": "$17.17", - "D9": "$16.07", - "E9": "$17.21", - "F9": "$18.53", - "G9": "$84.54" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "ABC", - "B1": "-0.08" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Carla White", - "A2": "$155.00", - "B1": "$8,758.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1e4ede56-163f-4815-b634-7946cf9e60c0", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Content Marketing", - "A2": "Email", - "A3": "PPC", - "A4": "SEO", - "A5": "Social Media", - "B1": "Harness Frictionless Users", - "B2": "Transform Out-Of-The-Box Schemas", - "B3": "Morph Back-End E-Business", - "B4": "Facilitate Dynamic Channels", - "B5": "Re-Intermediate Cutting-Edge Web-Readiness", - "C1": "6.61", - "C2": "5.99", - "C3": "5.02", - "C4": "5.49", - "C5": "5.68" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "8499e399-48e6-4603-bd96-9b45f5aabed0", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Omar Donovan", - "A2": "Omar Donovan" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1876764.43" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "991105", - "A2": "1044283", - "A3": "1095308", - "A4": "1169204", - "B1": "4299900" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "764ca583-7091-4b7f-8663-5e996db26a2c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "16.74%", - "B1": "27.80%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9f71aa71-07f7-4421-8757-bbccbeab0984", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/gold_solution_3.xlsx" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Star City", - "B1": "39.11%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Gotham", - "B1": "4" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Finance", - "B1": "1,000" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "21680.086" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d2c56ee0-7863-45be-b503-eb1639454629", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Scenario 3", - "A2": "Scenario 2", - "A3": "Scenario 1" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "077926f8-6e93-406b-9eb9-b44907173c74", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Vendor", - "A2": "vendor 18", - "A3": "vendor 15", - "A4": "vendor 19", - "A5": "vendor 1", - "A6": "vendor 14", - "A7": "vendor 9", - "A8": "vendor 8", - "A9": "vendor 11", - "B1": "1099 Report amount", - "B2": "$4.00", - "B3": "$48.00", - "B4": "$81.00", - "B5": "$92.60", - "B6": "$177.00", - "B7": "$866.00", - "B8": "$1,661.00", - "B9": "$2,615.00", - "A10": "vendor 4", - "A11": "vendor 5", - "B10": "$6,446.00", - "B11": "$6,519.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "f1afee7f-df70-4a11-a65e-767a718f2117", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2025-02" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "36e43b84-e583-4f02-a160-74049bcab901", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "Platform", - "A5": "Facebook", - "A9": "Instagram", - "B4": "Values", - "B5": "Sum of Posts", - "B6": "Sum of Likes", - "B7": "Sum of Shares", - "B8": "Sum of Comments", - "B9": "Sum of Posts", - "C4": "August", - "C5": "66", - "C6": "3899", - "C7": "618", - "C8": "241", - "C9": "54", - "D4": "July", - "D5": "65", - "D6": "4008", - "D7": "749", - "D8": "260", - "D9": "58", - "E4": "June", - "E5": "54", - "E6": "3236", - "E7": "560", - "E8": "296", - "E9": "70", - "F4": "November", - "F5": "39", - "F6": "2066", - "F7": "430", - "F8": "209", - "F9": "52", - "G4": "October", - "G5": "68", - "G6": "3048", - "G7": "773", - "G8": "271", - "G9": "54", - "H4": "September", - "H5": "64", - "H6": "2918", - "H7": "769", - "H8": "262", - "H9": "57", - "I4": "Grand Total", - "I5": "356", - "I6": "19175", - "I7": "3899", - "I8": "1539", - "I9": "345", - "A13": "LinkedIn", - "A17": "Twitter", - "A21": "Grand Total", - "B10": "Sum of Likes", - "B11": "Sum of Shares", - "B12": "Sum of Comments", - "B13": "Sum of Posts", - "B14": "Sum of Likes", - "B15": "Sum of Shares", - "B16": "Sum of Comments", - "B17": "Sum of Posts", - "B18": "Sum of Likes", - "B19": "Sum of Shares", - "B20": "Sum of Comments", - "B21": "Sum of Posts", - "B22": "Sum of Likes", - "B23": "Sum of Shares", - "B24": "Sum of Comments", - "C10": "3231", - "C11": "541", - "C12": "226", - "C13": "61", - "C14": "2923", - "C15": "668", - "C16": "295", - "C17": "60", - "C18": "3290", - "C19": "671", - "C20": "295", - "C21": "241", - "C22": "13343", - "C23": "2498", - "C24": "1057", - "D10": "3025", - "D11": "700", - "D12": "232", - "D13": "60", - "D14": "3122", - "D15": "589", - "D16": "314", - "D17": "54", - "D18": "2950", - "D19": "454", - "D20": "271", - "D21": "237", - "D22": "13105", - "D23": "2492", - "D24": "1077", - "E10": "4195", - "E11": "834", - "E12": "317", - "E13": "45", - "E14": "2291", - "E15": "451", - "E16": "185", - "E17": "63", - "E18": "3366", - "E19": "797", - "E20": "278", - "E21": "232", - "E22": "13088", - "E23": "2642", - "E24": "1076", - "F10": "2860", - "F11": "514", - "F12": "249", - "F13": "61", - "F14": "3116", - "F15": "708", - "F16": "246", - "F17": "63", - "F18": "2714", - "F19": "581", - "F20": "243", - "F21": "215", - "F22": "10756", - "F23": "2233", - "F24": "947", - "G10": "2993", - "G11": "623", - "G12": "265", - "G13": "70", - "G14": "3596", - "G15": "716", - "G16": "336", - "G17": "58", - "G18": "2382", - "G19": "578", - "G20": "293", - "G21": "250", - "G22": "12019", - "G23": "2690", - "G24": "1165", - "H10": "3409", - "H11": "401", - "H12": "313", - "H13": "65", - "H14": "3271", - "H15": "771", - "H16": "278", - "H17": "51", - "H18": "2156", - "H19": "431", - "H20": "312", - "H21": "237", - "H22": "11754", - "H23": "2372", - "H24": "1165", - "I10": "19713", - "I11": "3613", - "I12": "1602", - "I13": "362", - "I14": "18319", - "I15": "3903", - "I16": "1654", - "I17": "349", - "I18": "16858", - "I19": "3512", - "I20": "1692", - "I21": "1412", - "I22": "74065", - "I23": "14927", - "I24": "6487" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Platform", - "A2": "Twitter", - "A3": "Facebook", - "A4": "Instagram", - "A5": "LinkedIn", - "B1": "RATIO OF HIGH / LOW", - "B2": "1.88", - "B3": "2.64", - "B4": "3.17", - "B5": "2.05" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "c437f78d-6382-43b2-b857-153db6efa1c9", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "Jun", - "A5": "Jul", - "A6": "Aug", - "A7": "Sep", - "A8": "Oct", - "A9": "Nov", - "B3": "Sum of Posts", - "B4": "124", - "B5": "123", - "B6": "120", - "B7": "121", - "B8": "122", - "B9": "91", - "C3": "Sum of Likes", - "C4": "7431", - "C5": "7033", - "C6": "7130", - "C7": "6327", - "C8": "6041", - "C9": "4926", - "D3": "Sum of Comments", - "D4": "613", - "D5": "492", - "D6": "467", - "D7": "575", - "D8": "536", - "D9": "458", - "E3": "Sum of Shares", - "E4": "1394", - "E5": "1449", - "E6": "1159", - "E7": "1170", - "E8": "1396", - "E9": "944", - "A10": "Grand Total", - "B10": "701", - "C10": "38888", - "D10": "3141", - "E10": "7512" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.68%", - "E3": "43.14%", - "E4": "43.77%", - "E5": "43.27%", - "E6": "44.90%", - "E7": "44.03%", - "E8": "44.06%", - "E9": "43.97%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "E10": "44.37%", - "E11": "44.10%", - "E12": "46.70%", - "E13": "48.06%", - "E14": "45.94%", - "E15": "45.48%", - "E16": "44.06%", - "E17": "43.93%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9ba9631e-8560-4dc4-a285-8d186715a542", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.7%", - "E3": "43.1%", - "E4": "43.8%", - "E5": "43.3%", - "E6": "44.9%", - "E7": "44.0%", - "E8": "44.1%", - "E9": "44.0%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "A18": "5", - "A19": "6", - "A20": "7", - "A21": "8", - "A22": "9", - "A23": "10", - "A24": "11", - "A25": "12", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "B18": "2024", - "B19": "2024", - "B20": "2024", - "B21": "2024", - "B22": "2024", - "B23": "2024", - "B24": "2024", - "B25": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "C18": "21095", - "C19": "21015", - "C20": "21141", - "C21": "21311", - "C22": "21064", - "C23": "21182", - "C24": "21209", - "C25": "21238", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "D18": "34237", - "D19": "33819", - "D20": "33547", - "D21": "33832", - "D22": "33424", - "D23": "33162", - "D24": "33042", - "D25": "32929", - "E10": "44.4%", - "E11": "44.1%", - "E12": "46.7%", - "E13": "48.1%", - "E14": "45.9%", - "E15": "45.5%", - "E16": "44.1%", - "E17": "43.9%", - "E18": "43.1%", - "E19": "41.8%", - "E20": "41.3%", - "E21": "40.4%", - "E22": "39.7%", - "E23": "38.7%", - "E24": "37.9%", - "E25": "37.2%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2023-04-30", - "A2": "0.48", - "A3": "0.00296" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "D2": "Rank Based on 2024 Sales", - "D3": "4", - "D4": "3", - "D5": "7", - "D6": "17", - "D7": "12", - "D8": "1", - "D9": "18", - "D10": "8", - "D11": "6", - "D12": "14", - "D13": "5", - "D14": "13", - "D15": "9", - "D16": "20", - "D17": "11", - "D18": "10", - "D19": "19", - "D20": "15", - "D21": "2", - "D22": "16" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "20507108-4188-402e-8cfe-2354309bad6c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A2": "Region", - "A3": "Central", - "A4": "East", - "A5": "North", - "A6": "South", - "A7": "West", - "B2": "2023", - "B3": "351351.87", - "B4": "364129.72", - "B5": "377252.87", - "B6": "396259.93", - "B7": "395672.78", - "C2": "2024", - "C3": "368644.54", - "C4": "399862.83", - "C5": "364113.70", - "C6": "345896.80", - "C7": "393456.82", - "D2": "YoY Growth", - "D3": "4.92%", - "D4": "9.81%", - "D5": "-3.48%", - "D6": "-12.71%", - "D7": "-0.56%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Shopping", - "B1": "$ (16,401.91)" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "6.17%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "24.20%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "371507f7-721a-457e-9618-bc8fba1b909b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$316,309.56" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "4a081ed2-532c-4895-a034-ab0076927c7c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2026-03", - "A2": "2025-09", - "A3": "2026-12" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "JPY", - "A2": "USD", - "A3": "EUR", - "A4": "GBP", - "B1": "3500000.00", - "B2": "64351.25", - "B3": "43501.00", - "B4": "15000.25" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "This spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "31508ec6-c993-4e00-b70c-093dba016fcc", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1.07%", - "A2": "1.73%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Which city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "133ef005-207c-490f-b55c-7734fd1678da", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Central City" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Work in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header “Date” plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "eb410896-3e1e-4491-9460-061579b65c6f", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Date", - "A2": "2025-01-05", - "A3": "2025-01-12", - "A4": "2025-01-15", - "A5": "2025-01-20", - "A6": "2025-01-25", - "A7": "2025-01-30", - "A8": "2025-02-05", - "A9": "2025-02-08", - "B1": "Description", - "B2": "Membership Fee", - "B3": "Project Income", - "B4": "Interest", - "B5": "Event Revenue", - "B6": "Refund", - "B7": "Consulting Fee", - "B8": "Website Hosting", - "B9": "Maintenance", - "C1": "Amount", - "C2": "-250.00", - "C3": "5600.00", - "C4": "350.00", - "C5": "7000.00", - "C6": "-5000.00", - "C7": "7800.00", - "C8": "-99.99", - "C9": "-750.25", - "D1": "Currency", - "D2": "USD", - "D3": "USD", - "D4": "USD", - "D5": "USD", - "D6": "USD", - "D7": "USD", - "D8": "USD", - "D9": "USD", - "A10": "2025-02-15", - "A11": "2025-02-18", - "A12": "2025-02-20", - "A13": "2025-02-25", - "A14": "2025-02-28", - "A15": "2025-03-01", - "A16": "2025-03-05", - "A17": "2025-03-10", - "A18": "2025-03-15", - "A19": "2025-03-20", - "A20": "2025-03-25", - "B10": "Invoice Payment", - "B11": "Equipment Purchase", - "B12": "Software License", - "B13": "Bonus", - "B14": "Travel Expenses", - "B15": "Subscription", - "B16": "Advertising", - "B17": "Office Supplies", - "B18": "Utilities", - "B19": "Legal Fees", - "B20": "Marketing", - "C10": "2500.00", - "C11": "-3600.00", - "C12": "-3000.00", - "C13": "4800.00", - "C14": "-1750.00", - "C15": "-1200.00", - "C16": "-2450.50", - "C17": "-450.75", - "C18": "-899.00", - "C19": "-4000.00", - "C20": "-2200.00", - "D10": "USD", - "D11": "USD", - "D12": "USD", - "D13": "USD", - "D14": "USD", - "D15": "USD", - "D16": "USD", - "D17": "USD", - "D18": "USD", - "D19": "USD", - "D20": "USD" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - } -] \ No newline at end of file diff --git a/SheetBench-50.json b/SheetBench-50.json deleted file mode 100644 index 67f759656..000000000 --- a/SheetBench-50.json +++ /dev/null @@ -1,4211 +0,0 @@ -[ - { - "prompt": "Calculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", - "mcp_config": { - "hud": { - "url": "https://orcstaging.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.3" - } - } - }, - "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1/12/2024", - "A2": "1/10/2024", - "A3": "1/15/2024", - "A4": "1/11/2024", - "A5": "1/17/2024" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Calculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B2": "CustomerID", - "B3": "476232", - "B4": "963589", - "B5": "1430820", - "B6": "2410043", - "B7": "3789483", - "B8": "4226444", - "B9": "4308096", - "C2": "2023", - "C5": "686.40", - "C8": "966.76", - "C9": "863.37", - "D2": "2024", - "D3": "912.70", - "D4": "397.62", - "D6": "383.38", - "D7": "619.56", - "E2": "Grand Total", - "E3": "912.70", - "E4": "397.62", - "E5": "686.40", - "E6": "383.38", - "E7": "619.56", - "E8": "966.76", - "E9": "863.37", - "B10": "5907496", - "B11": "6224271", - "B12": "6538101", - "B13": "6584993", - "B14": "6791810", - "B15": "7250781", - "B16": "8518187", - "B17": "8668097", - "B18": "8885050", - "B19": "9296053", - "B20": "10414682", - "B21": "10839621", - "B22": "12313143", - "B23": "13672423", - "B24": "14744105", - "B25": "14780608", - "B26": "15144539", - "B27": "15616289", - "B28": "15694782", - "B29": "15904102", - "B30": "16385696", - "B31": "17254618", - "B32": "17441704", - "B33": "17624107", - "B34": "18043433", - "B35": "18296075", - "B36": "18600797", - "B37": "19026954", - "B38": "19395319", - "B39": "19650142", - "B40": "19842318", - "B41": "20241148", - "B42": "20587659", - "B43": "21647994", - "B44": "21671104", - "B45": "21935183", - "B46": "22115836", - "B47": "22846037", - "B48": "23621954", - "B49": "24962520", - "B50": "25073201", - "B51": "25090031", - "B52": "25096194", - "B53": "25211207", - "B54": "26168579", - "B55": "26250755", - "B56": "26419282", - "B57": "26953175", - "B58": "27281086", - "B59": "27646724", - "B60": "30370166", - "B61": "30738270", - "B62": "30876644", - "B63": "32255126", - "B64": "32373769", - "B65": "33121017", - "B66": "33897455", - "B67": "34507001", - "B68": "35475491", - "B69": "36972111", - "B70": "37040802", - "B71": "37543198", - "B72": "37609900", - "B73": "37674649", - "B74": "38019388", - "B75": "38658747", - "B76": "38793189", - "B77": "39134950", - "B78": "40399799", - "B79": "41039038", - "B80": "41271414", - "B81": "41870334", - "B82": "42495130", - "B83": "43189090", - "B84": "43235421", - "B85": "43837144", - "B86": "44115166", - "B87": "44270797", - "B88": "45380457", - "B89": "45386282", - "B90": "46196026", - "B91": "46300157", - "B92": "48512428", - "B93": "49546866", - "B94": "49687477", - "B95": "50893874", - "B96": "51093532", - "B97": "51126698", - "B98": "51397982", - "B99": "52376160", - "C10": "714.36", - "C11": "369.01", - "C12": "525.20", - "C20": "120.72", - "C21": "269.30", - "C22": "422.36", - "C26": "971.83", - "C28": "110.65", - "C29": "496.66", - "C30": "841.89", - "C31": "886.06", - "C32": "536.09", - "C34": "16.13", - "C38": "90.40", - "C39": "76.23", - "C40": "185.33", - "C41": "65.32", - "C42": "111.51", - "C43": "493.89", - "C44": "787.04", - "C45": "578.58", - "C46": "204.48", - "C48": "985.49", - "C50": "268.33", - "C52": "34.51", - "C54": "61.19", - "C60": "62.83", - "C63": "326.11", - "C64": "898.66", - "C65": "251.17", - "C66": "902.46", - "C67": "442.33", - "C70": "561.10", - "C71": "469.05", - "C74": "477.14", - "C76": "284.69", - "C80": "266.21", - "C82": "699.49", - "C85": "82.73", - "C87": "542.97", - "C88": "446.42", - "C89": "380.14", - "C91": "683.73", - "C92": "566.99", - "C94": "646.75", - "C97": "748.88", - "C98": "253.40", - "D13": "174.59", - "D14": "374.65", - "D15": "339.03", - "D16": "140.66", - "D17": "862.92", - "D18": "282.39", - "D19": "847.67", - "D23": "616.14", - "D24": "376.83", - "D25": "523.78", - "D27": "701.88", - "D29": "496.66", - "D33": "805.83", - "D35": "579.30", - "D36": "577.84", - "D37": "673.27", - "D38": "90.40", - "D47": "47.03", - "D48": "985.49", - "D49": "779.72", - "D51": "277.24", - "D53": "702.84", - "D55": "334.42", - "D56": "875.84", - "D57": "570.46", - "D58": "60.52", - "D59": "29.15", - "D61": "822.61", - "D62": "931.36", - "D68": "728.49", - "D69": "454.44", - "D72": "866.51", - "D73": "968.06", - "D74": "477.14", - "D75": "154.14", - "D77": "69.76", - "D78": "323.27", - "D79": "862.51", - "D81": "335.73", - "D83": "555.46", - "D84": "685.07", - "D86": "942.65", - "D90": "813.06", - "D93": "671.93", - "D95": "995.04", - "D96": "896.64", - "D98": "253.40", - "D99": "66.10", - "E10": "714.36", - "E11": "369.01", - "E12": "525.20", - "E13": "174.59", - "E14": "374.65", - "E15": "339.03", - "E16": "140.66", - "E17": "862.92", - "E18": "282.39", - "E19": "847.67", - "E20": "120.72", - "E21": "269.30", - "E22": "422.36", - "E23": "616.14", - "E24": "376.83", - "E25": "523.78", - "E26": "971.83", - "E27": "701.88", - "E28": "110.65", - "E29": "993.32", - "E30": "841.89", - "E31": "886.06", - "E32": "536.09", - "E33": "805.83", - "E34": "16.13", - "E35": "579.30", - "E36": "577.84", - "E37": "673.27", - "E38": "180.80", - "E39": "76.23", - "E40": "185.33", - "E41": "65.32", - "E42": "111.51", - "E43": "493.89", - "E44": "787.04", - "E45": "578.58", - "E46": "204.48", - "E47": "47.03", - "E48": "1970.98", - "E49": "779.72", - "E50": "268.33", - "E51": "277.24", - "E52": "34.51", - "E53": "702.84", - "E54": "61.19", - "E55": "334.42", - "E56": "875.84", - "E57": "570.46", - "E58": "60.52", - "E59": "29.15", - "E60": "62.83", - "E61": "822.61", - "E62": "931.36", - "E63": "326.11", - "E64": "898.66", - "E65": "251.17", - "E66": "902.46", - "E67": "442.33", - "E68": "728.49", - "E69": "454.44", - "E70": "561.10", - "E71": "469.05", - "E72": "866.51", - "E73": "968.06", - "E74": "954.28", - "E75": "154.14", - "E76": "284.69", - "E77": "69.76", - "E78": "323.27", - "E79": "862.51", - "E80": "266.21", - "E81": "335.73", - "E82": "699.49", - "E83": "555.46", - "E84": "685.07", - "E85": "82.73", - "E86": "942.65", - "E87": "542.97", - "E88": "446.42", - "E89": "380.14", - "E90": "813.06", - "E91": "683.73", - "E92": "566.99", - "E93": "671.93", - "E94": "646.75", - "E95": "995.04", - "E96": "896.64", - "E97": "748.88", - "E98": "506.80", - "E99": "66.10", - "B100": "52697630", - "B101": "52870804", - "B102": "53239020", - "B103": "53335630", - "B104": "53858146", - "B105": "54040755", - "B106": "54479395", - "B107": "54528192", - "B108": "54730083", - "B109": "55444897", - "B110": "55450065", - "B111": "55853989", - "B112": "55859479", - "B113": "55978545", - "B114": "56702798", - "B115": "56940756", - "B116": "58152560", - "B117": "59169574", - "B118": "59736806", - "B119": "61602489", - "B120": "62665919", - "B121": "62758192", - "B122": "62943277", - "B123": "63051186", - "B124": "65657133", - "B125": "65899458", - "B126": "66761945", - "B127": "66846840", - "B128": "67283606", - "B129": "67548873", - "B130": "69613838", - "B131": "69765117", - "B132": "70098167", - "B133": "70534368", - "B134": "71078229", - "B135": "71319902", - "B136": "71369648", - "B137": "71376141", - "B138": "72217443", - "B139": "72659809", - "B140": "72883709", - "B141": "73191421", - "B142": "73684220", - "B143": "74014279", - "B144": "74088126", - "B145": "75410476", - "B146": "75817527", - "B147": "77140593", - "B148": "77558800", - "B149": "78916470", - "B150": "79031936", - "B151": "79751742", - "B152": "80286530", - "B153": "80765899", - "B154": "82060237", - "B155": "82306595", - "B156": "83101113", - "B157": "83211478", - "B158": "83713620", - "B159": "84770820", - "B160": "84800206", - "B161": "84952943", - "B162": "86407021", - "B163": "86619158", - "B164": "86663007", - "B165": "87144451", - "B166": "87254792", - "B167": "88194202", - "B168": "88266169", - "B169": "88761732", - "B170": "88882642", - "B171": "89421277", - "B172": "89565544", - "B173": "90841330", - "B174": "91483447", - "B175": "91590435", - "B176": "91939241", - "B177": "92335820", - "B178": "92422728", - "B179": "92676749", - "B180": "93202190", - "B181": "93479509", - "B182": "95353791", - "B183": "95696393", - "B184": "95804583", - "B185": "95860510", - "B186": "96051566", - "B187": "96263709", - "B188": "96456101", - "B189": "99519803", - "B190": "Grand Total", - "C100": "409.46", - "C101": "28.08", - "C106": "184.81", - "C107": "906.16", - "C108": "1094.00", - "C109": "818.87", - "C110": "192.68", - "C111": "385.72", - "C117": "976.44", - "C118": "310.80", - "C119": "354.99", - "C120": "898.04", - "C122": "397.81", - "C128": "537.90", - "C129": "886.96", - "C131": "420.66", - "C133": "359.50", - "C134": "60.29", - "C139": "740.19", - "C142": "554.52", - "C144": "759.21", - "C145": "116.62", - "C148": "302.90", - "C152": "187.16", - "C153": "847.55", - "C154": "700.62", - "C155": "190.59", - "C161": "432.99", - "C163": "886.00", - "C167": "715.20", - "C169": "101.82", - "C171": "122.95", - "C174": "609.90", - "C175": "867.14", - "C177": "675.11", - "C178": "623.53", - "C182": "40.39", - "C184": "554.62", - "C187": "839.28", - "C189": "717.61", - "C190": "43541.41", - "D102": "995.08", - "D103": "761.74", - "D104": "773.13", - "D105": "364.27", - "D112": "841.73", - "D113": "560.15", - "D114": "824.63", - "D115": "783.50", - "D116": "788.60", - "D117": "976.44", - "D118": "310.80", - "D121": "92.40", - "D123": "171.84", - "D124": "336.61", - "D125": "206.92", - "D126": "345.03", - "D127": "505.00", - "D130": "539.76", - "D132": "189.69", - "D135": "713.31", - "D136": "993.98", - "D137": "541.38", - "D138": "790.14", - "D139": "740.19", - "D140": "481.88", - "D141": "83.31", - "D143": "883.45", - "D146": "394.60", - "D147": "281.52", - "D149": "64.48", - "D150": "645.82", - "D151": "771.06", - "D156": "405.74", - "D157": "741.29", - "D158": "196.84", - "D159": "152.71", - "D160": "356.93", - "D162": "138.69", - "D164": "929.08", - "D165": "887.13", - "D166": "554.21", - "D168": "820.22", - "D170": "939.97", - "D171": "122.95", - "D172": "335.69", - "D173": "917.55", - "D176": "514.59", - "D179": "288.94", - "D180": "214.12", - "D181": "16.74", - "D183": "27.28", - "D185": "458.58", - "D186": "648.09", - "D188": "985.01", - "D190": "56717.97", - "E100": "409.46", - "E101": "28.08", - "E102": "995.08", - "E103": "761.74", - "E104": "773.13", - "E105": "364.27", - "E106": "184.81", - "E107": "906.16", - "E108": "1094.00", - "E109": "818.87", - "E110": "192.68", - "E111": "385.72", - "E112": "841.73", - "E113": "560.15", - "E114": "824.63", - "E115": "783.50", - "E116": "788.60", - "E117": "1952.88", - "E118": "621.60", - "E119": "354.99", - "E120": "898.04", - "E121": "92.40", - "E122": "397.81", - "E123": "171.84", - "E124": "336.61", - "E125": "206.92", - "E126": "345.03", - "E127": "505.00", - "E128": "537.90", - "E129": "886.96", - "E130": "539.76", - "E131": "420.66", - "E132": "189.69", - "E133": "359.50", - "E134": "60.29", - "E135": "713.31", - "E136": "993.98", - "E137": "541.38", - "E138": "790.14", - "E139": "1480.38", - "E140": "481.88", - "E141": "83.31", - "E142": "554.52", - "E143": "883.45", - "E144": "759.21", - "E145": "116.62", - "E146": "394.60", - "E147": "281.52", - "E148": "302.90", - "E149": "64.48", - "E150": "645.82", - "E151": "771.06", - "E152": "187.16", - "E153": "847.55", - "E154": "700.62", - "E155": "190.59", - "E156": "405.74", - "E157": "741.29", - "E158": "196.84", - "E159": "152.71", - "E160": "356.93", - "E161": "432.99", - "E162": "138.69", - "E163": "886.00", - "E164": "929.08", - "E165": "887.13", - "E166": "554.21", - "E167": "715.20", - "E168": "820.22", - "E169": "101.82", - "E170": "939.97", - "E171": "245.90", - "E172": "335.69", - "E173": "917.55", - "E174": "609.90", - "E175": "867.14", - "E176": "514.59", - "E177": "675.11", - "E178": "623.53", - "E179": "288.94", - "E180": "214.12", - "E181": "16.74", - "E182": "40.39", - "E183": "27.28", - "E184": "554.62", - "E185": "458.58", - "E186": "648.09", - "E187": "839.28", - "E188": "985.01", - "E189": "717.61", - "E190": "100259.38" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Find the GET request which most commonly results in an error. Place the URL in ANSWER!A1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "/api/users" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "For a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2026-04" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "For the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0963d367-f0ac-4be0-8f46-337bf335e68f", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "ABC", - "B1": "4999972", - "C1": "145%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$112,281.49" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "49ccea4d-4aaf-4faf-9659-b389686568a7", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$24,642.86" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "48d43d57-ed1d-4df0-8b36-62380bba7865", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1664934.45" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "ff114e80-196d-4a7a-99ca-152bac4fba90", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "MAY", - "A2": "OCT", - "B1": "95", - "B2": "73" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Category", - "A2": "Meals & Entertainment", - "A3": "Travel", - "A4": "Office Supplies", - "B1": "Sum of Amount", - "B2": "0", - "B3": "345", - "B4": "45760" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A3": "Sum of Amount (USD Millions)", - "A5": "Net Income", - "A6": "Revenue", - "A7": "Total Assets", - "A8": "Grand Total", - "B4": "FORD2024Q1", - "B5": "1.33", - "B6": "42.78", - "B7": "274.34", - "B8": "318.45", - "C4": "FORD2024Q2", - "C5": "1.83", - "C6": "44.81", - "C7": "276.59", - "C8": "323.23", - "D4": "FORD2024Q3", - "D5": "896.00", - "D6": "43.07", - "D7": "287.05", - "D8": "1226.12", - "E4": "FORD2025Q1", - "E5": "471.00", - "E6": "40.66", - "E7": "284.54", - "E8": "796.20", - "F4": "Grand Total", - "F5": "1370.17", - "F6": "171.32", - "F7": "1122.51", - "F8": "2664.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "C3": "Count of TicketID", - "D3": "Average of ResolutionTimeHours", - "A4": "Agent-101", - "B4": "Closed", - "C4": "73", - "D4": "13.90", - "B5": "In Progress", - "C5": "30", - "B6": "Open", - "C6": "31", - "B7": "Resolved", - "C7": "66", - "D7": "22.04", - "B8": "Waiting for Customer", - "C8": "12", - "C9": "212", - "D9": "17.76", - "A10": "Agent-102", - "B10": "Closed", - "C10": "70", - "D10": "28.53", - "B11": "In Progress", - "C11": "36", - "B12": "Open", - "C12": "32", - "B13": "Resolved", - "C13": "77", - "D13": "17.09", - "B14": "Waiting for Customer", - "C14": "10", - "C15": "225", - "D15": "22.54", - "A16": "Agent-103", - "B16": "Closed", - "C16": "73", - "D16": "23.11", - "B17": "In Progress", - "C17": "39", - "B18": "Open", - "C18": "14", - "B19": "Resolved", - "C19": "92", - "D19": "17.08", - "B20": "Waiting for Customer", - "C20": "11", - "C21": "229", - "D21": "19.75", - "A22": "Agent-104", - "B22": "Closed", - "C22": "63", - "D22": "15.48", - "B23": "In Progress", - "C23": "32", - "B24": "Open", - "C24": "18", - "B25": "Resolved", - "C25": "97", - "D25": "15.40", - "B26": "Waiting for Customer", - "C26": "13", - "C27": "223", - "D27": "15.43", - "A28": "Agent-105", - "B28": "Closed", - "C28": "70", - "D28": "22.20", - "B29": "In Progress", - "C29": "27", - "B30": "Open", - "C30": "16", - "B31": "Resolved", - "C31": "69", - "D31": "15.21", - "B32": "Waiting for Customer", - "C32": "10", - "C33": "192", - "D33": "18.73", - "A34": "Agent-106", - "B34": "Closed", - "C34": "65", - "D34": "26.15", - "B35": "In Progress", - "C35": "38", - "B36": "Open", - "C36": "20", - "B37": "Resolved", - "C37": "87", - "D37": "21.56", - "B38": "Waiting for Customer", - "C38": "10", - "C39": "220", - "D39": "23.52", - "A40": "Agent-107", - "B40": "Closed", - "C40": "68", - "D40": "18.41", - "B41": "In Progress", - "C41": "35", - "B42": "Open", - "C42": "17", - "B43": "Resolved", - "C43": "106", - "D43": "16.52", - "B44": "Waiting for Customer", - "C44": "11", - "C45": "237", - "D45": "17.26", - "A46": "Agent-108", - "B46": "Closed", - "C46": "65", - "D46": "16.81", - "B47": "In Progress", - "C47": "33", - "B48": "Open", - "C48": "20", - "B49": "Resolved", - "C49": "93", - "D49": "14.93", - "B50": "Waiting for Customer", - "C50": "8", - "C51": "219", - "D51": "15.70", - "B52": "(blank)", - "C52": "243", - "D52": "20.98", - "B53": "Closed", - "C53": "74", - "D53": "24.44", - "B54": "In Progress", - "C54": "32", - "B55": "Open", - "C55": "26", - "B56": "Resolved", - "C56": "99", - "D56": "18.39", - "B57": "Waiting for Customer", - "C57": "12", - "B58": "Grand Total", - "C58": "2000", - "D58": "19.05" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "7598db5d-0fd1-44db-ab1a-593cf026317b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B2": "1", - "B3": "2", - "B4": "3", - "B5": "4", - "B6": "5", - "B8": "1", - "B9": "2", - "C1": "Count of LastPromotionDate", - "C2": "12", - "C3": "13", - "C4": "10", - "C5": "15", - "C6": "5", - "C8": "1", - "C9": "5", - "D1": "Average of SalaryUSD", - "D2": "84965.91", - "D3": "87028.38", - "D4": "86094.90", - "D5": "78111.64", - "D6": "86015.00", - "D8": "82727.50", - "D9": "78876.00", - "A38": "Grand Total", - "B10": "3", - "B11": "4", - "B12": "5", - "B14": "1", - "B15": "2", - "B16": "3", - "B17": "4", - "B18": "5", - "B20": "1", - "B21": "2", - "B22": "3", - "B23": "4", - "B24": "5", - "B26": "1", - "B27": "2", - "B28": "3", - "B29": "4", - "B30": "5", - "B32": "1", - "B33": "2", - "B34": "3", - "B35": "4", - "B36": "5", - "C10": "4", - "C15": "3", - "C16": "2", - "C17": "2", - "C20": "1", - "C21": "4", - "C22": "6", - "C23": "3", - "C24": "5", - "C26": "10", - "C27": "9", - "C28": "4", - "C29": "6", - "C30": "4", - "C32": "5", - "C33": "2", - "C34": "4", - "C35": "4", - "C36": "1", - "C38": "140", - "D10": "76759.43", - "D11": "78318.00", - "D12": "80036.25", - "D14": "76599.00", - "D15": "93749.17", - "D16": "79929.00", - "D17": "88994.75", - "D18": "102902.00", - "D20": "77878.50", - "D21": "77480.38", - "D22": "84212.67", - "D23": "77508.83", - "D24": "77182.50", - "D26": "56034.08", - "D27": "52096.20", - "D28": "63190.71", - "D29": "60750.00", - "D30": "50299.25", - "D32": "71131.57", - "D33": "89669.50", - "D34": "81155.50", - "D35": "82037.80", - "D36": "61600.50", - "D38": "77770.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": " 27,301,058.62 " - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Year", - "A2": "2025", - "A3": "2026", - "A4": "2027", - "A5": "2028", - "A6": "2029", - "A9": "Year", - "B1": "Revenue", - "B2": "108,000,000.00", - "B3": "116,640,000.00", - "B4": "125,971,200.00", - "B5": "136,048,896.00", - "B6": "146,932,807.68", - "B9": "Revenue", - "C1": "COGS", - "C2": "43,200,000.00", - "C3": "46,656,000.00", - "C4": "50,388,480.00", - "C5": "54,419,558.40", - "C6": "58,773,123.07", - "C9": "COGS", - "D1": "Operating Expenses", - "D2": "21,600,000.00", - "D3": "23,328,000.00", - "D4": "25,194,240.00", - "D5": "27,209,779.20", - "D6": "29,386,561.54", - "D9": "Operating Expenses", - "E1": "EBITDA", - "E2": "43,200,000.00", - "E3": "46,656,000.00", - "E4": "50,388,480.00", - "E5": "54,419,558.40", - "E6": "58,773,123.07", - "E9": "EBITDA", - "F1": "Net Income", - "F2": "34,560,000.00", - "F3": "37,324,800.00", - "F4": "40,310,784.00", - "F5": "43,535,646.72", - "F6": "47,018,498.46", - "F9": "Net Income", - "A10": "2025", - "A11": "2026", - "A12": "2027", - "A13": "2028", - "A14": "2029", - "A17": "Year", - "A18": "2025", - "A19": "2026", - "A20": "2027", - "A21": "2028", - "A22": "2029", - "B10": "112,000,000.00", - "B11": "125,440,000.00", - "B12": "140,492,800.00", - "B13": "157,351,936.00", - "B14": "176,234,168.32", - "B17": "Revenue", - "B18": "103,000,000.00", - "B19": "106,090,000.00", - "B20": "109,272,700.00", - "B21": "112,550,881.00", - "B22": "115,927,407.43", - "C10": "39,200,000.00", - "C11": "43,904,000.00", - "C12": "49,172,480.00", - "C13": "55,073,177.60", - "C14": "61,681,958.91", - "C17": "COGS", - "C18": "46,350,000.00", - "C19": "47,740,500.00", - "C20": "49,172,715.00", - "C21": "50,647,896.45", - "C22": "52,167,333.34", - "D10": "33,600,000.00", - "D11": "37,632,000.00", - "D12": "42,147,840.00", - "D13": "47,205,580.80", - "D14": "52,870,250.50", - "D17": "Operating Expenses", - "D18": "20,600,000.00", - "D19": "21,218,000.00", - "D20": "21,854,540.00", - "D21": "22,510,176.20", - "D22": "23,185,481.49", - "E10": "39,200,000.00", - "E11": "43,904,000.00", - "E12": "49,172,480.00", - "E13": "55,073,177.60", - "E14": "61,681,958.91", - "E17": "EBITDA", - "E18": "36,050,000.00", - "E19": "37,131,500.00", - "E20": "38,245,445.00", - "E21": "39,392,808.35", - "E22": "40,574,592.60", - "F10": "31,360,000.00", - "F11": "35,123,200.00", - "F12": "39,337,984.00", - "F13": "44,058,542.08", - "F14": "49,345,567.13", - "F17": "Net Income", - "F18": "28,840,000.00", - "F19": "29,705,200.00", - "F20": "30,596,356.00", - "F21": "31,514,246.68", - "F22": "32,459,674.08" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "cea3b19a-6855-45f0-863e-42694346d487", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "2024-04-01", - "A5": "2024-04-02", - "A6": "2024-04-03", - "A7": "2024-04-04", - "A8": "2024-04-05", - "A9": "2024-04-08", - "B4": "1.020", - "B5": "0.953", - "B6": "0.989", - "B7": "0.046", - "B8": "0.046", - "B9": "0.046", - "A10": "2024-04-09", - "A11": "2024-04-10", - "A12": "2024-04-11", - "A13": "2024-04-12", - "A14": "Grand Total", - "B10": "0.046", - "B11": "0.046", - "B12": "0.046", - "B13": "0.046", - "B14": "0.328" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1,758,109,357.43" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A2": "Kressa", - "A3": "Saumya", - "A4": "Jeane", - "A5": "Sarah", - "A6": "Anaiya", - "A7": "Varun", - "A8": "Sofie", - "B2": "$27.00", - "B3": "$30.50", - "B4": "$27.00", - "B5": "$18.00", - "B6": "$50.00", - "B7": "$52.00", - "B8": "$19.00", - "C2": "40", - "C3": "50", - "C4": "20", - "C5": "20", - "C6": "80", - "C7": "80", - "C8": "20", - "D2": "$1,080.00", - "D3": "$1,525.00", - "D4": "$540.00", - "D5": "$360.00", - "D6": "$4,000.00", - "D7": "$4,160.00", - "D8": "$380.00", - "E2": "$66.96", - "E3": "$94.55", - "E4": "$33.48", - "E5": "$22.32", - "E6": "$248.00", - "E7": "$257.92", - "E8": "$23.56", - "F2": "$15.66", - "F3": "$22.11", - "F4": "$7.83", - "F5": "$5.22", - "F6": "$58.00", - "F7": "$60.32", - "F8": "$5.51", - "G2": "$2.24", - "G3": "$2.80", - "G4": "$1.12", - "G5": "$1.12", - "G6": "$4.48", - "G7": "$4.48", - "G8": "$1.12", - "H2": "$0.32", - "H3": "$0.46", - "H4": "$0.16", - "H5": "$0.11", - "H6": "$1.20", - "H7": "$1.25", - "H8": "$0.11", - "I2": "$5.71", - "I3": "$8.06", - "I4": "$2.85", - "I5": "$1.90", - "I6": "$21.14", - "I7": "$21.99", - "I8": "$2.01", - "J2": "$6.26", - "J3": "$8.85", - "J4": "$3.13", - "J5": "$2.09", - "J6": "$23.20", - "J7": "$24.13", - "J8": "$2.20", - "K2": "$97.16", - "K3": "$136.82", - "K4": "$48.58", - "K5": "$32.76", - "K6": "$356.02", - "K7": "$370.08", - "K8": "$34.52" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "B3": "Maturity", - "A4": "CurveDate", - "B4": "10Y", - "C4": "1Y", - "D4": "2Y", - "E4": "30Y", - "F4": "5Y", - "G4": "Grand Total", - "A5": "2024-01-01", - "B5": "$3.52", - "C5": "$3.73", - "D5": "$3.71", - "E5": "$3.86", - "F5": "$4.71", - "G5": "$19.53", - "A6": "2024-01-02", - "B6": "$3.73", - "C6": "$3.62", - "D6": "$3.79", - "E6": "$4.81", - "F6": "$4.54", - "G6": "$20.49", - "A7": "2024-01-03", - "B7": "$4.28", - "C7": "$4.98", - "D7": "$4.63", - "E7": "$3.70", - "F7": "$4.79", - "G7": "$22.38", - "A8": "2024-01-04", - "B8": "$4.03", - "C8": "$4.84", - "D8": "$3.94", - "E8": "$4.84", - "F8": "$4.49", - "G8": "$22.14", - "A9": "Grand Total", - "B9": "$15.56", - "C9": "$17.17", - "D9": "$16.07", - "E9": "$17.21", - "F9": "$18.53", - "G9": "$84.54" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "ABC", - "B1": "-0.08" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Carla White", - "A2": "$155.00", - "B1": "$8,758.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1e4ede56-163f-4815-b634-7946cf9e60c0", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Content Marketing", - "A2": "Email", - "A3": "PPC", - "A4": "SEO", - "A5": "Social Media", - "B1": "Harness Frictionless Users", - "B2": "Transform Out-Of-The-Box Schemas", - "B3": "Morph Back-End E-Business", - "B4": "Facilitate Dynamic Channels", - "B5": "Re-Intermediate Cutting-Edge Web-Readiness", - "C1": "6.61", - "C2": "5.99", - "C3": "5.02", - "C4": "5.49", - "C5": "5.68" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "8499e399-48e6-4603-bd96-9b45f5aabed0", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Omar Donovan", - "A2": "Omar Donovan" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1876764.43" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "991105", - "A2": "1044283", - "A3": "1095308", - "A4": "1169204", - "B1": "4299900" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "764ca583-7091-4b7f-8663-5e996db26a2c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "16.74%", - "B1": "27.80%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9f71aa71-07f7-4421-8757-bbccbeab0984", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/gold_solution_3.xlsx" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Star City", - "B1": "39.11%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Gotham", - "B1": "4" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Finance", - "B1": "1,000" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "21680.086" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "d2c56ee0-7863-45be-b503-eb1639454629", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Scenario 3", - "A2": "Scenario 2", - "A3": "Scenario 1" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "077926f8-6e93-406b-9eb9-b44907173c74", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Vendor", - "A2": "vendor 18", - "A3": "vendor 15", - "A4": "vendor 19", - "A5": "vendor 1", - "A6": "vendor 14", - "A7": "vendor 9", - "A8": "vendor 8", - "A9": "vendor 11", - "B1": "1099 Report amount", - "B2": "$4.00", - "B3": "$48.00", - "B4": "$81.00", - "B5": "$92.60", - "B6": "$177.00", - "B7": "$866.00", - "B8": "$1,661.00", - "B9": "$2,615.00", - "A10": "vendor 4", - "A11": "vendor 5", - "B10": "$6,446.00", - "B11": "$6,519.00" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "f1afee7f-df70-4a11-a65e-767a718f2117", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2025-02" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "36e43b84-e583-4f02-a160-74049bcab901", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "Platform", - "A5": "Facebook", - "A9": "Instagram", - "B4": "Values", - "B5": "Sum of Posts", - "B6": "Sum of Likes", - "B7": "Sum of Shares", - "B8": "Sum of Comments", - "B9": "Sum of Posts", - "C4": "August", - "C5": "66", - "C6": "3899", - "C7": "618", - "C8": "241", - "C9": "54", - "D4": "July", - "D5": "65", - "D6": "4008", - "D7": "749", - "D8": "260", - "D9": "58", - "E4": "June", - "E5": "54", - "E6": "3236", - "E7": "560", - "E8": "296", - "E9": "70", - "F4": "November", - "F5": "39", - "F6": "2066", - "F7": "430", - "F8": "209", - "F9": "52", - "G4": "October", - "G5": "68", - "G6": "3048", - "G7": "773", - "G8": "271", - "G9": "54", - "H4": "September", - "H5": "64", - "H6": "2918", - "H7": "769", - "H8": "262", - "H9": "57", - "I4": "Grand Total", - "I5": "356", - "I6": "19175", - "I7": "3899", - "I8": "1539", - "I9": "345", - "A13": "LinkedIn", - "A17": "Twitter", - "A21": "Grand Total", - "B10": "Sum of Likes", - "B11": "Sum of Shares", - "B12": "Sum of Comments", - "B13": "Sum of Posts", - "B14": "Sum of Likes", - "B15": "Sum of Shares", - "B16": "Sum of Comments", - "B17": "Sum of Posts", - "B18": "Sum of Likes", - "B19": "Sum of Shares", - "B20": "Sum of Comments", - "B21": "Sum of Posts", - "B22": "Sum of Likes", - "B23": "Sum of Shares", - "B24": "Sum of Comments", - "C10": "3231", - "C11": "541", - "C12": "226", - "C13": "61", - "C14": "2923", - "C15": "668", - "C16": "295", - "C17": "60", - "C18": "3290", - "C19": "671", - "C20": "295", - "C21": "241", - "C22": "13343", - "C23": "2498", - "C24": "1057", - "D10": "3025", - "D11": "700", - "D12": "232", - "D13": "60", - "D14": "3122", - "D15": "589", - "D16": "314", - "D17": "54", - "D18": "2950", - "D19": "454", - "D20": "271", - "D21": "237", - "D22": "13105", - "D23": "2492", - "D24": "1077", - "E10": "4195", - "E11": "834", - "E12": "317", - "E13": "45", - "E14": "2291", - "E15": "451", - "E16": "185", - "E17": "63", - "E18": "3366", - "E19": "797", - "E20": "278", - "E21": "232", - "E22": "13088", - "E23": "2642", - "E24": "1076", - "F10": "2860", - "F11": "514", - "F12": "249", - "F13": "61", - "F14": "3116", - "F15": "708", - "F16": "246", - "F17": "63", - "F18": "2714", - "F19": "581", - "F20": "243", - "F21": "215", - "F22": "10756", - "F23": "2233", - "F24": "947", - "G10": "2993", - "G11": "623", - "G12": "265", - "G13": "70", - "G14": "3596", - "G15": "716", - "G16": "336", - "G17": "58", - "G18": "2382", - "G19": "578", - "G20": "293", - "G21": "250", - "G22": "12019", - "G23": "2690", - "G24": "1165", - "H10": "3409", - "H11": "401", - "H12": "313", - "H13": "65", - "H14": "3271", - "H15": "771", - "H16": "278", - "H17": "51", - "H18": "2156", - "H19": "431", - "H20": "312", - "H21": "237", - "H22": "11754", - "H23": "2372", - "H24": "1165", - "I10": "19713", - "I11": "3613", - "I12": "1602", - "I13": "362", - "I14": "18319", - "I15": "3903", - "I16": "1654", - "I17": "349", - "I18": "16858", - "I19": "3512", - "I20": "1692", - "I21": "1412", - "I22": "74065", - "I23": "14927", - "I24": "6487" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Platform", - "A2": "Twitter", - "A3": "Facebook", - "A4": "Instagram", - "A5": "LinkedIn", - "B1": "RATIO OF HIGH / LOW", - "B2": "1.88", - "B3": "2.64", - "B4": "3.17", - "B5": "2.05" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "c437f78d-6382-43b2-b857-153db6efa1c9", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A4": "Jun", - "A5": "Jul", - "A6": "Aug", - "A7": "Sep", - "A8": "Oct", - "A9": "Nov", - "B3": "Sum of Posts", - "B4": "124", - "B5": "123", - "B6": "120", - "B7": "121", - "B8": "122", - "B9": "91", - "C3": "Sum of Likes", - "C4": "7431", - "C5": "7033", - "C6": "7130", - "C7": "6327", - "C8": "6041", - "C9": "4926", - "D3": "Sum of Comments", - "D4": "613", - "D5": "492", - "D6": "467", - "D7": "575", - "D8": "536", - "D9": "458", - "E3": "Sum of Shares", - "E4": "1394", - "E5": "1449", - "E6": "1159", - "E7": "1170", - "E8": "1396", - "E9": "944", - "A10": "Grand Total", - "B10": "701", - "C10": "38888", - "D10": "3141", - "E10": "7512" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.68%", - "E3": "43.14%", - "E4": "43.77%", - "E5": "43.27%", - "E6": "44.90%", - "E7": "44.03%", - "E8": "44.06%", - "E9": "43.97%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "E10": "44.37%", - "E11": "44.10%", - "E12": "46.70%", - "E13": "48.06%", - "E14": "45.94%", - "E15": "45.48%", - "E16": "44.06%", - "E17": "43.93%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "9ba9631e-8560-4dc4-a285-8d186715a542", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.7%", - "E3": "43.1%", - "E4": "43.8%", - "E5": "43.3%", - "E6": "44.9%", - "E7": "44.0%", - "E8": "44.1%", - "E9": "44.0%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "A18": "5", - "A19": "6", - "A20": "7", - "A21": "8", - "A22": "9", - "A23": "10", - "A24": "11", - "A25": "12", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "B18": "2024", - "B19": "2024", - "B20": "2024", - "B21": "2024", - "B22": "2024", - "B23": "2024", - "B24": "2024", - "B25": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "C18": "21095", - "C19": "21015", - "C20": "21141", - "C21": "21311", - "C22": "21064", - "C23": "21182", - "C24": "21209", - "C25": "21238", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "D18": "34237", - "D19": "33819", - "D20": "33547", - "D21": "33832", - "D22": "33424", - "D23": "33162", - "D24": "33042", - "D25": "32929", - "E10": "44.4%", - "E11": "44.1%", - "E12": "46.7%", - "E13": "48.1%", - "E14": "45.9%", - "E15": "45.5%", - "E16": "44.1%", - "E17": "43.9%", - "E18": "43.1%", - "E19": "41.8%", - "E20": "41.3%", - "E21": "40.4%", - "E22": "39.7%", - "E23": "38.7%", - "E24": "37.9%", - "E25": "37.2%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Given the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2023-04-30", - "A2": "0.48", - "A3": "0.00296" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "D2": "Rank Based on 2024 Sales", - "D3": "4", - "D4": "3", - "D5": "7", - "D6": "17", - "D7": "12", - "D8": "1", - "D9": "18", - "D10": "8", - "D11": "6", - "D12": "14", - "D13": "5", - "D14": "13", - "D15": "9", - "D16": "20", - "D17": "11", - "D18": "10", - "D19": "19", - "D20": "15", - "D21": "2", - "D22": "16" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "20507108-4188-402e-8cfe-2354309bad6c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A2": "Region", - "A3": "Central", - "A4": "East", - "A5": "North", - "A6": "South", - "A7": "West", - "B2": "2023", - "B3": "351351.87", - "B4": "364129.72", - "B5": "377252.87", - "B6": "396259.93", - "B7": "395672.78", - "C2": "2024", - "C3": "368644.54", - "C4": "399862.83", - "C5": "364113.70", - "C6": "345896.80", - "C7": "393456.82", - "D2": "YoY Growth", - "D3": "4.92%", - "D4": "9.81%", - "D5": "-3.48%", - "D6": "-12.71%", - "D7": "-0.56%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "In your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Shopping", - "B1": "$ (16,401.91)" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "6.17%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "24.20%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "371507f7-721a-457e-9618-bc8fba1b909b", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "$316,309.56" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "4a081ed2-532c-4895-a034-ab0076927c7c", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "2026-03", - "A2": "2025-09", - "A3": "2026-12" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Sum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/gold_solution_3.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "JPY", - "A2": "USD", - "A3": "EUR", - "A4": "GBP", - "B1": "3500000.00", - "B2": "64351.25", - "B3": "43501.00", - "B4": "15000.25" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "This spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "31508ec6-c993-4e00-b70c-093dba016fcc", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "1.07%", - "A2": "1.73%" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Which city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "133ef005-207c-490f-b55c-7734fd1678da", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Central City" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - }, - { - "prompt": "Work in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header “Date” plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", - "mcp_config": { - "hud": { - "url": "https://mcp.hud.so/v3/mcp", - "headers": { - "Authorization": "Bearer ${HUD_API_KEY}", - "Mcp-Image": "hudevals/hud-remote-browser:0.1.1" - } - } - }, - "id": "eb410896-3e1e-4491-9460-061579b65c6f", - "metadata": { - "partial": true, - "gold_file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/gold_solution_2.xlsx?" - }, - "setup_tool": { - "name": "setup", - "arguments": { - "name": "sheets_from_xlsx", - "arguments": { - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?" - } - } - }, - "evaluate_tool": { - "name": "evaluate", - "arguments": { - "name": "sheets_cell_values", - "arguments": { - "args": { - "A1": "Date", - "A2": "2025-01-05", - "A3": "2025-01-12", - "A4": "2025-01-15", - "A5": "2025-01-20", - "A6": "2025-01-25", - "A7": "2025-01-30", - "A8": "2025-02-05", - "A9": "2025-02-08", - "B1": "Description", - "B2": "Membership Fee", - "B3": "Project Income", - "B4": "Interest", - "B5": "Event Revenue", - "B6": "Refund", - "B7": "Consulting Fee", - "B8": "Website Hosting", - "B9": "Maintenance", - "C1": "Amount", - "C2": "-250.00", - "C3": "5600.00", - "C4": "350.00", - "C5": "7000.00", - "C6": "-5000.00", - "C7": "7800.00", - "C8": "-99.99", - "C9": "-750.25", - "D1": "Currency", - "D2": "USD", - "D3": "USD", - "D4": "USD", - "D5": "USD", - "D6": "USD", - "D7": "USD", - "D8": "USD", - "D9": "USD", - "A10": "2025-02-15", - "A11": "2025-02-18", - "A12": "2025-02-20", - "A13": "2025-02-25", - "A14": "2025-02-28", - "A15": "2025-03-01", - "A16": "2025-03-05", - "A17": "2025-03-10", - "A18": "2025-03-15", - "A19": "2025-03-20", - "A20": "2025-03-25", - "B10": "Invoice Payment", - "B11": "Equipment Purchase", - "B12": "Software License", - "B13": "Bonus", - "B14": "Travel Expenses", - "B15": "Subscription", - "B16": "Advertising", - "B17": "Office Supplies", - "B18": "Utilities", - "B19": "Legal Fees", - "B20": "Marketing", - "C10": "2500.00", - "C11": "-3600.00", - "C12": "-3000.00", - "C13": "4800.00", - "C14": "-1750.00", - "C15": "-1200.00", - "C16": "-2450.50", - "C17": "-450.75", - "C18": "-899.00", - "C19": "-4000.00", - "C20": "-2200.00", - "D10": "USD", - "D11": "USD", - "D12": "USD", - "D13": "USD", - "D14": "USD", - "D15": "USD", - "D16": "USD", - "D17": "USD", - "D18": "USD", - "D19": "USD", - "D20": "USD" - } - } - } - }, - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "agent_config": { - "system_prompt": "\n All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n ", - "allowed_tools": [ - "*" - ], - "disallowed_tools": [ - "setup", - "evaluate" - ], - "append_setup_tool": true, - "initial_screenshot": true - } - } -] \ No newline at end of file diff --git a/sheetbench_tasks.json b/sheetbench_tasks.json deleted file mode 100644 index 7e7671577..000000000 --- a/sheetbench_tasks.json +++ /dev/null @@ -1,2411 +0,0 @@ -[ - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nCalculate from the RawData tab the z-scores from the mean close price for each row. Return, starting in ANSWER!A1 and descending to ANSWER!A5, the 5 dates with the greatest absolute value of standard deviations from the mean", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c6ddeb9a-0c16-4f5e-8a06-f148ebb4be8a/setup_input_2.xlsx?", - "expected_cells": { - "A1": "1/12/2024", - "A2": "1/10/2024", - "A3": "1/15/2024", - "A4": "1/11/2024", - "A5": "1/17/2024" - } - }, - "id": "6e4744c7-b2c9-4bb6-807e-2cc144a4e8c2" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nCalculate the # of unique customer IDs in the worksheet ANSWER cell A1 and calculate the # of duplicate IDs in cell A2. Create a pivot table in the ANSWER tab, cell B1 with the CustomerID field as a row, Date (at the years level) as a column, and insert the Amount field as a value. The values should be in basic form without thousands separators and two decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0226adc3-051a-49e9-b1da-feb5c72ddfc7/setup_input_2.xlsx?", - "expected_cells": { - "B2": "CustomerID", - "B3": "476232", - "B4": "963589", - "B5": "1430820", - "B6": "2410043", - "B7": "3789483", - "B8": "4226444", - "B9": "4308096", - "C2": "2023", - "C5": "686.40", - "C8": "966.76", - "C9": "863.37", - "D2": "2024", - "D3": "912.70", - "D4": "397.62", - "D6": "383.38", - "D7": "619.56", - "E2": "Grand Total", - "E3": "912.70", - "E4": "397.62", - "E5": "686.40", - "E6": "383.38", - "E7": "619.56", - "E8": "966.76", - "E9": "863.37", - "B10": "5907496", - "B11": "6224271", - "B12": "6538101", - "B13": "6584993", - "B14": "6791810", - "B15": "7250781", - "B16": "8518187", - "B17": "8668097", - "B18": "8885050", - "B19": "9296053", - "B20": "10414682", - "B21": "10839621", - "B22": "12313143", - "B23": "13672423", - "B24": "14744105", - "B25": "14780608", - "B26": "15144539", - "B27": "15616289", - "B28": "15694782", - "B29": "15904102", - "B30": "16385696", - "B31": "17254618", - "B32": "17441704", - "B33": "17624107", - "B34": "18043433", - "B35": "18296075", - "B36": "18600797", - "B37": "19026954", - "B38": "19395319", - "B39": "19650142", - "B40": "19842318", - "B41": "20241148", - "B42": "20587659", - "B43": "21647994", - "B44": "21671104", - "B45": "21935183", - "B46": "22115836", - "B47": "22846037", - "B48": "23621954", - "B49": "24962520", - "B50": "25073201", - "B51": "25090031", - "B52": "25096194", - "B53": "25211207", - "B54": "26168579", - "B55": "26250755", - "B56": "26419282", - "B57": "26953175", - "B58": "27281086", - "B59": "27646724", - "B60": "30370166", - "B61": "30738270", - "B62": "30876644", - "B63": "32255126", - "B64": "32373769", - "B65": "33121017", - "B66": "33897455", - "B67": "34507001", - "B68": "35475491", - "B69": "36972111", - "B70": "37040802", - "B71": "37543198", - "B72": "37609900", - "B73": "37674649", - "B74": "38019388", - "B75": "38658747", - "B76": "38793189", - "B77": "39134950", - "B78": "40399799", - "B79": "41039038", - "B80": "41271414", - "B81": "41870334", - "B82": "42495130", - "B83": "43189090", - "B84": "43235421", - "B85": "43837144", - "B86": "44115166", - "B87": "44270797", - "B88": "45380457", - "B89": "45386282", - "B90": "46196026", - "B91": "46300157", - "B92": "48512428", - "B93": "49546866", - "B94": "49687477", - "B95": "50893874", - "B96": "51093532", - "B97": "51126698", - "B98": "51397982", - "B99": "52376160", - "C10": "714.36", - "C11": "369.01", - "C12": "525.20", - "C20": "120.72", - "C21": "269.30", - "C22": "422.36", - "C26": "971.83", - "C28": "110.65", - "C29": "496.66", - "C30": "841.89", - "C31": "886.06", - "C32": "536.09", - "C34": "16.13", - "C38": "90.40", - "C39": "76.23", - "C40": "185.33", - "C41": "65.32", - "C42": "111.51", - "C43": "493.89", - "C44": "787.04", - "C45": "578.58", - "C46": "204.48", - "C48": "985.49", - "C50": "268.33", - "C52": "34.51", - "C54": "61.19", - "C60": "62.83", - "C63": "326.11", - "C64": "898.66", - "C65": "251.17", - "C66": "902.46", - "C67": "442.33", - "C70": "561.10", - "C71": "469.05", - "C74": "477.14", - "C76": "284.69", - "C80": "266.21", - "C82": "699.49", - "C85": "82.73", - "C87": "542.97", - "C88": "446.42", - "C89": "380.14", - "C91": "683.73", - "C92": "566.99", - "C94": "646.75", - "C97": "748.88", - "C98": "253.40", - "D13": "174.59", - "D14": "374.65", - "D15": "339.03", - "D16": "140.66", - "D17": "862.92", - "D18": "282.39", - "D19": "847.67", - "D23": "616.14", - "D24": "376.83", - "D25": "523.78", - "D27": "701.88", - "D29": "496.66", - "D33": "805.83", - "D35": "579.30", - "D36": "577.84", - "D37": "673.27", - "D38": "90.40", - "D47": "47.03", - "D48": "985.49", - "D49": "779.72", - "D51": "277.24", - "D53": "702.84", - "D55": "334.42", - "D56": "875.84", - "D57": "570.46", - "D58": "60.52", - "D59": "29.15", - "D61": "822.61", - "D62": "931.36", - "D68": "728.49", - "D69": "454.44", - "D72": "866.51", - "D73": "968.06", - "D74": "477.14", - "D75": "154.14", - "D77": "69.76", - "D78": "323.27", - "D79": "862.51", - "D81": "335.73", - "D83": "555.46", - "D84": "685.07", - "D86": "942.65", - "D90": "813.06", - "D93": "671.93", - "D95": "995.04", - "D96": "896.64", - "D98": "253.40", - "D99": "66.10", - "E10": "714.36", - "E11": "369.01", - "E12": "525.20", - "E13": "174.59", - "E14": "374.65", - "E15": "339.03", - "E16": "140.66", - "E17": "862.92", - "E18": "282.39", - "E19": "847.67", - "E20": "120.72", - "E21": "269.30", - "E22": "422.36", - "E23": "616.14", - "E24": "376.83", - "E25": "523.78", - "E26": "971.83", - "E27": "701.88", - "E28": "110.65", - "E29": "993.32", - "E30": "841.89", - "E31": "886.06", - "E32": "536.09", - "E33": "805.83", - "E34": "16.13", - "E35": "579.30", - "E36": "577.84", - "E37": "673.27", - "E38": "180.80", - "E39": "76.23", - "E40": "185.33", - "E41": "65.32", - "E42": "111.51", - "E43": "493.89", - "E44": "787.04", - "E45": "578.58", - "E46": "204.48", - "E47": "47.03", - "E48": "1970.98", - "E49": "779.72", - "E50": "268.33", - "E51": "277.24", - "E52": "34.51", - "E53": "702.84", - "E54": "61.19", - "E55": "334.42", - "E56": "875.84", - "E57": "570.46", - "E58": "60.52", - "E59": "29.15", - "E60": "62.83", - "E61": "822.61", - "E62": "931.36", - "E63": "326.11", - "E64": "898.66", - "E65": "251.17", - "E66": "902.46", - "E67": "442.33", - "E68": "728.49", - "E69": "454.44", - "E70": "561.10", - "E71": "469.05", - "E72": "866.51", - "E73": "968.06", - "E74": "954.28", - "E75": "154.14", - "E76": "284.69", - "E77": "69.76", - "E78": "323.27", - "E79": "862.51", - "E80": "266.21", - "E81": "335.73", - "E82": "699.49", - "E83": "555.46", - "E84": "685.07", - "E85": "82.73", - "E86": "942.65", - "E87": "542.97", - "E88": "446.42", - "E89": "380.14", - "E90": "813.06", - "E91": "683.73", - "E92": "566.99", - "E93": "671.93", - "E94": "646.75", - "E95": "995.04", - "E96": "896.64", - "E97": "748.88", - "E98": "506.80", - "E99": "66.10", - "B100": "52697630", - "B101": "52870804", - "B102": "53239020", - "B103": "53335630", - "B104": "53858146", - "B105": "54040755", - "B106": "54479395", - "B107": "54528192", - "B108": "54730083", - "B109": "55444897", - "B110": "55450065", - "B111": "55853989", - "B112": "55859479", - "B113": "55978545", - "B114": "56702798", - "B115": "56940756", - "B116": "58152560", - "B117": "59169574", - "B118": "59736806", - "B119": "61602489", - "B120": "62665919", - "B121": "62758192", - "B122": "62943277", - "B123": "63051186", - "B124": "65657133", - "B125": "65899458", - "B126": "66761945", - "B127": "66846840", - "B128": "67283606", - "B129": "67548873", - "B130": "69613838", - "B131": "69765117", - "B132": "70098167", - "B133": "70534368", - "B134": "71078229", - "B135": "71319902", - "B136": "71369648", - "B137": "71376141", - "B138": "72217443", - "B139": "72659809", - "B140": "72883709", - "B141": "73191421", - "B142": "73684220", - "B143": "74014279", - "B144": "74088126", - "B145": "75410476", - "B146": "75817527", - "B147": "77140593", - "B148": "77558800", - "B149": "78916470", - "B150": "79031936", - "B151": "79751742", - "B152": "80286530", - "B153": "80765899", - "B154": "82060237", - "B155": "82306595", - "B156": "83101113", - "B157": "83211478", - "B158": "83713620", - "B159": "84770820", - "B160": "84800206", - "B161": "84952943", - "B162": "86407021", - "B163": "86619158", - "B164": "86663007", - "B165": "87144451", - "B166": "87254792", - "B167": "88194202", - "B168": "88266169", - "B169": "88761732", - "B170": "88882642", - "B171": "89421277", - "B172": "89565544", - "B173": "90841330", - "B174": "91483447", - "B175": "91590435", - "B176": "91939241", - "B177": "92335820", - "B178": "92422728", - "B179": "92676749", - "B180": "93202190", - "B181": "93479509", - "B182": "95353791", - "B183": "95696393", - "B184": "95804583", - "B185": "95860510", - "B186": "96051566", - "B187": "96263709", - "B188": "96456101", - "B189": "99519803", - "B190": "Grand Total", - "C100": "409.46", - "C101": "28.08", - "C106": "184.81", - "C107": "906.16", - "C108": "1094.00", - "C109": "818.87", - "C110": "192.68", - "C111": "385.72", - "C117": "976.44", - "C118": "310.80", - "C119": "354.99", - "C120": "898.04", - "C122": "397.81", - "C128": "537.90", - "C129": "886.96", - "C131": "420.66", - "C133": "359.50", - "C134": "60.29", - "C139": "740.19", - "C142": "554.52", - "C144": "759.21", - "C145": "116.62", - "C148": "302.90", - "C152": "187.16", - "C153": "847.55", - "C154": "700.62", - "C155": "190.59", - "C161": "432.99", - "C163": "886.00", - "C167": "715.20", - "C169": "101.82", - "C171": "122.95", - "C174": "609.90", - "C175": "867.14", - "C177": "675.11", - "C178": "623.53", - "C182": "40.39", - "C184": "554.62", - "C187": "839.28", - "C189": "717.61", - "C190": "43541.41", - "D102": "995.08", - "D103": "761.74", - "D104": "773.13", - "D105": "364.27", - "D112": "841.73", - "D113": "560.15", - "D114": "824.63", - "D115": "783.50", - "D116": "788.60", - "D117": "976.44", - "D118": "310.80", - "D121": "92.40", - "D123": "171.84", - "D124": "336.61", - "D125": "206.92", - "D126": "345.03", - "D127": "505.00", - "D130": "539.76", - "D132": "189.69", - "D135": "713.31", - "D136": "993.98", - "D137": "541.38", - "D138": "790.14", - "D139": "740.19", - "D140": "481.88", - "D141": "83.31", - "D143": "883.45", - "D146": "394.60", - "D147": "281.52", - "D149": "64.48", - "D150": "645.82", - "D151": "771.06", - "D156": "405.74", - "D157": "741.29", - "D158": "196.84", - "D159": "152.71", - "D160": "356.93", - "D162": "138.69", - "D164": "929.08", - "D165": "887.13", - "D166": "554.21", - "D168": "820.22", - "D170": "939.97", - "D171": "122.95", - "D172": "335.69", - "D173": "917.55", - "D176": "514.59", - "D179": "288.94", - "D180": "214.12", - "D181": "16.74", - "D183": "27.28", - "D185": "458.58", - "D186": "648.09", - "D188": "985.01", - "D190": "56717.97", - "E100": "409.46", - "E101": "28.08", - "E102": "995.08", - "E103": "761.74", - "E104": "773.13", - "E105": "364.27", - "E106": "184.81", - "E107": "906.16", - "E108": "1094.00", - "E109": "818.87", - "E110": "192.68", - "E111": "385.72", - "E112": "841.73", - "E113": "560.15", - "E114": "824.63", - "E115": "783.50", - "E116": "788.60", - "E117": "1952.88", - "E118": "621.60", - "E119": "354.99", - "E120": "898.04", - "E121": "92.40", - "E122": "397.81", - "E123": "171.84", - "E124": "336.61", - "E125": "206.92", - "E126": "345.03", - "E127": "505.00", - "E128": "537.90", - "E129": "886.96", - "E130": "539.76", - "E131": "420.66", - "E132": "189.69", - "E133": "359.50", - "E134": "60.29", - "E135": "713.31", - "E136": "993.98", - "E137": "541.38", - "E138": "790.14", - "E139": "1480.38", - "E140": "481.88", - "E141": "83.31", - "E142": "554.52", - "E143": "883.45", - "E144": "759.21", - "E145": "116.62", - "E146": "394.60", - "E147": "281.52", - "E148": "302.90", - "E149": "64.48", - "E150": "645.82", - "E151": "771.06", - "E152": "187.16", - "E153": "847.55", - "E154": "700.62", - "E155": "190.59", - "E156": "405.74", - "E157": "741.29", - "E158": "196.84", - "E159": "152.71", - "E160": "356.93", - "E161": "432.99", - "E162": "138.69", - "E163": "886.00", - "E164": "929.08", - "E165": "887.13", - "E166": "554.21", - "E167": "715.20", - "E168": "820.22", - "E169": "101.82", - "E170": "939.97", - "E171": "245.90", - "E172": "335.69", - "E173": "917.55", - "E174": "609.90", - "E175": "867.14", - "E176": "514.59", - "E177": "675.11", - "E178": "623.53", - "E179": "288.94", - "E180": "214.12", - "E181": "16.74", - "E182": "40.39", - "E183": "27.28", - "E184": "554.62", - "E185": "458.58", - "E186": "648.09", - "E187": "839.28", - "E188": "985.01", - "E189": "717.61", - "E190": "100259.38" - } - }, - "id": "67d1c961-47c6-42a5-8a68-67bee5d1f1c1" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFind the GET request which most commonly results in an error. Place the URL in ANSWER!A1", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/aba12ab9-a833-4ae9-ab60-80044111621f/setup_input_2.xlsx?", - "expected_cells": { - "A1": "/api/users" - } - }, - "id": "a9efbeeb-3fe0-4e15-9a6b-773437858ad4" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFor a company with USD 5 million in cash, they want to expand and increase per month 2 employees. Consider the increase per month on sales is 3.5%, determine if the company could contining hiring employees or not, if not when they will have spend USD 3 million of their cash, put the month and the year on ANSWER!A1 formatted as YYYY-MM. If they can continue hiring such that they will NOT drop below a cash balance of 3M, place FALSE in ANSWER!A1.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d00d7968-047c-4db6-af0c-3d14f1a1e751/setup_input_2.xlsx?", - "expected_cells": { - "A1": "2026-04" - } - }, - "id": "e3edb2a9-6f28-4d2a-9352-3739b6919643" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nFor the ticker that has the greatest correlation between volume and next day price change (%) find the day with the greatest volume and the next days price change (%)\n - put the ticker in ANSWER!A1\n - put the volume in ANSWER B1 (basic number with no thousands separators and no decimal precision and no dollar sign)\n - put the next day price change in ANSWER C1 (percentage format with no decimal points)\nNOTE\n- use CORREL to determine correlation\n- create a pivot table to compare each ticker's volume and price side by side, and then create a separate array to determine day over day price change (%)s over time. Lastly, run the CORREL function across these side by side arrays to generate correlation for each ticker", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/715426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?", - "expected_cells": { - "A1": "ABC", - "B1": "4999972", - "C1": "145%" - } - }, - "id": "0963d367-f0ac-4be0-8f46-337bf335e68f" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven a beginning loan balance of $150,000 dated March 21, 2025 at 12% interest, with payment amounts of $6965 on April 1, $25,000 on April 5th, and $7500 on May 1st. Please calculate the remaining principal balance after the May 1, 2025 payment, assuming that for each payment detailed in cells B3:B5 in the \"INPUT\" sheet, the payment went (i) first to pay any interest accrued since the prior payment (or in the case of the first payment in row 3, since the loan origination) and that (ii) the remainder of such payment then went towards paying down the outstanding principal balance. Place the answer in cell A1 of the ANSWER tab. Round it to 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/31b03ce3-43e8-4ff6-b3a3-8ebbcbbcc8b3/setup_input_2.xlsx?", - "expected_cells": { - "A1": "$112,281.49" - } - }, - "id": "1bb82b07-9899-4360-a3ce-1815eeb5c80d" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven an automobile was purchased in Feb 1, 2022 with an estimated life of 7 years at a cost of 45,000 and was disposed of in April 1, 2025 with no salvage value and monthly depreciation is calculted to the nearest cent, calculate the loss on disposal. Put your answer in the ANSWER tab in cell A1. Format with dollar sign, thousands separators and 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/cc654e52-d186-459d-8416-81bf1c80462f/setup_input_2.xlsx?", - "expected_cells": { - "A1": "$24,642.86" - } - }, - "id": "49ccea4d-4aaf-4faf-9659-b389686568a7" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the amounts in foreign currency, convert them to usd using the FX tab. Sum the total amount in USD, put the result on ANSWER!A1. The answer should have no thousands separator with 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9fb9cb55-a466-40a0-afec-dad4d8fb3362/setup_input_2.xlsx?", - "expected_cells": { - "A1": "1664934.45" - } - }, - "id": "48d43d57-ed1d-4df0-8b36-62380bba7865" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the customer churn data, identify the months with the highest and lowest net new signups. Place the month with the highest signups in ANSWER!A1 and the number of signup for that month in B1. Place the month with the lowest signups in ANSWER!A2 and the number of signups for that month in B2. Format the month in all caps and three letters.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c233ba4-799e-4eae-92e7-f82077e62c38/setup_input_2.xlsx?", - "expected_cells": { - "A1": "MAY", - "A2": "OCT", - "B1": "95", - "B2": "73" - } - }, - "id": "ff114e80-196d-4a7a-99ca-152bac4fba90" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data from the client, format the rows and create a pivot table on answer A1 with Category as column 1 and Sum of Amount as column 2, sort from smallest to largest by Sum of Amounts. Round the number to no decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0e9ca100-9e38-4e70-a31b-750b7e9ae4b7/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Category", - "A2": "Meals & Entertainment", - "A3": "Travel", - "A4": "Office Supplies", - "B1": "Sum of Amount", - "B2": "0", - "B3": "345", - "B4": "45760" - } - }, - "id": "1c0a2b67-393c-4a92-8cde-1cd1de4fe00b" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data provided, make a pivot table on ANSWER!A3, using the metrics: Net Income, Revenue and Total Assets and the title for the headings the Quarter, for the quarter use the structure: 4 Digits of the company, year and quarter.. Ensure that the metrics are the rows and the quarters are the columns. Label the quarters like this: FORD2024Q1 for Q1 2024. There should be both grand totals.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/abbb6413-bc91-473f-bf8b-be51fcf7b189/setup_input_2.xlsx?", - "expected_cells": { - "A3": "Sum of Amount (USD Millions)", - "A5": "Net Income", - "A6": "Revenue", - "A7": "Total Assets", - "A8": "Grand Total", - "B4": "FORD2024Q1", - "B5": "1.33", - "B6": "42.78", - "B7": "274.34", - "B8": "318.45", - "C4": "FORD2024Q2", - "C5": "1.83", - "C6": "44.81", - "C7": "276.59", - "C8": "323.23", - "D4": "FORD2024Q3", - "D5": "896.00", - "D6": "43.07", - "D7": "287.05", - "D8": "1226.12", - "E4": "FORD2025Q1", - "E5": "471.00", - "E6": "40.66", - "E7": "284.54", - "E8": "796.20", - "F4": "Grand Total", - "F5": "1370.17", - "F6": "171.32", - "F7": "1122.51", - "F8": "2664.00" - } - }, - "id": "9d1a335a-aeae-4b63-ba09-4b9ac0b1501c" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the data, create a pivot table in the ANSWER tab with AssignedAgentID and Status fields as rows. Add a column for the count of the # of TicketID and a calculated field that averages the ResolutionTimeHours and replacing an errors with zeros. Average of ResolutionTimeHours should be formatted with 2 decimal places. Note that the zeros should not be included in the calculation of the average. If you are using GoogleSheets, start your pivot table on cell A3.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/749b8a15-cdb3-454e-bfc3-f7d0e2b40830/setup_input_2.xlsx?", - "expected_cells": { - "C3": "Count of TicketID", - "D3": "Average of ResolutionTimeHours", - "A4": "Agent-101", - "B4": "Closed", - "C4": "73", - "D4": "13.90", - "B5": "In Progress", - "C5": "30", - "B6": "Open", - "C6": "31", - "B7": "Resolved", - "C7": "66", - "D7": "22.04", - "B8": "Waiting for Customer", - "C8": "12", - "C9": "212", - "D9": "17.76", - "A10": "Agent-102", - "B10": "Closed", - "C10": "70", - "D10": "28.53", - "B11": "In Progress", - "C11": "36", - "B12": "Open", - "C12": "32", - "B13": "Resolved", - "C13": "77", - "D13": "17.09", - "B14": "Waiting for Customer", - "C14": "10", - "C15": "225", - "D15": "22.54", - "A16": "Agent-103", - "B16": "Closed", - "C16": "73", - "D16": "23.11", - "B17": "In Progress", - "C17": "39", - "B18": "Open", - "C18": "14", - "B19": "Resolved", - "C19": "92", - "D19": "17.08", - "B20": "Waiting for Customer", - "C20": "11", - "C21": "229", - "D21": "19.75", - "A22": "Agent-104", - "B22": "Closed", - "C22": "63", - "D22": "15.48", - "B23": "In Progress", - "C23": "32", - "B24": "Open", - "C24": "18", - "B25": "Resolved", - "C25": "97", - "D25": "15.40", - "B26": "Waiting for Customer", - "C26": "13", - "C27": "223", - "D27": "15.43", - "A28": "Agent-105", - "B28": "Closed", - "C28": "70", - "D28": "22.20", - "B29": "In Progress", - "C29": "27", - "B30": "Open", - "C30": "16", - "B31": "Resolved", - "C31": "69", - "D31": "15.21", - "B32": "Waiting for Customer", - "C32": "10", - "C33": "192", - "D33": "18.73", - "A34": "Agent-106", - "B34": "Closed", - "C34": "65", - "D34": "26.15", - "B35": "In Progress", - "C35": "38", - "B36": "Open", - "C36": "20", - "B37": "Resolved", - "C37": "87", - "D37": "21.56", - "B38": "Waiting for Customer", - "C38": "10", - "C39": "220", - "D39": "23.52", - "A40": "Agent-107", - "B40": "Closed", - "C40": "68", - "D40": "18.41", - "B41": "In Progress", - "C41": "35", - "B42": "Open", - "C42": "17", - "B43": "Resolved", - "C43": "106", - "D43": "16.52", - "B44": "Waiting for Customer", - "C44": "11", - "C45": "237", - "D45": "17.26", - "A46": "Agent-108", - "B46": "Closed", - "C46": "65", - "D46": "16.81", - "B47": "In Progress", - "C47": "33", - "B48": "Open", - "C48": "20", - "B49": "Resolved", - "C49": "93", - "D49": "14.93", - "B50": "Waiting for Customer", - "C50": "8", - "C51": "219", - "D51": "15.70", - "B52": "(blank)", - "C52": "243", - "D52": "20.98", - "B53": "Closed", - "C53": "74", - "D53": "24.44", - "B54": "In Progress", - "C54": "32", - "B55": "Open", - "C55": "26", - "B56": "Resolved", - "C56": "99", - "D56": "18.39", - "B57": "Waiting for Customer", - "C57": "12", - "B58": "Grand Total", - "C58": "2000", - "D58": "19.05" - } - }, - "id": "c7d6cdf8-7c66-48f7-b185-21f1c77fa9cb" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the employee data, create a pivot table in the ANSWER tab on A1 with two rows: Department and PerformanceRating. The values should be Count of LastPromotionDate and Average of SalaryUSD. The salary column should be rounded to 2 decimal places, no currency symbol, no thousands separators.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/33a38ce1-f7fb-4488-9c2f-1e2b0d90267e/setup_input_2.xlsx?", - "expected_cells": { - "B2": "1", - "B3": "2", - "B4": "3", - "B5": "4", - "B6": "5", - "B8": "1", - "B9": "2", - "C1": "Count of LastPromotionDate", - "C2": "12", - "C3": "13", - "C4": "10", - "C5": "15", - "C6": "5", - "C8": "1", - "C9": "5", - "D1": "Average of SalaryUSD", - "D2": "84965.91", - "D3": "87028.38", - "D4": "86094.90", - "D5": "78111.64", - "D6": "86015.00", - "D8": "82727.50", - "D9": "78876.00", - "A38": "Grand Total", - "B10": "3", - "B11": "4", - "B12": "5", - "B14": "1", - "B15": "2", - "B16": "3", - "B17": "4", - "B18": "5", - "B20": "1", - "B21": "2", - "B22": "3", - "B23": "4", - "B24": "5", - "B26": "1", - "B27": "2", - "B28": "3", - "B29": "4", - "B30": "5", - "B32": "1", - "B33": "2", - "B34": "3", - "B35": "4", - "B36": "5", - "C10": "4", - "C15": "3", - "C16": "2", - "C17": "2", - "C20": "1", - "C21": "4", - "C22": "6", - "C23": "3", - "C24": "5", - "C26": "10", - "C27": "9", - "C28": "4", - "C29": "6", - "C30": "4", - "C32": "5", - "C33": "2", - "C34": "4", - "C35": "4", - "C36": "1", - "C38": "140", - "D10": "76759.43", - "D11": "78318.00", - "D12": "80036.25", - "D14": "76599.00", - "D15": "93749.17", - "D16": "79929.00", - "D17": "88994.75", - "D18": "102902.00", - "D20": "77878.50", - "D21": "77480.38", - "D22": "84212.67", - "D23": "77508.83", - "D24": "77182.50", - "D26": "56034.08", - "D27": "52096.20", - "D28": "63190.71", - "D29": "60750.00", - "D30": "50299.25", - "D32": "71131.57", - "D33": "89669.50", - "D34": "81155.50", - "D35": "82037.80", - "D36": "61600.50", - "D38": "77770.00" - } - }, - "id": "7598db5d-0fd1-44db-ab1a-593cf026317b" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the EUR values of a company transactions and the FX of the day of the transaction, convert values to USD. Then, create a tab called \"Answer\" and provide the sum all of all amounts in USD in A1.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e491ec81-2ab2-4bee-b646-01c1d7940e31/setup_input_2.xlsx?", - "expected_cells": { - "A1": " 27,301,058.62 " - } - }, - "id": "b4462209-0822-4220-8e7e-a9a2a8e6b58e" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the financial data create 3 scenarios, one for moderate, bull and bear. For the first moderate, consider the CAGR as 8%, GM as 60% and OP as 20%. For the bull, replace the numers for 12%, 65% and 30%. For bear replace for 3%, 55% and 20%. Create a table for each scenario, where rows are 2025 through 2029. Columns should be Year Revenue COGS Operating Expenses EBITDA Net Income. Moderate should start in A1 and end in F6. Bull should start in A9 and end in F14. Bear should start in A17 and end in F22. Assume that in all cases the tax rate is 20% and that there is no D&A expense. CAGR is based of the previous years revenue and all other percentages are based off the current year revenue. All values should be in basic number with thousands separators and 2 decimal places and no dollar signs.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/98bb22e2-1bbd-4a81-8df4-d8665518d6be/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Year", - "A2": "2025", - "A3": "2026", - "A4": "2027", - "A5": "2028", - "A6": "2029", - "A9": "Year", - "B1": "Revenue", - "B2": "108,000,000.00", - "B3": "116,640,000.00", - "B4": "125,971,200.00", - "B5": "136,048,896.00", - "B6": "146,932,807.68", - "B9": "Revenue", - "C1": "COGS", - "C2": "43,200,000.00", - "C3": "46,656,000.00", - "C4": "50,388,480.00", - "C5": "54,419,558.40", - "C6": "58,773,123.07", - "C9": "COGS", - "D1": "Operating Expenses", - "D2": "21,600,000.00", - "D3": "23,328,000.00", - "D4": "25,194,240.00", - "D5": "27,209,779.20", - "D6": "29,386,561.54", - "D9": "Operating Expenses", - "E1": "EBITDA", - "E2": "43,200,000.00", - "E3": "46,656,000.00", - "E4": "50,388,480.00", - "E5": "54,419,558.40", - "E6": "58,773,123.07", - "E9": "EBITDA", - "F1": "Net Income", - "F2": "34,560,000.00", - "F3": "37,324,800.00", - "F4": "40,310,784.00", - "F5": "43,535,646.72", - "F6": "47,018,498.46", - "F9": "Net Income", - "A10": "2025", - "A11": "2026", - "A12": "2027", - "A13": "2028", - "A14": "2029", - "A17": "Year", - "A18": "2025", - "A19": "2026", - "A20": "2027", - "A21": "2028", - "A22": "2029", - "B10": "112,000,000.00", - "B11": "125,440,000.00", - "B12": "140,492,800.00", - "B13": "157,351,936.00", - "B14": "176,234,168.32", - "B17": "Revenue", - "B18": "103,000,000.00", - "B19": "106,090,000.00", - "B20": "109,272,700.00", - "B21": "112,550,881.00", - "B22": "115,927,407.43", - "C10": "39,200,000.00", - "C11": "43,904,000.00", - "C12": "49,172,480.00", - "C13": "55,073,177.60", - "C14": "61,681,958.91", - "C17": "COGS", - "C18": "46,350,000.00", - "C19": "47,740,500.00", - "C20": "49,172,715.00", - "C21": "50,647,896.45", - "C22": "52,167,333.34", - "D10": "33,600,000.00", - "D11": "37,632,000.00", - "D12": "42,147,840.00", - "D13": "47,205,580.80", - "D14": "52,870,250.50", - "D17": "Operating Expenses", - "D18": "20,600,000.00", - "D19": "21,218,000.00", - "D20": "21,854,540.00", - "D21": "22,510,176.20", - "D22": "23,185,481.49", - "E10": "39,200,000.00", - "E11": "43,904,000.00", - "E12": "49,172,480.00", - "E13": "55,073,177.60", - "E14": "61,681,958.91", - "E17": "EBITDA", - "E18": "36,050,000.00", - "E19": "37,131,500.00", - "E20": "38,245,445.00", - "E21": "39,392,808.35", - "E22": "40,574,592.60", - "F10": "31,360,000.00", - "F11": "35,123,200.00", - "F12": "39,337,984.00", - "F13": "44,058,542.08", - "F14": "49,345,567.13", - "F17": "Net Income", - "F18": "28,840,000.00", - "F19": "29,705,200.00", - "F20": "30,596,356.00", - "F21": "31,514,246.68", - "F22": "32,459,674.08" - } - }, - "id": "d4abb58a-47bf-4535-b1ab-d60a4ead37c8" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the FX data and dates, create column that identifies each day as weekday or weekend. Then create a pivot tabe in ANSWER!A3 with daily-average FX rates and filter out weekends. Dates should be YYYY-MM-DD format, FX rate should have 3 decimal places. Verify that values occupy B4-B13, with grand total in B14.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c2ca4cb3-8f15-4a9e-9489-95c67646ed25/setup_input_2.xlsx?", - "expected_cells": { - "A4": "2024-04-01", - "A5": "2024-04-02", - "A6": "2024-04-03", - "A7": "2024-04-04", - "A8": "2024-04-05", - "A9": "2024-04-08", - "B4": "1.020", - "B5": "0.953", - "B6": "0.989", - "B7": "0.046", - "B8": "0.046", - "B9": "0.046", - "A10": "2024-04-09", - "A11": "2024-04-10", - "A12": "2024-04-11", - "A13": "2024-04-12", - "A14": "Grand Total", - "B10": "0.046", - "B11": "0.046", - "B12": "0.046", - "B13": "0.046", - "B14": "0.328" - } - }, - "id": "cea3b19a-6855-45f0-863e-42694346d487" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the global revenue from the company, convert the foreign values into USD and sum all the values, answer on A1 on Answer. Use the conversion rate for 3/15/2023. Before submitting, remove the formula and just put the value. Should be formatted with a thousands separator and two decimal places, no currency symbol.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f199d15d-e8c1-4d34-9823-1b2999a43b36/setup_input_2.xlsx?", - "expected_cells": { - "A1": "1,758,109,357.43" - } - }, - "id": "9a96fc8b-75c9-49dc-bf0b-3e62e36a6ac2" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the gross wages file which includes name, rate and hours worked and a file which includes the federal and state taxes with rates and basis of calculations, calculate the employee payroll tax burden for each employee. Put the answer in the ANSWER tab in a table format which includes columns for the employee name, rate of pay, hours worked, total pay, and employee costs for social security, medicare, workers compensation, unemployment, family medical leave, and CARES (long term disability), and the total employee tax burden. The dollar values use currency type, with 2 decimal places of precision.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/774a3069-410d-4436-aa21-5f733aabaafd/setup_input_2.xlsx?", - "expected_cells": { - "A2": "Kressa", - "A3": "Saumya", - "A4": "Jeane", - "A5": "Sarah", - "A6": "Anaiya", - "A7": "Varun", - "A8": "Sofie", - "B2": "$27.00", - "B3": "$30.50", - "B4": "$27.00", - "B5": "$18.00", - "B6": "$50.00", - "B7": "$52.00", - "B8": "$19.00", - "C2": "40", - "C3": "50", - "C4": "20", - "C5": "20", - "C6": "80", - "C7": "80", - "C8": "20", - "D2": "$1,080.00", - "D3": "$1,525.00", - "D4": "$540.00", - "D5": "$360.00", - "D6": "$4,000.00", - "D7": "$4,160.00", - "D8": "$380.00", - "E2": "$66.96", - "E3": "$94.55", - "E4": "$33.48", - "E5": "$22.32", - "E6": "$248.00", - "E7": "$257.92", - "E8": "$23.56", - "F2": "$15.66", - "F3": "$22.11", - "F4": "$7.83", - "F5": "$5.22", - "F6": "$58.00", - "F7": "$60.32", - "F8": "$5.51", - "G2": "$2.24", - "G3": "$2.80", - "G4": "$1.12", - "G5": "$1.12", - "G6": "$4.48", - "G7": "$4.48", - "G8": "$1.12", - "H2": "$0.32", - "H3": "$0.46", - "H4": "$0.16", - "H5": "$0.11", - "H6": "$1.20", - "H7": "$1.25", - "H8": "$0.11", - "I2": "$5.71", - "I3": "$8.06", - "I4": "$2.85", - "I5": "$1.90", - "I6": "$21.14", - "I7": "$21.99", - "I8": "$2.01", - "J2": "$6.26", - "J3": "$8.85", - "J4": "$3.13", - "J5": "$2.09", - "J6": "$23.20", - "J7": "$24.13", - "J8": "$2.20", - "K2": "$97.16", - "K3": "$136.82", - "K4": "$48.58", - "K5": "$32.76", - "K6": "$356.02", - "K7": "$370.08", - "K8": "$34.52" - } - }, - "id": "ed7985e4-051b-4ca2-8490-e5d755f01c9a" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the information from curves dates, deduplicate records by date-maturity and Format Yield in USD as currency, then retain the latest entry using the As of Date, and create a pivot table on Answer A1 where rows are curvedate, columns are maturity, and values are yields. Format to 2 decimal places and a dollar sign. There should be both grand totals.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/88a420cb-b5d0-47a2-a155-7ba54d4fb6e9/setup_input_2.xlsx?", - "expected_cells": { - "B3": "Maturity", - "A4": "CurveDate", - "B4": "10Y", - "C4": "1Y", - "D4": "2Y", - "E4": "30Y", - "F4": "5Y", - "G4": "Grand Total", - "A5": "2024-01-01", - "B5": "$3.52", - "C5": "$3.73", - "D5": "$3.71", - "E5": "$3.86", - "F5": "$4.71", - "G5": "$19.53", - "A6": "2024-01-02", - "B6": "$3.73", - "C6": "$3.62", - "D6": "$3.79", - "E6": "$4.81", - "F6": "$4.54", - "G6": "$20.49", - "A7": "2024-01-03", - "B7": "$4.28", - "C7": "$4.98", - "D7": "$4.63", - "E7": "$3.70", - "F7": "$4.79", - "G7": "$22.38", - "A8": "2024-01-04", - "B8": "$4.03", - "C8": "$4.84", - "D8": "$3.94", - "E8": "$4.84", - "F8": "$4.49", - "G8": "$22.14", - "A9": "Grand Total", - "B9": "$15.56", - "C9": "$17.17", - "D9": "$16.07", - "E9": "$17.21", - "F9": "$18.53", - "G9": "$84.54" - } - }, - "id": "3506dbf9-71cc-4af5-b2d5-5a994de4e0a4" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the Input data, determine the ticker with the greatest correlation between volume and next day price change.\n- in ANSWER tab put the Ticker in A1 and the correlation in B1\n - use CORREL to determine correlation\n- be sure to first sort the date by ticker Z to A (descending) and then date ascending before calculating next-day price change %\nCorrelation should be rounded to 2 decimal places", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/615426c8-9df7-4ffa-92e9-200134a84da9/setup_input_2.xlsx?", - "expected_cells": { - "A1": "ABC", - "B1": "-0.08" - } - }, - "id": "0b87f523-22b7-4988-a276-8fbdf434eb2c" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the input data\n 1. Which salesperson generated the highest total sales in terms of value? Put their name in ANSWER!A1 and the amount in ANSWER!B1\n 2. How much more sales, in terms of total value, did they generate than the second place salesperon? Put the amount in ANSWER!A2. Format all dollar numbers with a dollar sign and 2 decimal places of precision.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8c2f7db9-b47e-49aa-8c08-86277ec03860/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Carla White", - "A2": "$155.00", - "B1": "$8,758.00" - } - }, - "id": "bab851c8-51e0-4278-b4f3-29575ecae2f1" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the marketing campaign data calculate the cost per conversion within each Channel. List each channel in Answer Column A sorted A-Z. In column B, provide the Campaign which corresponds to the lowest cost per conversion. Finally, in column C provide the cost per conversion for that campaign", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/8a20abce-f84f-46ff-b6dc-1ea06c9b1d86/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Content Marketing", - "A2": "Email", - "A3": "PPC", - "A4": "SEO", - "A5": "Social Media", - "B1": "Harness Frictionless Users", - "B2": "Transform Out-Of-The-Box Schemas", - "B3": "Morph Back-End E-Business", - "B4": "Facilitate Dynamic Channels", - "B5": "Re-Intermediate Cutting-Edge Web-Readiness", - "C1": "6.61", - "C2": "5.99", - "C3": "5.02", - "C4": "5.49", - "C5": "5.68" - } - }, - "id": "1e4ede56-163f-4815-b634-7946cf9e60c0" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the project management export, figure out who is the most accurate estimator (the most projects completed exactly on predicted time) and who is the most efficient employee (most projects completed under estimated time). Put your answer for most accurate in ANSWER!A1 and most efficient in ANSWER!A2. Don't consider unfinished projects.\n", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/80869e7a-34bd-42fb-a010-ee43edf8399a/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Omar Donovan", - "A2": "Omar Donovan" - } - }, - "id": "8499e399-48e6-4603-bd96-9b45f5aabed0" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the provided metrics from my company, calculate the average opening ARR between months 2023-01 and 2024-01. Assume all ARR comes in the start of the month.\nPut your answer in ANSWER!A1. No thousands separators and dollar signs, two decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/04601676-b195-4b34-82f4-0b08a17a5a75/setup_input_2.xlsx?", - "expected_cells": { - "A1": "1876764.43" - } - }, - "id": "304c4d2c-72a4-4c1d-97ce-6e1b32d9b447" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the quarterly revenue data in the INPUTS tab project the quartely revenue for 2025 using an average of the growth for the corresponding quarters from prior years. Sum those to find the total. Place the quaterly values (Q1-Q4) in ANSWER cell A1 to A4. Place the total in B1. All numbers should be without thousands separators, with no dollar sign and no decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/14114f14-d4e5-4b47-97b1-dc654351c014/setup_input_2.xlsx?", - "expected_cells": { - "A1": "991105", - "A2": "1044283", - "A3": "1095308", - "A4": "1169204", - "B1": "4299900" - } - }, - "id": "3c8d3cfb-ce35-45fb-828b-b4e2c6209435" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the quarterly revenue data in the INPUTS tab, calculate the total Annual revenue for 2022, 2023 and 2024 and use these data points to calculate CAGR in ANSWER tab cell A1. Calculate which year has the highest revenue growth % and place the value of this revenue growth % in cell B1 of the ANSWER tab. Both numbers should format as percentages with two decimal places of precision.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/923e579f-2f38-493d-a85e-29cf4524e4c6/setup_input_2.xlsx?", - "expected_cells": { - "A1": "16.74%", - "B1": "27.80%" - } - }, - "id": "764ca583-7091-4b7f-8663-5e996db26a2c" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the real estate data, which city has the highest average monthly growth rate in total list price? Use ListDate to determine the month each ID is in.\nProvide the city name in the ANSWER tab in cell A1. Provide the average monthly growth rate, formatted as a percent with two decimal places, for that city in B1", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/4e26d81b-0be1-4fc1-b32f-cc043cb5bb37/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Star City", - "B1": "39.11%" - } - }, - "id": "9f71aa71-07f7-4421-8757-bbccbeab0984" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the real estate data, which city has the highest count of sales (status sold) of houses with greater than 4 bed in the last calender year? Assume the latest date in the file is the current date\nProvide the city name in the ANSWER tab in cell A1. Provide the number of houses sold with more than 4 beds in that year and city in B1", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/87f0f0e4-abc3-4075-a181-0d18fd5227bc/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Gotham", - "B1": "4" - } - }, - "id": "d5d7cc74-c47e-4988-9c8c-8bc08e19f7af" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the results of budget vs. actual find the difference, identify if its favorable or not. Place the value of the smallest absolute difference on ANSWER tab B1 and the name of the cost center on A1. The value should have no decimal points and no dollar sign.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/739c14d9-2651-43e8-a4a4-dcd2c876c932/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Finance", - "B1": "1,000" - } - }, - "id": "6ee1f05a-9d68-4efc-b807-2efdb8d0c74b" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the salaries and the country's tax, The goal is to identify the amount received by employees by country. Determine the value of the sum of net pay by country. Place the value of greatest summed net pay ANSWER!A1. Number should have no thousands seperator, 3 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/c707579d-ead6-4623-a8ea-68d5dc3e2a0d/setup_input_2.xlsx?", - "expected_cells": { - "A1": "21680.086" - } - }, - "id": "2774e5e5-10ec-42b1-84cb-c2d4819a5342" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the Sales and COGS projection for the next 2 years (2024-2025) predict into 2026 using 3 scenarios, Rank in ANSWER from A1 to A3 the scenerio with the highest to lowest COGS in dollars. Scenario 1: rev growth 5%, gross margin 18%. Scenario 2: rev growth 12%, gross margin 5%. Scenario 3: rev growth 15%, gross margin 2%. All growth percentages should be applied to the previous year", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/630f74b1-6855-46fb-acc7-d01e9a0581da/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Scenario 3", - "A2": "Scenario 2", - "A3": "Scenario 1" - } - }, - "id": "d2c56ee0-7863-45be-b503-eb1639454629" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the set of past payment transactions, identify the vendors where a 1099 is required to be delivered based on w-9 reported entity (see \"Input 1\" tab). In the ANSWER tab list the vendors in order with the amounts to be reported with two columns: Vendor, 1099 Report amount. Have the vendors be ascending by #. Omit the vendors that don't need to report 1099. Use Currency input type, rounded to 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/0af775fe-a6e7-47d6-968c-e8a7b089162b/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Vendor", - "A2": "vendor 18", - "A3": "vendor 15", - "A4": "vendor 19", - "A5": "vendor 1", - "A6": "vendor 14", - "A7": "vendor 9", - "A8": "vendor 8", - "A9": "vendor 11", - "B1": "1099 Report amount", - "B2": "$4.00", - "B3": "$48.00", - "B4": "$81.00", - "B5": "$92.60", - "B6": "$177.00", - "B7": "$866.00", - "B8": "$1,661.00", - "B9": "$2,615.00", - "A10": "vendor 4", - "A11": "vendor 5", - "B10": "$6,446.00", - "B11": "$6,519.00" - } - }, - "id": "077926f8-6e93-406b-9eb9-b44907173c74" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the set of past transactions (inflows into our account), project our breakeven month given our implied monthly inflow growth rate (as determined by taking a straight average of the monthly growth rates observed for the five month-over-month periods observable in the inflow data) and fixed expenses of 150k per mo. Produce your answer as a value in the cell A1 of a sheet in the spreadsheet in the format YYYY-MM. Ensure there is nothing else in the ANSWER tab. You may create as many additional sheets as you need to conduct your analysis. If you build a model, put it in its own tab separate from the raw data.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35571ea4-82a5-4ca7-ab36-d679b523282e/setup_input_2.xlsx?", - "expected_cells": { - "A1": "2025-02" - } - }, - "id": "f1afee7f-df70-4a11-a65e-767a718f2117" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab populate column G by using a vlookup function to look up the month text value in the REFERENCE tab corresponding to the numeric month value from column A in the INPUTS tab. In the ANSWER tab create a pivot table on A3 with the Platform field as a row and Month field as a column. Months should be sorted alphabetically. Sum each of the Posts, Likes, Comments, and Shares for each month as rows too.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b105e2c6-6e65-4715-9c42-f9015e5d5751/setup_input_2.xlsx?", - "expected_cells": { - "A4": "Platform", - "A5": "Facebook", - "A9": "Instagram", - "B4": "Values", - "B5": "Sum of Posts", - "B6": "Sum of Likes", - "B7": "Sum of Shares", - "B8": "Sum of Comments", - "B9": "Sum of Posts", - "C4": "August", - "C5": "66", - "C6": "3899", - "C7": "618", - "C8": "241", - "C9": "54", - "D4": "July", - "D5": "65", - "D6": "4008", - "D7": "749", - "D8": "260", - "D9": "58", - "E4": "June", - "E5": "54", - "E6": "3236", - "E7": "560", - "E8": "296", - "E9": "70", - "F4": "November", - "F5": "39", - "F6": "2066", - "F7": "430", - "F8": "209", - "F9": "52", - "G4": "October", - "G5": "68", - "G6": "3048", - "G7": "773", - "G8": "271", - "G9": "54", - "H4": "September", - "H5": "64", - "H6": "2918", - "H7": "769", - "H8": "262", - "H9": "57", - "I4": "Grand Total", - "I5": "356", - "I6": "19175", - "I7": "3899", - "I8": "1539", - "I9": "345", - "A13": "LinkedIn", - "A17": "Twitter", - "A21": "Grand Total", - "B10": "Sum of Likes", - "B11": "Sum of Shares", - "B12": "Sum of Comments", - "B13": "Sum of Posts", - "B14": "Sum of Likes", - "B15": "Sum of Shares", - "B16": "Sum of Comments", - "B17": "Sum of Posts", - "B18": "Sum of Likes", - "B19": "Sum of Shares", - "B20": "Sum of Comments", - "B21": "Sum of Posts", - "B22": "Sum of Likes", - "B23": "Sum of Shares", - "B24": "Sum of Comments", - "C10": "3231", - "C11": "541", - "C12": "226", - "C13": "61", - "C14": "2923", - "C15": "668", - "C16": "295", - "C17": "60", - "C18": "3290", - "C19": "671", - "C20": "295", - "C21": "241", - "C22": "13343", - "C23": "2498", - "C24": "1057", - "D10": "3025", - "D11": "700", - "D12": "232", - "D13": "60", - "D14": "3122", - "D15": "589", - "D16": "314", - "D17": "54", - "D18": "2950", - "D19": "454", - "D20": "271", - "D21": "237", - "D22": "13105", - "D23": "2492", - "D24": "1077", - "E10": "4195", - "E11": "834", - "E12": "317", - "E13": "45", - "E14": "2291", - "E15": "451", - "E16": "185", - "E17": "63", - "E18": "3366", - "E19": "797", - "E20": "278", - "E21": "232", - "E22": "13088", - "E23": "2642", - "E24": "1076", - "F10": "2860", - "F11": "514", - "F12": "249", - "F13": "61", - "F14": "3116", - "F15": "708", - "F16": "246", - "F17": "63", - "F18": "2714", - "F19": "581", - "F20": "243", - "F21": "215", - "F22": "10756", - "F23": "2233", - "F24": "947", - "G10": "2993", - "G11": "623", - "G12": "265", - "G13": "70", - "G14": "3596", - "G15": "716", - "G16": "336", - "G17": "58", - "G18": "2382", - "G19": "578", - "G20": "293", - "G21": "250", - "G22": "12019", - "G23": "2690", - "G24": "1165", - "H10": "3409", - "H11": "401", - "H12": "313", - "H13": "65", - "H14": "3271", - "H15": "771", - "H16": "278", - "H17": "51", - "H18": "2156", - "H19": "431", - "H20": "312", - "H21": "237", - "H22": "11754", - "H23": "2372", - "H24": "1165", - "I10": "19713", - "I11": "3613", - "I12": "1602", - "I13": "362", - "I14": "18319", - "I15": "3903", - "I16": "1654", - "I17": "349", - "I18": "16858", - "I19": "3512", - "I20": "1692", - "I21": "1412", - "I22": "74065", - "I23": "14927", - "I24": "6487" - } - }, - "id": "36e43b84-e583-4f02-a160-74049bcab901" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab produce an ANSWER tab:\n1. for each row categorize them into \"LOW\" or \"HIGH\" engagements days.\n - start with the ratio of likes to posts, comments to posts, shares to posts\n - normalize each of these against overall ratios (eg (Value - MIN(ratio)) / (MAX(ratio) - MIN(ratio)))\n - produce an engagement metric for each day by averaging the three normalized metrics\n - if this metric is >=0.6 categorize row as \"HIGH\" if its <= 0.3 its \"LOW\"\n2. For each platform, compute the ratio of High / Low days\n\nProduce a table in ANSWER. Where row 1 is the header. column A is 'Platform' and column B is the ratio of high to low days 'RATIO OF HIGH / LOW'.\n - Twitter should be in A2, ratio in B2\n - Facebook should be in A3, ratio in B3\n - Instagram should be in A4, ratio in B4\n - LinkedIn should be in A5, ratio in B5\nRatios in the final table should have 2 decimal places", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/86f51853-bb54-4b87-8cf9-77473610a0ad/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Platform", - "A2": "Twitter", - "A3": "Facebook", - "A4": "Instagram", - "A5": "LinkedIn", - "B1": "RATIO OF HIGH / LOW", - "B2": "1.88", - "B3": "2.64", - "B4": "3.17", - "B5": "2.05" - } - }, - "id": "871eefda-4c69-4e3a-abb4-d4215f4a6849" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the social media engagement data in the INPUTS tab, create a pivot table in ANSWER tab. In the ANSWER tab, create a filter using the Platform field and filter for Facebook and Instagram . Include the Date field on the month level as a row (formated by 3 letters) and include values from the Posts, Likes, Shares, and Comments fields summarized by SUM. Columns should be named Sum of X, where X is the field name. Verify that columns occupy Row 3, numbers occupy B2 to E10, with a Grand Total in row 10.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9c172ee4-5b31-4b7c-9c48-fb0cd1e2c18a/setup_input_2.xlsx?", - "expected_cells": { - "A4": "Jun", - "A5": "Jul", - "A6": "Aug", - "A7": "Sep", - "A8": "Oct", - "A9": "Nov", - "B3": "Sum of Posts", - "B4": "124", - "B5": "123", - "B6": "120", - "B7": "121", - "B8": "122", - "B9": "91", - "C3": "Sum of Likes", - "C4": "7431", - "C5": "7033", - "C6": "7130", - "C7": "6327", - "C8": "6041", - "C9": "4926", - "D3": "Sum of Comments", - "D4": "613", - "D5": "492", - "D6": "467", - "D7": "575", - "D8": "536", - "D9": "458", - "E3": "Sum of Shares", - "E4": "1394", - "E5": "1449", - "E6": "1159", - "E7": "1170", - "E8": "1396", - "E9": "944", - "A10": "Grand Total", - "B10": "701", - "C10": "38888", - "D10": "3141", - "E10": "7512" - } - }, - "id": "c437f78d-6382-43b2-b857-153db6efa1c9" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name \"Month\" in cell A1, \"Year\" in cell B1, \"Total Monthly Unique Users\" in cell C1, \"Total Monthly Page Views\" in cell D1, and \"Avg Monthly Bounce Rate (%)\" in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the \"Year\" column starting in cells B14 to B17 with 2024. Calculate the \"Total Monthly Unique Users\" from cells C2 to C13, \"Total Monthly Page Views\" from cells D2 to D13, and \"Avg Monthly Bounce Rate (%)\" from cells E2 to E13. The growth values should be percentages with 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/35f323aa-bf61-4a0a-bb15-3cc9f3c00a0f/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.68%", - "E3": "43.14%", - "E4": "43.77%", - "E5": "43.27%", - "E6": "44.90%", - "E7": "44.03%", - "E8": "44.06%", - "E9": "43.97%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "E10": "44.37%", - "E11": "44.10%", - "E12": "46.70%", - "E13": "48.06%", - "E14": "45.94%", - "E15": "45.48%", - "E16": "44.06%", - "E17": "43.93%" - } - }, - "id": "67a53dc7-e07e-46df-a4c8-ecbcad1dd0bc" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS tab, parse the Date column into a Year column and a month column. In the ANSWER tab, create a header name Month in cell A1, Year in cell B1, Total Monthly Unique Usersi n cell C1, Total Monthly Page Views in cell D1, and Avg Monthly Bounce Rate (%) in cell E1. Populate the Month column starting in cells A2 to A13 starting from 1 to 12 where Year is 2023. Populate the Year column starting in cells B2 to B13 with 2023. Populate the Month column starting in cells A14 to A17 starting from 1 to 4. Populate the Year column starting in cells B14 to B17 with 2024. Calculate the Total Monthly Unique Users from cells C2 to C17, Total Monthly Page Views from cells D2 to D17, and Ave Monthly Bounce Rate (%) from cells E2 to E17. For each cell from C18 to C25, D18 to D25, and E18 to E25 calculate the average based on the previous 6 cells in order to forecast what the subsequent Total Monthly Unique Users, Total Monthly Page Views, and Ave Monthly Bounce Rate would be for the next 8 months. The users should be rounded to 0 decimal places, rate to 1 decimal place.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1ad92615-f52e-482d-8e8f-bf74079c5594/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Month", - "A2": "1", - "A3": "2", - "A4": "3", - "A5": "4", - "A6": "5", - "A7": "6", - "A8": "7", - "A9": "8", - "B1": "Year", - "B2": "2023", - "B3": "2023", - "B4": "2023", - "B5": "2023", - "B6": "2023", - "B7": "2023", - "B8": "2023", - "B9": "2023", - "C1": "Total Monthly Unique Users", - "C2": "21928", - "C3": "19296", - "C4": "21453", - "C5": "20987", - "C6": "21944", - "C7": "21024", - "C8": "20875", - "C9": "21495", - "D1": "Total Monthly Page Views", - "D2": "34560", - "D3": "32162", - "D4": "35497", - "D5": "34263", - "D6": "35875", - "D7": "34896", - "D8": "36121", - "D9": "35382", - "E1": "Avg Monthly Bounce Rate (%)", - "E2": "44.7%", - "E3": "43.1%", - "E4": "43.8%", - "E5": "43.3%", - "E6": "44.9%", - "E7": "44.0%", - "E8": "44.1%", - "E9": "44.0%", - "A10": "9", - "A11": "10", - "A12": "11", - "A13": "12", - "A14": "1", - "A15": "2", - "A16": "3", - "A17": "4", - "A18": "5", - "A19": "6", - "A20": "7", - "A21": "8", - "A22": "9", - "A23": "10", - "A24": "11", - "A25": "12", - "B10": "2023", - "B11": "2023", - "B12": "2023", - "B13": "2023", - "B14": "2024", - "B15": "2024", - "B16": "2024", - "B17": "2024", - "B18": "2024", - "B19": "2024", - "B20": "2024", - "B21": "2024", - "B22": "2024", - "B23": "2024", - "B24": "2024", - "B25": "2024", - "C10": "21054", - "C11": "21153", - "C12": "20957", - "C13": "21493", - "C14": "21238", - "C15": "20205", - "C16": "21617", - "C17": "21094", - "C18": "21095", - "C19": "21015", - "C20": "21141", - "C21": "21311", - "C22": "21064", - "C23": "21182", - "C24": "21209", - "C25": "21238", - "D10": "34959", - "D11": "36723", - "D12": "34626", - "D13": "35262", - "D14": "36110", - "D15": "33484", - "D16": "34397", - "D17": "34643", - "D18": "34237", - "D19": "33819", - "D20": "33547", - "D21": "33832", - "D22": "33424", - "D23": "33162", - "D24": "33042", - "D25": "32929", - "E10": "44.4%", - "E11": "44.1%", - "E12": "46.7%", - "E13": "48.1%", - "E14": "45.9%", - "E15": "45.5%", - "E16": "44.1%", - "E17": "43.9%", - "E18": "43.1%", - "E19": "41.8%", - "E20": "41.3%", - "E21": "40.4%", - "E22": "39.7%", - "E23": "38.7%", - "E24": "37.9%", - "E25": "37.2%" - } - }, - "id": "9ba9631e-8560-4dc4-a285-8d186715a542" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nGiven the web traffic data in the INPUTS, produce an ANSWER tab.\n1. Which day had the highest number of unique visitors? put this value in ANSWER!A1 (YYYY-MM-DD)\n2. On this day, what was the bounce rate? put this value in ANSWER!A2 (two decimal places)\n3. What is the correlation of bounce rate to unique visitors as measured by coefficient of determination. Put your answer in ANSWER!A3 (5 decimal places)", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/d8e5298e-68fb-4d8a-8c56-62adf2cde1ff/setup_input_2.xlsx?", - "expected_cells": { - "A1": "2023-04-30", - "A2": "0.48", - "A3": "0.00296" - } - }, - "id": "0721e45c-8677-4aef-b9b2-65ad6bea1392" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In column I extract the numeric month based on the date value in column b. In column J extract the numeric year based on the date value in column B. In the ANSWER tab, A1, create a pivot table with the ProductID field in the row, the Year field in the column, and Sales field as the value. In cell D2 create a field called \"Rank based on 2024 Sales\" and rank each ProductID based on 2024 sales with 1 being the highest sales. Check that numerical values occupy D3 to D22.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/7c134f47-c8a7-4a68-ba47-4e783fea21ba/setup_input_2.xlsx?", - "expected_cells": { - "D2": "Rank Based on 2024 Sales", - "D3": "4", - "D4": "3", - "D5": "7", - "D6": "17", - "D7": "12", - "D8": "1", - "D9": "18", - "D10": "8", - "D11": "6", - "D12": "14", - "D13": "5", - "D14": "13", - "D15": "9", - "D16": "20", - "D17": "11", - "D18": "10", - "D19": "19", - "D20": "15", - "D21": "2", - "D22": "16" - } - }, - "id": "60ee8e1a-6d95-4ff5-bf80-35f899ea7277" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn the INPUTS tab, add a Sales field to column H and calculate the Sales amount. In the ANSWER tab starting in cell A1, create a pivot table with the Region field in the row, the Year field in the column, and Sales field as the value. Calculate the year of year growth in column D called \"YoY Growth\", make the values in this column percent data types with two decimal places. All other values should format as numbers with no thousands separators, no dollar signs and 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/ba2d7ea0-9722-48fa-b6de-fc0f8e9c263c/setup_input_2.xlsx?", - "expected_cells": { - "A2": "Region", - "A3": "Central", - "A4": "East", - "A5": "North", - "A6": "South", - "A7": "West", - "B2": "2023", - "B3": "351351.87", - "B4": "364129.72", - "B5": "377252.87", - "B6": "396259.93", - "B7": "395672.78", - "C2": "2024", - "C3": "368644.54", - "C4": "399862.83", - "C5": "364113.70", - "C6": "345896.80", - "C7": "393456.82", - "D2": "YoY Growth", - "D3": "4.92%", - "D4": "9.81%", - "D5": "-3.48%", - "D6": "-12.71%", - "D7": "-0.56%" - } - }, - "id": "20507108-4188-402e-8cfe-2354309bad6c" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nIn your ANSWER tab, have A1 be a dropdown selector for the different catagories of spending from the \"RAW_INFO\" sheet, and B1 be the total spend within that catagory. The spend should be formatted in Accounting form \"$ (...)\". Select the value of the dropdown as Shopping.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/9ef22631-f42b-4c0d-b987-8d78181feb03/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Shopping", - "B1": "$ (16,401.91)" - } - }, - "id": "a0c8e617-aa64-4e7a-8dd5-8878684720b2" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheet LOAN_AMORT contains the following columns: Date, Loan Issuance, Payment, Principal, and Interest. In each column you will see the cash flows in such category over time, as detailed by the date column. By summing the net loan cash flows for each monthly period, determine what the effective annual interest rate was on the loan in the percentage format with two decimals (e.g., 7.43%), which was fully paid off via the last payment made on 12/31/2029, and place the answer in ANSWER!A1; nothing else in ANSWER.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e7681ead-fa36-408a-8f3c-5e5165a7b610/setup_input_2.xlsx?", - "expected_cells": { - "A1": "6.17%" - } - }, - "id": "2b2ab7bf-edb8-4d11-b8fb-0fea1799aa59" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheet REV_QTR lists quarterly revenue from Q1-2022 through Q1-2025. Compute the compound annual growth rate between those two points. Enter the result in ANSWER!A1 formatted as a percent with two decimal places. Assume an even period between quarters.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/852a6a9e-7e9f-4563-8298-20e80ee0a66a/setup_input_2.xlsx?", - "expected_cells": { - "A1": "24.20%" - } - }, - "id": "5b58a0f7-dbdb-4f56-abc3-08640052af3a" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheets provided: MULTI_CCY (cash movements) and FX (daily USD rates). Add a column in MULTI_CCY converting every amount to USD by matching date and currency. Sum all USD-equivalent amounts. Put that single total in ANSWER!A1; nothing else in ANSWER. Round to 2 decimal places.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/1c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?", - "expected_cells": { - "A1": "$316,309.56" - } - }, - "id": "371507f7-721a-457e-9618-bc8fba1b909b" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSheets: HIST_REV (36-month history) and SCENARIOS (base, bull, bear monthly growth rates). Build a 24-month forecast under each scenario by applying the monthly growth rate to the monthly revenue in 2024-12 and then to each subsequent monthly revenue amount thereafter. Determine the following: base-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bull-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. bear-case calendar month (YYYY-MM) when monthly revenue first exceeds $20 million. Place base-case month in ANSWER!A1, bull-case month in ANSWER!A2, and bear-case month in ANSWER!A3", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/e83487c6-3a71-409a-9da3-8b0b19bb93f3/setup_input_2.xlsx?", - "expected_cells": { - "A1": "2026-03", - "A2": "2025-09", - "A3": "2026-12" - } - }, - "id": "4a081ed2-532c-4895-a034-ab0076927c7c" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nSum the total amount for each currency from data in March 2025 and list in ANSWER column A the abbreviation of the currencies with the most to least amount. In column B provide the corresponding amount. Use two decimal places of precision.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/2c4e710d-d430-4cfd-a3fb-f2d0b6db38b6/setup_input_3.xlsx?", - "expected_cells": { - "A1": "JPY", - "A2": "USD", - "A3": "EUR", - "A4": "GBP", - "B1": "3500000.00", - "B2": "64351.25", - "B3": "43501.00", - "B4": "15000.25" - } - }, - "id": "1b737be7-eeb6-45c3-88d8-4bc3bbc7a008" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nThis spreadsheet contains individual customer IDs in column A, their signup date in column B, their churn date in column C, and other data in columns D and beyond.\n\nUsing this data and assuming the date is 12/31/24, determine the blended average annual churn rates for those customer cohorts who signed up as customers in 2022 and separately for those who signed up in 2023. Place the answers on the ANSWER tab in cells A1 and B1, respectively, formatted as a percent with two decimals.\n\nIn a given year, the annual churn rate is defined as the number of customers who churned in such year divded by the total number of customers who were active in that year. The blended average annual churn rate is the straight average of the observable annual churn rates. For the avoidance of doubt, the average excludes any churn rates for years prior to the origin of the customer cohort (e.g., the annual churn rates factored into the blended annual average churn rate for the 2023 cohort of customers excludes the activity in such cohort in years prior to its existence (i.e., 2022 and prior)).", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/979a71c1-3c88-4781-9d46-0389d89f92e0/setup_input_2.xlsx?", - "expected_cells": { - "A1": "1.07%", - "A2": "1.73%" - } - }, - "id": "31508ec6-c993-4e00-b70c-093dba016fcc" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nWhich city has the highest 2023 quarterly CAGR at the end of 2023. Place the name of the city in ANSWER!A1.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/b6fa9285-4d72-47f5-99e1-93b21c9a5001/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Central City" - } - }, - "id": "133ef005-207c-490f-b55c-7734fd1678da" - }, - { - "env": { - "name": "hud-remote-browser" - }, - "scenario": "remote-browser:sheet-from-file", - "args": { - "prompt": "All solutions should be put in the sheet called \"ANSWER\". In the answer sheet, all dates should use the American standard format MM/DD/YYYY with no leading zero. All numbers should use the format and decimal place precision given in the input sheets (e.g., with or without a thousands separator should depend on the inputs), unless specified otherwise.\n\nWork in sheet RAW_TRANSACTIONS. Delete exact duplicate rows (all-column match). Convert every value in the Date column to ISO YYYY-MM-DD. Copy the header \u201cDate\u201d plus the cleaned, unique dates into column A of a sheet named ANSWER (no blanks, descending order not required). No other content may appear in ANSWER. Sort by date. All amounts should have 2 decimal places, no dollar sign and no thousands separators.", - "file_url": "https://gahludmjcsmszgyufydt.supabase.co//storage/v1/object/public/sheetbench/f24e4430-c965-4bcc-b56d-a77f9ed9eb7f/setup_input_2.xlsx?", - "expected_cells": { - "A1": "Date", - "A2": "2025-01-05", - "A3": "2025-01-12", - "A4": "2025-01-15", - "A5": "2025-01-20", - "A6": "2025-01-25", - "A7": "2025-01-30", - "A8": "2025-02-05", - "A9": "2025-02-08", - "B1": "Description", - "B2": "Membership Fee", - "B3": "Project Income", - "B4": "Interest", - "B5": "Event Revenue", - "B6": "Refund", - "B7": "Consulting Fee", - "B8": "Website Hosting", - "B9": "Maintenance", - "C1": "Amount", - "C2": "-250.00", - "C3": "5600.00", - "C4": "350.00", - "C5": "7000.00", - "C6": "-5000.00", - "C7": "7800.00", - "C8": "-99.99", - "C9": "-750.25", - "D1": "Currency", - "D2": "USD", - "D3": "USD", - "D4": "USD", - "D5": "USD", - "D6": "USD", - "D7": "USD", - "D8": "USD", - "D9": "USD", - "A10": "2025-02-15", - "A11": "2025-02-18", - "A12": "2025-02-20", - "A13": "2025-02-25", - "A14": "2025-02-28", - "A15": "2025-03-01", - "A16": "2025-03-05", - "A17": "2025-03-10", - "A18": "2025-03-15", - "A19": "2025-03-20", - "A20": "2025-03-25", - "B10": "Invoice Payment", - "B11": "Equipment Purchase", - "B12": "Software License", - "B13": "Bonus", - "B14": "Travel Expenses", - "B15": "Subscription", - "B16": "Advertising", - "B17": "Office Supplies", - "B18": "Utilities", - "B19": "Legal Fees", - "B20": "Marketing", - "C10": "2500.00", - "C11": "-3600.00", - "C12": "-3000.00", - "C13": "4800.00", - "C14": "-1750.00", - "C15": "-1200.00", - "C16": "-2450.50", - "C17": "-450.75", - "C18": "-899.00", - "C19": "-4000.00", - "C20": "-2200.00", - "D10": "USD", - "D11": "USD", - "D12": "USD", - "D13": "USD", - "D14": "USD", - "D15": "USD", - "D16": "USD", - "D17": "USD", - "D18": "USD", - "D19": "USD", - "D20": "USD" - } - }, - "id": "eb410896-3e1e-4491-9460-061579b65c6f" - } -] \ No newline at end of file From b1c91b58d88bcf611b9e534808b50a120a1541de Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 16:47:03 -0800 Subject: [PATCH 26/27] format --- hud/environment/environment.py | 4 +--- hud/environment/tests/test_environment.py | 2 +- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/hud/environment/environment.py b/hud/environment/environment.py index e14357ca0..4ed44b325 100644 --- a/hud/environment/environment.py +++ b/hud/environment/environment.py @@ -408,9 +408,7 @@ async def _env_list_tools(self) -> list[mcp_types.Tool]: """Return all tools including those from connectors.""" return self._router.tools - async def _env_call_tool( - self, name: str, arguments: dict[str, Any] | None = None - ) -> list[Any]: + async def _env_call_tool(self, name: str, arguments: dict[str, Any] | None = None) -> list[Any]: """Route tool calls through our router (handles both local and connector tools).""" result = await self._execute_tool(name, arguments or {}) return result.content or [] diff --git a/hud/environment/tests/test_environment.py b/hud/environment/tests/test_environment.py index 96d9cbb24..60b544e75 100644 --- a/hud/environment/tests/test_environment.py +++ b/hud/environment/tests/test_environment.py @@ -326,4 +326,4 @@ def test_setup_handlers_registers_custom_handlers(self) -> None: assert hasattr(env, "_env_list_tools") assert hasattr(env, "_env_call_tool") assert callable(env._env_list_tools) - assert callable(env._env_call_tool) \ No newline at end of file + assert callable(env._env_call_tool) From cd0cc40f2b7120b516486a01614ca65b7b1dc919 Mon Sep 17 00:00:00 2001 From: lorenss-m Date: Thu, 8 Jan 2026 17:49:56 -0800 Subject: [PATCH 27/27] provider fix --- hud/agents/__init__.py | 2 +- hud/agents/resolver.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/hud/agents/__init__.py b/hud/agents/__init__.py index b9be1b6d2..03f9512a1 100644 --- a/hud/agents/__init__.py +++ b/hud/agents/__init__.py @@ -54,7 +54,7 @@ def create_agent(model: str, **kwargs: Any) -> MCPAgent: # Determine provider: from gateway info, or infer from agent class if gateway_info: - provider = gateway_info.get("provider", "openai") + provider = gateway_info.get("provider") or "openai" else: # Map agent class to provider for known types from hud.agents.claude import ClaudeAgent diff --git a/hud/agents/resolver.py b/hud/agents/resolver.py index b2d01d7cd..80351800f 100644 --- a/hud/agents/resolver.py +++ b/hud/agents/resolver.py @@ -60,7 +60,7 @@ def resolve_cls(model: str) -> tuple[type[MCPAgent], dict[str, Any] | None]: # Gateway lookup for m in _fetch_gateway_models(): if model in (m.get("id"), m.get("name"), m.get("model")): - provider = m.get("provider", "openai_compatible").lower() + provider = (m.get("provider") or "openai_compatible").lower() agent_str = _PROVIDER_TO_AGENT.get(provider, provider) try: return AgentType(agent_str).cls, m