From 28175e022d29e5630a75dbbf25931767d237d9c4 Mon Sep 17 00:00:00 2001 From: Naseem AlNaji Date: Thu, 9 Jul 2026 15:18:57 -0400 Subject: [PATCH] test(community-v3): cover OpenAPI and other runtime-state tools The community test suite previously exercised only plain function tools, which deep-copy cleanly. That gap let the context-injection regression fixed in #38 ship green: FastMCP v3 tools that hold live runtime state (an httpx client on OpenAPI-generated tools, a client factory on proxy tools, a sub-server reference on mounted tools) failed to deep-copy and silently lost context injection. Add end-to-end coverage that drives the real middleware dispatch against those tool types, entirely in-process via httpx.MockTransport (no network): - tests/test_utils/community_openapi_server.py: factories for OpenAPI, proxy, and mounted FastMCP v3 servers, plus a multi-endpoint OpenAPI spec. - tests/community/test_community_v3_openapi.py: tools/list injects context on every tool, no copy failure is logged, cached tools are not mutated, tools/call strips context and captures it as intent without leaking it downstream, HTTP errors are captured, the tools/list event serializes OpenAPI tools, and get_more_tools coexists with generated tools. - tests/community/test_community_v3_runtime_state_tools.py: parametrized guard over openapi/proxy/mounted asserting context injection with no copy failures. All modules are FastMCP v3-only and skip cleanly when FastMCP is absent or on the v2 compatibility matrix. Verified on FastMCP 3.0.0b1 and 3.4.4. --- tests/community/test_community_v3_openapi.py | 231 ++++++++++++++++++ .../test_community_v3_runtime_state_tools.py | 79 ++++++ tests/test_utils/community_openapi_server.py | 186 ++++++++++++++ 3 files changed, 496 insertions(+) create mode 100644 tests/community/test_community_v3_openapi.py create mode 100644 tests/community/test_community_v3_runtime_state_tools.py create mode 100644 tests/test_utils/community_openapi_server.py diff --git a/tests/community/test_community_v3_openapi.py b/tests/community/test_community_v3_openapi.py new file mode 100644 index 0000000..1caa96e --- /dev/null +++ b/tests/community/test_community_v3_openapi.py @@ -0,0 +1,231 @@ +"""End-to-end tests for AgentCat against OpenAPI-generated FastMCP v3 servers. + +These drive the real middleware dispatch (``tools/list`` and ``tools/call``) +through a FastMCP ``Client`` against tools that hold a live ``httpx.AsyncClient`` +(mock transport, no network). This is the coverage that was missing when the +``copy.deepcopy(tool)`` regression (PR #38) shipped: the whole suite only ever +exercised plain function tools, which deep-copy cleanly. +""" + +import copy +import time +from unittest.mock import patch + +import pytest + +from agentcat import AgentCatOptions, track +from agentcat.modules.event_queue import EventQueue, set_event_queue +from agentcat.modules.overrides.community_v3 import middleware as v3_middleware + +from ..test_utils.community_client import create_community_test_client +from ..test_utils.community_openapi_server import ( + HAS_FASTMCP_V3, + OPENAPI_TOOL_NAMES, + create_community_openapi_server, +) + +pytestmark = pytest.mark.skipif( + not HAS_FASTMCP_V3, + reason="Requires FastMCP v3+ (OpenAPI middleware path)", +) + +CONTEXT_DESC = "Why are you making this tool call?" + + +@pytest.fixture +def captured_events(): + """Swap in a mock-backed event queue and collect published events.""" + from agentcat.modules.event_queue import event_queue as original_queue + from unittest.mock import MagicMock + + events: list = [] + mock_api_client = MagicMock() + + def capture_event(publish_event_request): + events.append(publish_event_request) + + mock_api_client.publish_event = MagicMock(side_effect=capture_event) + set_event_queue(EventQueue(api_client=mock_api_client)) + try: + yield events + finally: + set_event_queue(original_queue) + + +def _options(**overrides) -> AgentCatOptions: + base = dict(enable_tracing=True, custom_context_description=CONTEXT_DESC) + base.update(overrides) + return AgentCatOptions(**base) + + +def _tool_call_events(events, name=None): + out = [e for e in events if e.event_type == "mcp:tools/call"] + return [e for e in out if name is None or e.resource_name == name] + + +@pytest.mark.asyncio +async def test_list_tools_injects_context(captured_events): + """Every OpenAPI tool exposes the injected context param to the client.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + async with create_community_test_client(server) as client: + tools = await client.list_tools() + + by_name = {t.name: t for t in tools} + for name in OPENAPI_TOOL_NAMES: + assert name in by_name, f"{name} missing from list_tools" + schema = by_name[name].inputSchema + props = schema.get("properties", {}) + assert "context" in props, f"context not injected into {name}" + assert props["context"]["description"] == CONTEXT_DESC + assert "context" in schema.get("required", []), f"context not required on {name}" + + +@pytest.mark.asyncio +async def test_no_copy_error_logged(captured_events): + """A tools/list against OpenAPI tools must not log any copy failure.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + with patch.object(v3_middleware, "write_to_log") as mock_log: + async with create_community_test_client(server) as client: + await client.list_tools() + + copy_errors = [ + c.args[0] + for c in mock_log.call_args_list + if c.args and "Error copying tool" in str(c.args[0]) + ] + assert copy_errors == [], f"unexpected copy failures logged: {copy_errors}" + + +@pytest.mark.asyncio +async def test_original_tools_not_mutated(captured_events): + """Injection must not mutate the server's cached tools across repeated lists.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + async def raw_severity_props(): + # run_middleware=False returns the server's cached tools, un-injected. + raw = {t.name: t for t in await server.list_tools(run_middleware=False)} + return (raw["get_severity"].parameters or {}).get("properties", {}) + + # The server's own cached tool never gains a context param. + assert "context" not in await raw_severity_props() + + async with create_community_test_client(server) as client: + first = {t.name: t.inputSchema for t in await client.list_tools()} + second = {t.name: t.inputSchema for t in await client.list_tools()} + + # Client sees context injected... + assert "context" in first["get_severity"]["properties"] + # ...repeated calls are stable... + assert first == second + # ...and the cached originals remain clean after the middleware ran. + assert "context" not in await raw_severity_props() + + +@pytest.mark.asyncio +async def test_call_tool_strips_context_and_captures_intent(captured_events): + """context is captured as intent and stripped before the downstream HTTP call.""" + requests: list = [] + server = create_community_openapi_server(record_requests=requests) + track(server, "test_project", _options()) + + async with create_community_test_client(server) as client: + await client.call_tool( + "get_severity", {"id": "42", "context": "investigating an outage"} + ) + time.sleep(1.0) # let the event-queue worker drain + + calls = _tool_call_events(captured_events, "get_severity") + assert calls, "no tools/call event captured" + event = calls[-1] + assert event.user_intent == "investigating an outage" + assert (event.parameters.get("arguments") or {}) == {"id": "42"} + # The intent string must never reach the customer's backend. + assert requests, "downstream request was not recorded" + assert not any(b"investigating an outage" in (r.content or b"") for r in requests) + assert not any("investigating an outage" in str(r.url) for r in requests) + + +@pytest.mark.asyncio +async def test_call_tool_error_captured(captured_events): + """A failing OpenAPI HTTP call surfaces to the client and is captured as an error.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + async with create_community_test_client(server) as client: + with pytest.raises(Exception): + await client.call_tool("boom", {}) + time.sleep(1.0) + + boom = _tool_call_events(captured_events, "boom") + assert boom, "no tools/call event captured for boom" + assert boom[-1].is_error is True + assert boom[-1].error is not None + + +@pytest.mark.asyncio +async def test_tools_list_event_serializes_openapi_tool(captured_events): + """The tools/list event response serializes OpenAPITool subclasses cleanly.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + async with create_community_test_client(server) as client: + await client.list_tools() + time.sleep(1.0) + + list_events = [e for e in captured_events if e.event_type == "mcp:tools/list"] + assert list_events, "no tools/list event captured" + response = list_events[-1].response + assert response and isinstance(response.get("tools"), list) + names = {t.get("name") for t in response["tools"]} + assert set(OPENAPI_TOOL_NAMES).issubset(names) + + +@pytest.mark.asyncio +async def test_get_more_tools_alongside_openapi(captured_events): + """get_more_tools coexists with OpenAPI tools and is excluded from context injection.""" + server = create_community_openapi_server() + track(server, "test_project", _options(enable_report_missing=True)) + + async with create_community_test_client(server) as client: + by_name = {t.name: t for t in await client.list_tools()} + assert "get_more_tools" in by_name + # get_more_tools carries its own context arg by design; other tools get one injected. + assert "context" in by_name["get_severity"].inputSchema.get("properties", {}) + result = await client.call_tool("get_more_tools", {"context": "need more tools"}) + assert result is not None + + +@pytest.mark.asyncio +async def test_many_tools_single_pass_no_errors(captured_events): + """The full multi-tool spec lists in one pass with context on all and no errors.""" + server = create_community_openapi_server() + track(server, "test_project", _options()) + + with patch.object(v3_middleware, "write_to_log") as mock_log: + async with create_community_test_client(server) as client: + tools = await client.list_tools() + + injected = [ + t for t in tools + if t.name != "get_more_tools" + and "context" in t.inputSchema.get("properties", {}) + ] + assert len(injected) >= len(OPENAPI_TOOL_NAMES) + assert not any( + c.args and "Error copying tool" in str(c.args[0]) + for c in mock_log.call_args_list + ) + + +@pytest.mark.asyncio +async def test_openapi_tool_not_deepcopyable(): + """Precondition guard: OpenAPI tools hold non-deepcopyable runtime state.""" + server = create_community_openapi_server() + tool = (await server.list_tools())[0] + with pytest.raises(TypeError, match="cannot pickle '_thread.RLock' object"): + copy.deepcopy(tool) diff --git a/tests/community/test_community_v3_runtime_state_tools.py b/tests/community/test_community_v3_runtime_state_tools.py new file mode 100644 index 0000000..7aeb79a --- /dev/null +++ b/tests/community/test_community_v3_runtime_state_tools.py @@ -0,0 +1,79 @@ +"""Regression guard across every FastMCP v3 tool type that holds runtime state. + +The ``copy.deepcopy(tool)`` regression (PR #38) was reported for OpenAPI tools, +but the same failure mode applies to any tool that references live runtime state: + +- ``OpenAPITool`` -> holds an ``httpx.AsyncClient`` (threading.RLock) +- ``ProxyTool`` -> holds a client factory +- ``FastMCPProviderTool`` -> holds a live sub-server reference + +This suite asserts the two subclass-agnostic invariants that matter for all of +them: after ``track()``, a client ``tools/list`` sees the injected ``context`` +param on every tool, and no "Error copying tool" is ever logged. +""" + +from unittest.mock import patch + +import pytest + +from agentcat import AgentCatOptions, track +from agentcat.modules.overrides.community_v3 import middleware as v3_middleware + +from ..test_utils.community_client import create_community_test_client +from ..test_utils.community_openapi_server import ( + HAS_FASTMCP_V3, + create_community_mounted_server, + create_community_openapi_server, + create_community_proxy_server, +) + +pytestmark = pytest.mark.skipif( + not HAS_FASTMCP_V3, + reason="Requires FastMCP v3+ (runtime-state tool types)", +) + + +def _build(factory): + """Build a server from a factory, skipping if this fastmcp version can't.""" + try: + return factory() + except Exception as exc: # pragma: no cover - version-dependent construction + pytest.skip(f"{factory.__name__} unavailable on this FastMCP: {exc!r}") + + +@pytest.mark.parametrize( + "factory", + [ + create_community_openapi_server, + create_community_proxy_server, + create_community_mounted_server, + ], + ids=["openapi", "proxy", "mounted"], +) +@pytest.mark.asyncio +async def test_context_injected_without_copy_errors(factory): + server = _build(factory) + track( + server, + "test_project", + AgentCatOptions(enable_tracing=True, custom_context_description="Why?"), + ) + + with patch.object(v3_middleware, "write_to_log") as mock_log: + async with create_community_test_client(server) as client: + tools = await client.list_tools() + + assert tools, "server exposed no tools" + for tool in tools: + if tool.name == "get_more_tools": + continue + props = tool.inputSchema.get("properties", {}) + assert "context" in props, f"context not injected into {tool.name}" + assert "context" in tool.inputSchema.get("required", []) + + copy_errors = [ + c.args[0] + for c in mock_log.call_args_list + if c.args and "Error copying tool" in str(c.args[0]) + ] + assert copy_errors == [], f"copy failures logged: {copy_errors}" diff --git a/tests/test_utils/community_openapi_server.py b/tests/test_utils/community_openapi_server.py new file mode 100644 index 0000000..b6ae56b --- /dev/null +++ b/tests/test_utils/community_openapi_server.py @@ -0,0 +1,186 @@ +"""Factories for FastMCP v3 servers whose tools hold live runtime state. + +These build the classes of tool that carry non-deepcopyable runtime state and +therefore broke context injection before PR #38: + +- ``OpenAPITool`` (from ``FastMCP.from_openapi``) holds a live ``httpx.AsyncClient`` + (which contains a ``threading.RLock``). +- ``ProxyTool`` (from ``create_proxy``) holds a client factory. +- ``FastMCPProviderTool`` (from a mounted sub-server) holds a live server reference. + +All are driven fully in-process: the OpenAPI server uses an ``httpx.MockTransport`` +so tool calls return canned responses with no network. Mirrors the flag / factory +shape of ``community_todo_server.py``. +""" + +from typing import TYPE_CHECKING, Any + +if TYPE_CHECKING: + from fastmcp import FastMCP + +# Lazy, guarded imports so importing this module never fails the no-FastMCP CI job +# or the FastMCP v2 compatibility matrix. Callers gate on HAS_FASTMCP_V3. +try: + import httpx + import fastmcp + from fastmcp import FastMCP as CommunityFastMCP + from fastmcp.server.providers.openapi import MCPType, RouteMap + + HAS_FASTMCP_V3 = int(fastmcp.__version__.split(".")[0]) >= 3 +except Exception: # pragma: no cover - import guard + httpx = None # type: ignore + CommunityFastMCP = None # type: ignore + MCPType = RouteMap = None # type: ignore + HAS_FASTMCP_V3 = False + + +# A Rootly-flavored spec with many distinct operationIds so list_tools returns a +# realistic multi-tool set. Every route is forced to a TOOL below. The ``boom`` +# endpoint is used to exercise the HTTP-error capture path. +OPENAPI_SPEC: dict[str, Any] = { + "openapi": "3.0.0", + "info": {"title": "rootly", "version": "1"}, + "paths": { + "/severities": { + "get": {"operationId": "list_severities", "responses": {"200": {"description": "ok"}}}, + "post": {"operationId": "create_severity", "responses": {"200": {"description": "ok"}}}, + }, + "/severities/{id}": { + "get": { + "operationId": "get_severity", + "parameters": [ + {"name": "id", "in": "path", "required": True, "schema": {"type": "string"}} + ], + "responses": {"200": {"description": "ok"}}, + } + }, + "/schedules": { + "get": {"operationId": "list_schedules", "responses": {"200": {"description": "ok"}}}, + "post": {"operationId": "create_schedule", "responses": {"200": {"description": "ok"}}}, + }, + "/schedules/{id}": { + "get": { + "operationId": "get_schedule", + "parameters": [ + {"name": "id", "in": "path", "required": True, "schema": {"type": "string"}} + ], + "responses": {"200": {"description": "ok"}}, + } + }, + "/incidents": { + "get": {"operationId": "list_incidents", "responses": {"200": {"description": "ok"}}}, + "post": {"operationId": "create_incident", "responses": {"200": {"description": "ok"}}}, + }, + "/incidents/{id}": { + "get": { + "operationId": "get_incident", + "parameters": [ + {"name": "id", "in": "path", "required": True, "schema": {"type": "string"}} + ], + "responses": {"200": {"description": "ok"}}, + } + }, + "/teams": { + "get": {"operationId": "list_teams", "responses": {"200": {"description": "ok"}}}, + }, + "/users": { + "get": {"operationId": "list_users", "responses": {"200": {"description": "ok"}}}, + }, + "/alerts": { + "get": {"operationId": "list_alerts", "responses": {"200": {"description": "ok"}}}, + }, + "/boom": { + "get": {"operationId": "boom", "responses": {"500": {"description": "boom"}}}, + }, + }, +} + +# Names generated from the operationIds above, excluding ``boom`` (error path). +OPENAPI_TOOL_NAMES = [ + "list_severities", "create_severity", "get_severity", + "list_schedules", "create_schedule", "get_schedule", + "list_incidents", "create_incident", "get_incident", + "list_teams", "list_users", "list_alerts", +] + + +def _require_v3() -> None: + if not HAS_FASTMCP_V3: + raise ImportError("FastMCP v3+ is required for these factories.") + + +def create_community_openapi_server(record_requests: list | None = None) -> "FastMCP": + """Build an OpenAPI-generated FastMCP v3 server backed by a mock transport. + + Args: + record_requests: if provided, every outbound ``httpx.Request`` is appended + to it, so tests can assert the downstream request never carried the + injected ``context`` intent. + + Returns: + A FastMCP server whose tools are ``OpenAPITool`` instances holding a live + ``httpx.AsyncClient`` (mock transport). + """ + _require_v3() + + def handler(request: "httpx.Request") -> "httpx.Response": + if record_requests is not None: + record_requests.append(request) + if request.url.path == "/boom": + return httpx.Response(500, json={"error": "boom"}) + return httpx.Response(200, json={"ok": True, "path": str(request.url.path)}) + + client = httpx.AsyncClient( + base_url="http://testapi", transport=httpx.MockTransport(handler) + ) + return CommunityFastMCP.from_openapi( + openapi_spec=OPENAPI_SPEC, + client=client, + name="rootly", + route_maps=[RouteMap(mcp_type=MCPType.TOOL)], + ) + + +def create_community_proxy_server() -> "FastMCP": + """Build a proxy FastMCP v3 server whose tools are ``ProxyTool`` instances.""" + _require_v3() + + backend = CommunityFastMCP("proxy-backend") + + @backend.tool + def ping(text: str) -> str: + """Echo the given text.""" + return text + + @backend.tool + def pong(text: str) -> str: + """Echo the given text back.""" + return text + + try: + from fastmcp.server import create_proxy + + return create_proxy(backend, name="proxy") + except Exception: # pragma: no cover - older v3 API fallback + return CommunityFastMCP.as_proxy(backend, name="proxy") + + +def create_community_mounted_server() -> "FastMCP": + """Build a parent server with a mounted sub-server (``FastMCPProviderTool``).""" + _require_v3() + + parent = CommunityFastMCP("parent") + sub = CommunityFastMCP("sub") + + @sub.tool + def sub_action(value: str) -> str: + """A tool provided by the mounted sub-server.""" + return value + + @sub.tool + def sub_query(value: str) -> str: + """Another tool provided by the mounted sub-server.""" + return value + + parent.mount(sub, namespace="sub") + return parent