fix(events): never drop events on non-JSON-serializable payloads

pyproject.toml

@@ -1,6 +1,6 @@
 [project]
 name = "agentcat"
-version = "1.0.1"
+version = "1.0.2"
 description = "Analytics tool for MCP (Model Context Protocol) servers and AI agents - tracks tool usage patterns and provides insights"
 authors = [
     { name = "AgentCat, Inc.", email = "[email protected]" },

src/agentcat/modules/overrides/community_v3/middleware.py

@@ -479,16 +479,21 @@ def _tool_to_dict(self, tool: Tool) -> dict[str, Any]:
             Dictionary representation of the tool.
         """
         try:
-            if hasattr(tool, "model_dump"):
-                return tool.model_dump()
-            elif hasattr(tool, "to_mcp_tool"):
+            # Prefer the canonical MCP shape. A FastMCP Tool.model_dump() includes
+            # SDK-internal fields that are not JSON-serializable — a callable ``fn``
+            # (FunctionTool) and a ``tags`` set — which would break event
+            # serialization; to_mcp_tool() yields the clean wire representation
+            # (name/description/inputSchema/...).
+            if hasattr(tool, "to_mcp_tool"):
                 mcp_tool = tool.to_mcp_tool()
-                return mcp_tool.model_dump() if hasattr(mcp_tool, "model_dump") else {}
-            else:
-                return {
-                    "name": getattr(tool, "name", "unknown"),
-                    "description": getattr(tool, "description", ""),
-                }
+                if hasattr(mcp_tool, "model_dump"):
+                    return mcp_tool.model_dump(mode="json")
+            if hasattr(tool, "model_dump"):
+                return tool.model_dump(mode="json")
+            return {
+                "name": getattr(tool, "name", "unknown"),
+                "description": getattr(tool, "description", ""),
+            }
         except Exception as e:
             write_to_log(f"Error converting tool to dict: {e}")
             return {"name": getattr(tool, "name", "unknown")}

src/agentcat/modules/truncation.py

@@ -9,6 +9,8 @@
 from datetime import date, datetime
 from typing import Any, TYPE_CHECKING
 
+import pydantic_core
+
 if TYPE_CHECKING:
     from agentcat.types import UnredactedEvent
 
@@ -24,6 +26,24 @@
 TRUNCATABLE_FIELDS = {"parameters", "response", "error", "identify_data", "user_intent", "additional_properties"}
 
 
+def make_json_safe(value: Any) -> Any:
+    """Best-effort conversion of *value* into JSON-serializable primitives.
+
+    Callables become their ``repr``, sets/tuples become lists, ``datetime``/``date``
+    become ISO strings, and any other unknown object falls back to ``str``. Never
+    raises. This is what keeps a non-JSON-safe value in an event payload (a tool's
+    ``fn`` callable, a ``set`` of tags, a custom object) from either aborting
+    truncation or being silently dropped by the API client on send.
+    """
+    try:
+        return pydantic_core.to_jsonable_python(value, fallback=str)
+    except Exception:
+        try:
+            return str(value)
+        except Exception:
+            return None
+
+
 def _truncate_string(value: str, max_bytes: int = MAX_STRING_BYTES) -> str:
     """Truncate a string if its UTF-8 byte size exceeds *max_bytes*."""
     byte_size = len(value.encode("utf-8"))
@@ -135,8 +155,18 @@ def truncate_event(event: "UnredactedEvent | None") -> "UnredactedEvent | None":
     if event is None:
         return None
 
+    # Coerce payload fields to JSON-safe primitives up front. This both lets the
+    # size checks below serialize cleanly and — because the API client re-serializes
+    # this same event object on send — prevents a non-serializable value (a callable,
+    # a set, a datetime, a custom object) from silently dropping the event downstream.
+    # The event reaching here is already a copy (see sanitize_event), so mutation is safe.
+    for field_name in TRUNCATABLE_FIELDS:
+        value = getattr(event, field_name, None)
+        if value is not None and not isinstance(value, (str, int, float, bool)):
+            setattr(event, field_name, make_json_safe(value))
+
     try:
-        serialized_bytes = event.model_dump_json().encode("utf-8")
+        serialized_bytes = event.model_dump_json(fallback=str).encode("utf-8")
         byte_size = len(serialized_bytes)
         if byte_size <= MAX_EVENT_BYTES:
             return event
@@ -167,7 +197,7 @@ def truncate_event(event: "UnredactedEvent | None") -> "UnredactedEvent | None":
                             max_breadth=breadth,
                         )
             candidate = event_cls.model_validate(event_dict)
-            result_bytes = len(candidate.model_dump_json().encode("utf-8"))
+            result_bytes = len(candidate.model_dump_json(fallback=str).encode("utf-8"))
             if result_bytes <= MAX_EVENT_BYTES:
                 return candidate
             write_to_log(

tests/community/test_community_v3_event_serialization.py

@@ -0,0 +1,93 @@
+"""End-to-end: tools/list events must be JSON-serializable (agentcat 1.0.1 data loss).
+
+On 1.0.1 every ``tools/list`` event carried the FastMCP tool's ``fn`` callable and
+``tags`` set (via ``_tool_to_dict`` -> ``tool.model_dump()``), so truncation logged
+``Unable to serialize unknown type: <class 'function'>`` and the event was then
+dropped by the API client (``'set' object has no attribute '__dict__'``). These
+tests drive a real ``tools/list`` and assert the captured event survives.
+"""
+
+import json
+import time
+from unittest.mock import MagicMock
+
+import pytest
+
+from agentcat import AgentCatOptions, track
+from agentcat.modules.event_queue import EventQueue, set_event_queue
+
+from ..test_utils.community_client import (
+    HAS_COMMUNITY_CLIENT,
+    create_community_test_client,
+)
+from ..test_utils.community_todo_server import (
+    HAS_COMMUNITY_FASTMCP,
+    create_community_todo_server,
+)
+
+pytestmark = pytest.mark.skipif(
+    not (HAS_COMMUNITY_FASTMCP and HAS_COMMUNITY_CLIENT),
+    reason="Community FastMCP not available",
+)
+
+
+@pytest.fixture
+def captured_events():
+    from agentcat.modules.event_queue import event_queue as original_queue
+
+    events: list = []
+    mock_api_client = MagicMock()
+    mock_api_client.publish_event = MagicMock(
+        side_effect=lambda publish_event_request: events.append(publish_event_request)
+    )
+    set_event_queue(EventQueue(api_client=mock_api_client))
+    try:
+        yield events
+    finally:
+        set_event_queue(original_queue)
+
+
+def _list_event(events):
+    matches = [e for e in events if e.event_type == "mcp:tools/list"]
+    return matches[-1] if matches else None
+
+
+@pytest.mark.asyncio
+async def test_tools_list_event_is_json_serializable(captured_events):
+    """The captured tools/list event.response must be fully JSON-safe."""
+    server = create_community_todo_server()  # FunctionTools carry the fn callable
+    track(server, "test_project", AgentCatOptions(enable_tracing=True))
+
+    async with create_community_test_client(server) as client:
+        await client.list_tools()
+        time.sleep(1.0)
+
+    event = _list_event(captured_events)
+    assert event is not None, "no tools/list event captured"
+    # The exact thing the generated API client does before sending — must not raise.
+    json.dumps(event.response)
+
+    # And the payload is the clean MCP shape (from to_mcp_tool), not a raw
+    # tool.model_dump() with an embedded callable ``fn``.
+    tool = event.response["tools"][0]
+    assert "fn" not in tool
+    assert "inputSchema" in tool
+
+
+@pytest.mark.asyncio
+async def test_tools_list_event_survives_api_client_serialization(captured_events):
+    """Reproduce the send path: the generated client's sanitizer must not raise."""
+    server = create_community_todo_server()
+    track(server, "test_project", AgentCatOptions(enable_tracing=True))
+
+    async with create_community_test_client(server) as client:
+        await client.list_tools()
+        time.sleep(1.0)
+
+    event = _list_event(captured_events)
+    assert event is not None
+
+    from agentcat_api.api_client import ApiClient
+
+    # 'set' object has no attribute '__dict__' was the 1.0.1 drop; must be gone now.
+    ApiClient.sanitize_for_serialization(ApiClient(), event.response)

tests/community/test_community_v3_inject_context.py

@@ -50,7 +50,7 @@ def _openapi_tool():
     """An OpenAPI-generated tool holding an httpx client (non-deepcopyable)."""
     spec = {
         "openapi": "3.0.0",
-        "info": {"title": "rootly", "version": "1"},
+        "info": {"title": "acme", "version": "1"},
         "paths": {
             "/severities": {
                 "get": {
@@ -62,7 +62,7 @@ def _openapi_tool():
         },
     }
     client = httpx.AsyncClient(base_url="https://example.com")
-    server = FastMCP.from_openapi(openapi_spec=spec, client=client, name="rootly")
+    server = FastMCP.from_openapi(openapi_spec=spec, client=client, name="acme")
     tools = server.list_tools()
     if asyncio.iscoroutine(tools):
         tools = asyncio.run(tools)

tests/test_truncation_json_safe.py

@@ -0,0 +1,54 @@
+"""Truncation must never abort (or silently drop an event) on a non-JSON-safe payload.
+
+Regression for agentcat 1.0.1: event payloads holding a raw callable made
+``truncate_event``'s ``model_dump_json()`` raise ``Unable to serialize unknown
+type: <class 'function'>``, and payloads holding a ``set`` made the downstream
+API client drop the event with ``'set' object has no attribute '__dict__'``.
+The pipeline must coerce such values to JSON-safe primitives instead.
+"""
+
+import json
+from unittest.mock import patch
+
+from agentcat.modules import truncation
+from agentcat.modules.truncation import truncate_event
+from agentcat.types import UnredactedEvent
+
+
+def _poison_response():
+    # Mirrors what tool.model_dump() embeds for FastMCP v3 tools: a callable and a set.
+    return {"tools": [{"name": "t", "fn": (lambda: 1), "tags": {"a", "b"}}]}
+
+
+def test_truncate_event_does_not_log_failure_on_callable_and_set():
+    event = UnredactedEvent(event_type="mcp:tools/list", response=_poison_response())
+    with patch.object(truncation, "write_to_log") as mock_log:
+        result = truncate_event(event)
+    failures = [
+        c.args[0]
+        for c in mock_log.call_args_list
+        if c.args and "Truncation failed" in str(c.args[0])
+    ]
+    assert failures == [], f"truncation aborted: {failures}"
+    assert result is not None
+
+
+def test_truncated_event_is_json_serializable():
+    """After truncation the event's payload must contain only JSON-safe primitives."""
+    event = UnredactedEvent(event_type="mcp:tools/list", response=_poison_response())
+    result = truncate_event(event)
+    # The event the API client re-serializes on send must not carry a callable/set.
+    json.dumps(result.response)  # would raise if a set/function survived
+    result.model_dump_json()     # would raise if pydantic still can't serialize it
+
+
+def test_make_json_safe_neutralizes_all_types():
+    import datetime
+
+    safe = truncation.make_json_safe(
+        {"fn": (lambda: 1), "tags": {"x"}, "when": datetime.datetime(2020, 1, 1)}
+    )
+    text = json.dumps(safe)  # must not raise
+    assert "tags" in safe and isinstance(safe["tags"], list)
+    assert isinstance(safe["fn"], str)
+    assert safe["when"].startswith("2020-01-01")

tests/test_utils/community_openapi_server.py

@@ -34,12 +34,12 @@
     HAS_FASTMCP_V3 = False
 
 
-# A Rootly-flavored spec with many distinct operationIds so list_tools returns a
+# An Acme Co-flavored spec with many distinct operationIds so list_tools returns a
 # realistic multi-tool set. Every route is forced to a TOOL below. The ``boom``
 # endpoint is used to exercise the HTTP-error capture path.
 OPENAPI_SPEC: dict[str, Any] = {
     "openapi": "3.0.0",
-    "info": {"title": "rootly", "version": "1"},
+    "info": {"title": "acme", "version": "1"},
     "paths": {
         "/severities": {
             "get": {"operationId": "list_severities", "responses": {"200": {"description": "ok"}}},
@@ -136,7 +136,7 @@ def handler(request: "httpx.Request") -> "httpx.Response":
     return CommunityFastMCP.from_openapi(
         openapi_spec=OPENAPI_SPEC,
         client=client,
-        name="rootly",
+        name="acme",
         route_maps=[RouteMap(mcp_type=MCPType.TOOL)],
     )