AI record(): capture full per-turn interaction (messages + latency + usage + tool calls)

example/agents/tests/units/agents/basic_job_search.json

@@ -1,15 +1,45 @@
 {
-  "921847ca3e137cf6fb59760ecb38e34522a6df8f702eea6329b466b5f488d786": {
-    "content": "I found one job opening for a Frontend Developer at Startup Inc. It is a full-time position and is remote. Would you like to explore this opportunity or see other options?",
-    "tool_calls": [
-      {
-        "args": {
-          "query": "python developer"
-        },
-        "id": "b940618e-6f0a-4ba3-a88b-cd9802d49081",
-        "name": "job_search_tool",
-        "type": "tool_call"
+  "921847ca3e137cf6fb59760ecb38e34522a6df8f702eea6329b466b5f488d786": [
+    {
+      "content": "Suggest python developer jobs",
+      "type": "human"
+    },
+    {
+      "response_time": 3.6957909469492733,
+      "tool_calls": [
+        {
+          "args": {
+            "query": "python developer"
+          },
+          "id": "call_basic",
+          "name": "job_search_tool",
+          "type": "tool_call"
+        }
+      ],
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 117,
+        "output_token": 22,
+        "total_token": 139
       }
-    ]
-  }
+    },
+    {
+      "content": "[{\"id\": 2, \"title\": \"Frontend Developer\", \"location\": \"Remote\", \"company\": \"Startup Inc\", \"type\": \"Full-time\"}]",
+      "content_type": "json",
+      "response_time": 0.7865419611334801,
+      "type": "tool_response"
+    },
+    {
+      "content": "I found one opening for a Frontend Developer at Startup Inc, which is a remote, full-time position.",
+      "response_time": 2.021583030000329,
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 140,
+        "output_token": 22,
+        "total_token": 162
+      }
+    }
+  ]
 }

example/agents/tests/units/agents/record_sales.json

@@ -1,6 +1,19 @@
 {
-  "64c72be4d72cc783cd8423906eef1db674cefba1ec988817da88df37eff05e2a": {
-    "content": "I do not have a plan to offer. I am a large language model, an AI. How can I help you?",
-    "tool_calls": []
-  }
+  "64c72be4d72cc783cd8423906eef1db674cefba1ec988817da88df37eff05e2a": [
+    {
+      "content": "What plans do you offer?",
+      "type": "human"
+    },
+    {
+      "content": "I am a large language model, I don't have plans. Is there anything else I can help you with?",
+      "response_time": 2.0521670230664313,
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 51,
+        "output_token": 24,
+        "total_token": 75
+      }
+    }
+  ]
 }

example/agents/tests/units/agents/record_stream.json

@@ -0,0 +1,83 @@
+{
+  "034751ef1322a12f3406dad43313f9cdb31f4ea85d9452c22bf7529ad8e92614": [
+    {
+      "content": "hello",
+      "type": "human"
+    },
+    {
+      "content": "Hello! How can I help you today?",
+      "response_time": 1.286374987103045,
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 54,
+        "output_token": 9,
+        "total_token": 63
+      }
+    }
+  ],
+  "358018dc096ce95b78b7561f562ba049848d03fb8b41953ed75c22c9ad20e228": [
+    {
+      "content": "suggest python developer jobs",
+      "type": "human"
+    },
+    {
+      "response_time": 1.1794579913839698,
+      "tool_calls": [
+        {
+          "args": {
+            "query": "python developer"
+          },
+          "id": "call_router_2",
+          "name": "job_search_tool",
+          "type": "tool_call"
+        }
+      ],
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 117,
+        "output_token": 24,
+        "total_token": 141
+      }
+    },
+    {
+      "content": "[{\"id\": 2, \"title\": \"Frontend Developer\", \"location\": \"Remote\", \"company\": \"Startup Inc\", \"type\": \"Full-time\"}]",
+      "content_type": "json",
+      "response_time": 0.3560830373317003,
+      "type": "tool_response"
+    }
+  ],
+  "b6579ba866634a66229ddbed0463c8c990eaaaf349385fbc67c2c2c1886085cd": [
+    {
+      "content": "suggest python developer jobs",
+      "type": "human"
+    },
+    {
+      "response_time": 1.1740420013666153,
+      "tool_calls": [
+        {
+          "args": {
+            "query": "python developer"
+          },
+          "id": "call_router_1",
+          "name": "job_search_tool",
+          "type": "tool_call"
+        }
+      ],
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 117,
+        "output_token": 24,
+        "total_token": 141
+      }
+    },
+    {
+      "content": "[{\"id\": 2, \"title\": \"Frontend Developer\", \"location\": \"Remote\", \"company\": \"Startup Inc\", \"type\": \"Full-time\"}]",
+      "content_type": "json",
+      "response_time": 0.40095800068229437,
+      "type": "tool_response"
+    }
+  ]
+}

example/agents/tests/units/agents/stream_job_search.json

@@ -1,21 +1,50 @@
 {
-  "921847ca3e137cf6fb59760ecb38e34522a6df8f702eea6329b466b5f488d786": {
-    "chunks": [
-      "[{\"id\": 2, \"title\": \"Frontend Developer\", \"location\": \"Remote\", \"company\": \"Startup Inc\", \"type\": \"Full-time\"}]",
-      "I",
-      " found a Python developer job for you at Startup Inc. It is a full-time position",
-      " and is remote.\n"
-    ],
-    "content": "I found a Python developer job for you at Startup Inc. It is a full-time position and is remote.\n",
-    "tool_calls": [
-      {
-        "args": {
-          "query": "python developer"
-        },
-        "id": "8ffe887c-db92-4ff8-9b2a-9ad8e318e3c4",
-        "name": "job_search_tool",
-        "type": "tool_call"
+  "921847ca3e137cf6fb59760ecb38e34522a6df8f702eea6329b466b5f488d786": [
+    {
+      "content": "Suggest python developer jobs",
+      "type": "human"
+    },
+    {
+      "response_time": 1.0,
+      "tool_calls": [
+        {
+          "args": {
+            "query": "python developer"
+          },
+          "id": "call_stream",
+          "name": "job_search_tool",
+          "type": "tool_call"
+        }
+      ],
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 117,
+        "output_token": 24,
+        "total_token": 141
       }
-    ]
-  }
-}
+    },
+    {
+      "content": "[{\"id\": 2, \"title\": \"Frontend Developer\", \"location\": \"Remote\", \"company\": \"Startup Inc\", \"type\": \"Full-time\"}]",
+      "content_type": "json",
+      "response_time": 0.5,
+      "type": "tool_response"
+    },
+    {
+      "chunks": [
+        "I found a job for you!",
+        " It's a Full-time Frontend Developer position at Startup Inc,",
+        " located remotely."
+      ],
+      "content": "I found a job for you! It's a Full-time Frontend Developer position at Startup Inc, located remotely.",
+      "response_time": 1.0,
+      "type": "ai",
+      "uses": {
+        "cache_token": 0,
+        "input_token": 140,
+        "output_token": 24,
+        "total_token": 164
+      }
+    }
+  ]
+}

example/agents/tests/units/agents/test_router_agent.py

@@ -1,4 +1,4 @@
-from fastapi_startkit.ai.tinker import ToolCall
+from fastapi_startkit.ai import AssertToolCall
 from langchain_core.messages import AIMessage, HumanMessage
 
 from app.agents.chat import ChatAgent
@@ -14,7 +14,7 @@ async def test_the_router_agent(self):
 
             agent.assert_response_time_lt(5)
 
-            def assert_tool_calls(tool: ToolCall):
+            def assert_tool_calls(tool: AssertToolCall):
                 return tool.name == "job_search_tool"
 
             await agent.prompt("suggest python developer jobs")

fastapi_startkit/src/fastapi_startkit/ai/__init__.py

@@ -13,7 +13,7 @@
 from .judge import JudgeAgent
 from .providers.ai_provider import AIProvider
 from .response import AgentResponse, AgentSnapshot
-from .testing import AgentFake, AgentRecordFake, ToolCallView
+from .testing import AgentFake, AgentRecordFake, AssertToolCall
 
 __all__ = [
     "Agent",
@@ -27,7 +27,7 @@
     "AnthropicConfig",
     "JudgeAgent",
     "AgentRecordFake",
-    "ToolCallView",
+    "AssertToolCall",
     "Audio",
     "AudioResponse",
     "AudioFactory",

fastapi_startkit/src/fastapi_startkit/ai/agent.py

@@ -40,23 +40,21 @@ def provider_options(self) -> dict:
         return {}
 
     async def prompt(
-            self,
-            message: str,
-            *,
-            model: str | None = None,
-            attachments: list[Document] | None = None,
-            provider_options: dict | None = None,
+        self,
+        message: str,
+        *,
+        model: str | None = None,
+        attachments: list[Document] | None = None,
+        provider_options: dict | None = None,
     ) -> AgentResponse:
-        return await self.runner().run(
-            message, model=model, attachments=attachments, provider_options=provider_options
-        )
+        return await self.runner().run(message, model=model, attachments=attachments, provider_options=provider_options)
 
     async def stream(
-            self,
-            message: str,
-            *,
-            model: str | None = None,
-            provider_options: dict | None = None,
+        self,
+        message: str,
+        *,
+        model: str | None = None,
+        provider_options: dict | None = None,
     ) -> AsyncIterator[str]:
         async for chunk in self.runner().stream(message, model=model, provider_options=provider_options):
             yield chunk