[WIP] [#521] Research Agent Tools

server/api/views/assistant/agentic_loop.py

@@ -0,0 +1,175 @@
+import json
+import logging
+
+from api.views.assistant.assistant_types import (
+    AgentResult,
+    ToolCallExecution,
+    ToolCallStatus,
+    TurnUsage,
+)
+
+logger = logging.getLogger(__name__)
+
+
+def run_agentic_loop(
+    response, client, model_defaults: dict, tools: list, user
+) -> AgentResult:
+
+    # Every tool call the agentic loop made before exiting
+    agentic_loop_tool_call_executions= []
+    # Token usage for every responses.create call, one entry per iteration
+    agentic_loop_turns: list[TurnUsage] = []
+
+    while True:
+        # At the top of the body, so the initial response and the terminal turn are each counted exactly once
+
+        # _turn_usage never raises: it runs on the web request path, so an unrecognized usage shape must not fail a user's request.
+        agentic_loop_turns.append(_turn_usage(response))
+
+        # user is threaded through so tools that need it get it at dispatch time
+        tool_output_schemas, tool_call_executions = handle_tool_calls(response, tools, user)
+
+        # TODO: Decide whether to add turn: int to ToolCallExecution — without it, the flat tool_calls can't be split back into turns
+        # .extend splices every iteration's list of tools into one list
+        agentic_loop_tool_call_executions.extend(tool_call_executions)
+
+        # Exit agentic loop when model response doesn't contain any tool calls
+        if not tool_output_schemas:
+            return AgentResult(
+                output_text=response.output_text,
+                response_id=response.id,
+                tool_calls=agentic_loop_tool_call_executions,
+                turns=agentic_loop_turns,
+            )
+
+        #TODO: Add error handling to collect partial AgentResult tool calls
+        response = client.responses.create(
+            input=tool_output_schemas,
+            previous_response_id=response.id,
+            **model_defaults,
+        )
+
+
+def _int_or_none(obj, field: str) -> int | None:
+    """Read one integer leaf, or None when it is missing or not an int.
+
+    bool is excluded deliberately: it is an int subclass, so True would otherwise be
+    recorded as a token count of 1.
+    """
+    value = getattr(obj, field, None)
+    if isinstance(value, bool) or not isinstance(value, int):
+        return None
+    return value
+
+
+def _turn_usage(response) -> TurnUsage:
+    """Token usage for one response, never raising.
+
+    This runs on the web request path as well as in the eval, so an unrecognized usage
+    shape must not fail a user's request.
+
+    getattr's default guards the *traversal*, not only the leaves: when usage is
+    missing, usage.output_tokens_details would raise before any leaf check ran.
+    getattr(None, ...) returns None instead, collapsing the whole chain.
+
+    The isinstance guard is what lets the tests fail. MagicMock implements
+    __add__/__radd__, so a mocked usage would otherwise accumulate into the CSV as mock
+    objects with the suite green.
+    """
+
+    #  getattr's default guards the traversal rather than only the leaves,
+    # since usage.output_tokens_details would raise before any leaf check when usage is missing
+
+    # That guard makes silent blanks the hazard, so the field names are pinned by a
+    # test building a real ResponseUsage - the only input in the suite not
+    # constructed from names we chose, and so the only one where a misspelling can
+    # fail rather than quietly blanking a column. Verified against openai 2.29.0.
+
+    usage = getattr(response, "usage", None)
+    input_details = getattr(usage, "input_tokens_details", None)
+    output_details = getattr(usage, "output_tokens_details", None)
+
+    return TurnUsage(
+        response_id=response.id,
+        input_tokens=_int_or_none(usage, "input_tokens"),
+        cached_input_tokens=_int_or_none(input_details, "cached_tokens"),
+        output_tokens=_int_or_none(usage, "output_tokens"),
+        reasoning_output_tokens=_int_or_none(output_details, "reasoning_tokens"),
+    )
+
+
+def handle_tool_calls(
+    response, tools: list, user
+) -> tuple[list[dict], list[ToolCallExecution]]:
+
+    # Index the tools by name so a model-supplied call name can be looked up. .get()
+    # returns None for an unknown name, handled explicitly below.
+    tools_by_name = {tool.name: tool for tool in tools}
+
+    tool_output_schemas = []
+    tool_call_executions: list[ToolCallExecution] = []
+
+    for response_item in response.output:
+        if response_item.type == "reasoning":
+            #logger.info(f"Reasoning step: {response_item.summary}")
+            pass
+
+        elif response_item.type == "function_call":
+
+            tool_output, tool_call_execution = _execute_function_call(response_item, tools_by_name, user)
+
+            tool_output_schemas.append(
+                {
+                    "type": "function_call_output",
+                    "call_id": response_item.call_id,
+                    "output": tool_output,
+                }
+            )
+
+            tool_call_executions.append(tool_call_execution)
+
+
+    return tool_output_schemas, tool_call_executions
+
+
+def _execute_function_call(
+    response_item, tools_by_name: dict, user
+) -> tuple[str, ToolCallExecution]:
+
+    target_tool = tools_by_name.get(response_item.name)
+
+    # Parsed below; stays None if the model's argument JSON can't be parsed,
+    # so a FAILED record still reports whatever we managed to read.
+    arguments = None
+
+    if target_tool is None:
+        msg = f"ERROR - No tool registered for function call: {response_item.name}"
+        logger.error(msg)
+        return msg, ToolCallExecution(
+            name=response_item.name,
+            status=ToolCallStatus.UNREGISTERED,
+            error=msg,
+        )
+
+    try:
+        arguments = json.loads(response_item.arguments)
+        logger.info(
+            f"Invoking tool: {response_item.name} with arguments: {arguments}"
+        )
+        tool_output = target_tool.run(user=user, **arguments)
+        logger.info(f"Tool {response_item.name} completed successfully")
+        return tool_output, ToolCallExecution(
+            name=response_item.name,
+            status=ToolCallStatus.OK,
+            arguments=arguments,
+            output=tool_output,
+        )
+    except Exception as e:
+        msg = f"Error executing function call: {response_item.name}: {e}"
+        logger.error(msg, exc_info=True)
+        return msg, ToolCallExecution(
+            name=response_item.name,
+            status=ToolCallStatus.FAILED,
+            arguments=arguments,
+            error=str(e),
+        )

server/api/views/assistant/assistant_prompts.py

@@ -1,3 +1,27 @@
+# TODO: rewrite the citation template below (RESPONSE FORMAT item 4) so the braces are not
+# emitted literally. `[Name {name}, Page {page_number}]` is read by the model as required
+# output *syntax* rather than as placeholders: the 20260807 eval returned
+#   [Pharmacological Treatment of Bipolar Depression: ... Options? {Pharmacological
+#    Treatment of Bipolar Depression: ... Options?}, Page 2]
+# — the name filled in AND the braces kept, duplicating the title. Also observed:
+# "Page: 3" (stray colon), "Page 4, Chunk 32" (extra field), "various pages",
+# "multiple pages including 1-5". Show a filled-in example instead of a brace template,
+# e.g. `[Name advancespharmaco.pdf, Page 9]`, and state that exactly one page number is
+# cited per reference.
+#
+# This is one of two separable citation defects; the other is search_tool.py handing the
+# model a UUID alongside the name (see the TODO there). Neither is cosmetic — citations
+# are unparseable until both land, which blocks the eval's scoring layer: citation
+# accuracy is the cheapest real signal available, and a parser written before these two
+# fixes would measure prompt drift rather than accuracy. That layer is not in the tree;
+# it is specified in WORKLOG.md under Blocked.
+#
+# Note both known importers pass this string through verbatim — assistant_services.py
+# hands it to the API as `instructions`, eval_assistant.py imports it for a planned
+# sidecar and does not use it — so no .format() reads the braces. They are inert to
+# Python; the only thing interpreting them is the model.
+
+# TODO: Mention ask_database — the prompt names only search_documents and says to "ALWAYS use" it first, steering the model away from ask_database
 INSTRUCTIONS = """
 You are an AI assistant that helps users find and understand information about bipolar disorder 
 from your internal library of bipolar disorder research sources using semantic search.

server/api/views/assistant/assistant_services.py

@@ -3,70 +3,51 @@
 
 from openai import OpenAI
 
-from .assistant_prompts import INSTRUCTIONS
-from .tool_services import (
-    SEARCH_TOOLS_SCHEMA,
-    make_search_tool_mapping,
-    handle_tool_calls_with_reasoning,
-)
+from api.views.assistant.assistant_prompts import INSTRUCTIONS
+from api.views.assistant.tool_services import TOOLS
+from api.views.assistant.assistant_types import AgentResult
+from api.views.assistant.agentic_loop import run_agentic_loop
 
 logger = logging.getLogger(__name__)
 
+# Module-level so eval_assistant.py can import it and label its CSV with the model that actually ran
+MODEL_NAME = "gpt-5-nano"
+
 
 def run_assistant(
-    message: str,
     user,
+    message: str,
     previous_response_id: str | None = None,
-) -> tuple[str, str]:
-    """Wire together the OpenAI client, retrieval, and the agentic reasoning loop.
-
-    Parameters
-    ----------
-    message : str
-        The user's input message.
-    user : User
-        The Django user object used for document access control in search_documents.
-    previous_response_id : str | None
-        ID of a prior response for multi-turn conversation continuity.
-
-    Returns
-    -------
-    tuple[str, str]
-        (final_response_output_text, final_response_id)
-    """
-    # TODO: Track total duration, cost metrics, and tool_calls_made count
-    # and return them from run_assistant for use in eval_assistant.py CSV output
+) -> AgentResult:
 
     client = OpenAI(api_key=os.environ.get("OPENAI_API_KEY"))
 
     MODEL_DEFAULTS = {
         "instructions": INSTRUCTIONS,
-        "model": "gpt-5-nano",  # 400,000 token context window
-        # A summary of the reasoning performed by the model. This can be useful for debugging and understanding the model's reasoning process.
+        "model": MODEL_NAME,
+        # TODO: Flip "summary" to "auto" once this org is confirmed verified with OpenAI
         "reasoning": {"effort": "low", "summary": None},
-        "tools": SEARCH_TOOLS_SCHEMA,
+        "tools": [tool.schema() for tool in TOOLS],
     }
 
-    # TOOLS_SCHEMA tells the model what tools exist and what arguments to generate.
-    # tool_mapping wires those tool names to the Python functions that execute them.
-    # They are separate because the model generates arguments (schema concern) but
-    # cannot supply request-time values like user (mapping concern).
-    tool_mapping = make_search_tool_mapping(user)
-
-    if not previous_response_id:
-        response = client.responses.create(
-            input=[
-                {"type": "message", "role": "user", "content": str(message)}
-            ],
-            **MODEL_DEFAULTS,
-        )
-    else:
-        response = client.responses.create(
+    if previous_response_id:
+        initial_response = client.responses.create(
             input=[
                 {"type": "message", "role": "user", "content": str(message)}
             ],
             previous_response_id=str(previous_response_id),
             **MODEL_DEFAULTS,
         )
 
-    return handle_tool_calls_with_reasoning(response, client, MODEL_DEFAULTS, tool_mapping)
+        # search_documents needs the request user for document access control
+        return run_agentic_loop(initial_response, client, MODEL_DEFAULTS, TOOLS, user)
+
+    initial_response = client.responses.create(
+        input=[
+            {"type": "message", "role": "user", "content": str(message)}
+        ],
+        **MODEL_DEFAULTS,
+    )
+
+    # search_documents needs the request user for document access control
+    return run_agentic_loop(initial_response, client, MODEL_DEFAULTS, TOOLS, user)

server/api/views/assistant/assistant_types.py

@@ -0,0 +1,98 @@
+from dataclasses import dataclass
+from enum import Enum
+from typing import Callable
+
+@dataclass(frozen=True)
+class Tool:
+    """
+    Instances are registered in tool_services.py's TOOLS list.
+    """
+    name: str
+    description: str
+    parameters: dict
+    # Function we run: run(user, **arguments) -> str. 
+    # Every tool takes the request `user` so the dispatch loop can call them uniformly; 
+    # A tool that doesn't need it simply ignores it.
+    run: Callable
+
+    # Schema that the model sees:  Flattened Responses-API shape
+    def schema(self) -> dict:
+        return {
+            "type": "function",
+            "name": self.name,
+            "description": self.description,
+            "parameters": self.parameters,
+        }
+
+
+class ToolCallStatus(str, Enum):
+    """
+    Evaluate tool selection and distinguish between FAILED and UNREGISTERED
+    """
+
+    OK = "ok"
+    # Tool matched but raised an error 
+    FAILED = "failed"
+    # No tool registered for the model's requested name:
+    # The model asked for a tool name we don't have    
+    UNREGISTERED = "unregistered"  
+
+
+@dataclass(frozen=True)
+class ToolCallExecution:
+    """
+    A record of one tool call the model made
+    """
+
+    name: str
+    # `output` and `error` are disjoint by status
+    status: ToolCallStatus
+    # the query the model generated (the primary tool selection signal)
+    # None only when the model's argument JSON could not be parsed
+    arguments: dict | None = None   
+    # the tool's result on success (retrieved content)
+    output: str | None = None
+    # the failure detail when status is not OK    
+    error: str | None = None        
+
+@dataclass(frozen=True)
+class TurnUsage:
+    """
+    Token usage for one responses.create call
+
+    Every count is int | None, where None means *unknown* — response.usage was absent
+    or an unrecognized shape. Not 0, which also means "used no tokens": fake zeros drag
+    down averages and make a run that failed after being billed look free.
+
+    There is no total_tokens field. It is input_tokens + output_tokens, so a stored
+    copy is a sixth number that can disagree with the ones determining it.
+    """
+
+    # Cross-references this turn against OpenAI's logs
+    response_id: str
+    input_tokens: int | None
+    # A subset of input_tokens, not an addend — the prefix stops a later cost
+    # calculation from double-counting. Every turn resends the context via
+    # previous_response_id, which is what makes this the interesting number.
+    cached_input_tokens: int | None
+    output_tokens: int | None
+    # A subset of output_tokens, not an addend
+    reasoning_output_tokens: int | None
+
+@dataclass(frozen=True)
+class AgentResult:
+    """
+    # Built by the agentic loop as a run proceeds, 
+    # and read by eval_assistant.py to fill the result CSV
+    """
+
+    # The model's final text
+    output_text: str
+    # The id of the final response (for multi-turn continuity)
+    response_id: str
+    # The ordered ToolCallExecution records for every tool invocation across all loop iterations
+    tool_calls: list[ToolCallExecution]
+    # One per loop iteration. Required rather than defaulted: a silent [] would report
+    # turn_count 0 for a run that had turns. The eval derives turn_count = len(turns)
+    # and the token totals = sums over them, so the two cannot drift apart.
+    turns: list[TurnUsage]