diff --git a/docs/agents/turns.md b/docs/agents/turns.md index 2224e44..1af853a 100644 --- a/docs/agents/turns.md +++ b/docs/agents/turns.md @@ -457,15 +457,17 @@ sequenceDiagram message starts afresh: the cancelled prompt is over. - The cancellation reaches the turn's work at once. Restate shuts down an attempt it has heard nothing from for the inactivity timeout (a minute by - default), and the next attempt runs the turn from its start. So while the - turn works on its own (a model request, a tool), its attempt tells Restate - every 20 seconds that it makes progress, and Restate keeps it however long - the step takes. While the turn waits for an answer of Restate's instead (a - tool waiting for an approval, a call), it tells Restate nothing: Restate - shuts the attempt down after the timeout, so that a wait of days holds no - process, and the next attempt gets the cancellation as it starts. A - deployment that sets an inactivity timeout under 20 seconds is shut down - between two of those messages. + default), and the next attempt runs the turn from its start. So while a + model request runs, or a framework's built-in local tool (a shell, file + edits), the attempt tells Restate every 20 seconds that it makes progress + (`ctx.working`), and Restate keeps it however long the step takes. Nothing + else of a turn does: a tool of your own only with `tool(keepalive=True)`. + While the turn waits for an answer of Restate's (a tool waiting for an + approval, a call), it tells Restate nothing: Restate shuts the attempt down + after the timeout, so that a wait of days holds no process, and the next + attempt gets the cancellation as it starts. A deployment that sets an + inactivity timeout under 20 seconds is shut down between two of those + messages. - A cancellation that comes once the turn answered changes nothing: the answer stands. - A tool cut short by the stop raises the stop's reason, so the model never diff --git a/python/restate/agents/_tools.py b/python/restate/agents/_tools.py index de02906..c892a01 100644 --- a/python/restate/agents/_tools.py +++ b/python/restate/agents/_tools.py @@ -126,6 +126,7 @@ class Tool: description: What the tool does, for the model. input_schema: The JSON schema of the tool's input, which the framework passes to the model. + keepalive: Whether the tool runs in ``ctx.working``: see ``tool``. """ name: str @@ -133,6 +134,7 @@ class Tool: input_schema: dict[str, Any] _run: Callable[[dict[str, Any], ToolCall], Awaitable[Any]] _validate: Callable[[dict[str, Any]], Any] + keepalive: bool = False async def run(self, args: object, call: ToolCall) -> str: """Run the tool with the model's raw arguments, and return its result as text for the model. @@ -157,6 +159,9 @@ async def run(self, args: object, call: ToolCall) -> str: except Exception as error: # pydantic's ValidationError, for the model to fix return f"Error: the arguments do not fit the tool's input: {error}" try: + if self.keepalive: + async with call.ctx.working(): + return _text(await self._run(arguments, call)) return _text(await self._run(arguments, call)) except BaseException as error: # The turn stopped (its invocation was cancelled, or its attempt @@ -219,15 +224,23 @@ def _text(value: object) -> str: @typing.overload -def tool(fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None) -> Tool: ... +def tool( + fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None, keepalive: bool = False +) -> Tool: ... @typing.overload -def tool(*, name: str | None = None, description: str | None = None) -> Callable[[Callable[..., Any]], Tool]: ... +def tool( + *, name: str | None = None, description: str | None = None, keepalive: bool = False +) -> Callable[[Callable[..., Any]], Tool]: ... def tool( - fn: Callable[..., Any] | None = None, *, name: str | None = None, description: str | None = None + fn: Callable[..., Any] | None = None, + *, + name: str | None = None, + description: str | None = None, + keepalive: bool = False, ) -> Tool | Callable[[Callable[..., Any]], Tool]: """Create a tool from a function, sync or async. @@ -251,12 +264,19 @@ async def get_weather(city: str) -> str: result, and can act on it. Any other exception fails the attempt, and Restate retries it. + A tool does not keep its attempt alive: one that works for longer than + Restate's inactivity timeout (a minute, by default) is cut off, as by a + crash. One that must work that long sets ``keepalive``, and runs in + ``ctx.working``; it keeps long waits on Restate (a call, a signal) out of + it, or holds its process for as long as they take. + Args: fn: The function. Omit it to use ``tool(name=..., description=...)`` as a decorator. name: The tool's name. Defaults to the function's name. description: What the tool does, for the model. Defaults to the function's docstring. + keepalive: Keep the attempt alive while the tool runs. Returns: The tool, or a decorator that creates it. @@ -267,7 +287,7 @@ async def get_weather(city: str) -> str: if fn is None: def decorate(fn: Callable[..., Any]) -> Tool: - return tool(fn, name=name, description=description) + return tool(fn, name=name, description=description, keepalive=keepalive) return decorate pydantic = _pydantic() @@ -295,6 +315,7 @@ def validate(args: dict[str, Any]) -> dict[str, Any]: input_schema=typing.cast(dict[str, Any], model.model_json_schema()), _run=run, _validate=validate, + keepalive=keepalive, ) diff --git a/python/restate/agents/adk/_agent.py b/python/restate/agents/adk/_agent.py index 8aad3f6..c79dc2a 100644 --- a/python/restate/agents/adk/_agent.py +++ b/python/restate/agents/adk/_agent.py @@ -47,6 +47,10 @@ with ``include_contents='none'`` starts each request at the latest message of the user: the requests after the one that takes a steered message start at it, without the prompt and the tools' results before it. +- The attempt is kept alive (``ctx.working``) while a model request runs (not the steering and commits around it) + and while one of ADK's built-in local tools does (the shell and the environment's tools: ``_LOCAL_TOOLS``), + not while one of yours does: a ``FunctionTool``, or a ``tool`` of ours unless it has ``keepalive``. A code + executor of an agent (``code_executor``) is not covered. - A run of more model calls than its ``RunConfig``'s ``max_llm_calls`` (500 by default) fails its prompt for good: a retry would carry on with a fresh count. - An agent or ``App`` of your own runs as you configured it, with the @@ -75,6 +79,7 @@ import time import typing from collections.abc import Awaitable, Callable, Sequence +from contextlib import AsyncExitStack from typing import Any, Unpack from google.adk.agents import BaseAgent, LlmAgent, RunConfig @@ -90,6 +95,8 @@ from google.adk.sessions import BaseSessionService, Session from google.adk.sessions.base_session_service import GetSessionConfig, ListSessionsResponse from google.adk.tools.base_tool import BaseTool +from google.adk.tools.bash_tool import ExecuteBashTool +from google.adk.tools.skill_toolset import RunSkillScriptTool from google.adk.tools.tool_context import ToolContext from google.adk.workflow import BaseNode from google.genai import types @@ -265,14 +272,59 @@ async def add_stored(self, session: Session, event: Event) -> None: await super().append_event(session, event) +_LOCAL_TOOLS = (ExecuteBashTool, RunSkillScriptTool) +"""ADK's built-in tools that work on the local machine (a shell, scripts), which run kept alive; and the +environment's (``EnvironmentToolset``: execute, read, write and edit a file), whose classes ADK keeps in private +modules, so they are known by their package (``_ENVIRONMENT_TOOLS``). + +Only these: not ours (``_AdkTool``), a user's own function (``FunctionTool``, which includes +``LongRunningFunctionTool``), a sub-agent (``AgentTool``, whose model requests are kept alive by the plugin), nor +the tools that call a service (MCP, OpenAPI, search, ...). Code executors (``LlmAgent.code_executor``) run as a +processor of the flow, not as a tool.""" + +_ENVIRONMENT_TOOLS = "google.adk.tools.environment." +"""The package of the environment's tools (``EnvironmentToolset``).""" + + +def _local(tool: BaseTool) -> bool: + """Whether ``tool`` is one of ADK's built-in local tools (see ``_LOCAL_TOOLS``).""" + return isinstance(tool, _LOCAL_TOOLS) or type(tool).__module__.startswith(_ENVIRONMENT_TOOLS) + + class _Restate(BasePlugin): - """Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, and give the model the - messages steered to the turn.""" + """Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, give the model the + messages steered to the turn, and keep the attempt alive while a model request or a built-in local tool runs. + + A scope (``ctx.working``) is opened by one hook and closed by another, so each is kept per request or per call: + the run closes what is left when it ends (``release_all``).""" def __init__(self, ctx: SessionContext, sessions: _Sessions) -> None: super().__init__(name="restate") self.ctx = ctx self.sessions = sessions + self.requests: dict[tuple[str, str, str], AsyncExitStack] = {} + """The scopes of the model requests in progress, by invocation, branch and agent: parallel agents each + have one.""" + self.local: dict[str, AsyncExitStack] = {} + """The scopes of the built-in local tool calls in progress, by call id: tools run in parallel.""" + + async def _keep(self) -> AsyncExitStack: + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + return stack + + @staticmethod + async def _close(stack: AsyncExitStack | None) -> None: + if stack is not None: + await stack.aclose() + + async def release_all(self) -> None: + """Close the scopes still open, as the run ends: whatever way, success, error or a stopped turn.""" + stacks = [*self.requests.values(), *self.local.values()] + self.requests.clear() + self.local.clear() + for stack in stacks: + await self._close(stack) async def before_model_callback( self, *, callback_context: CallbackContext, llm_request: LlmRequest @@ -287,16 +339,61 @@ async def before_model_callback( await self.sessions.add_stored(callback_context.session, event) if event.content is not None: llm_request.contents.append(event.content.model_copy(deep=True)) + # Kept alive from here, once the steering is done: the request is what follows (ADK resolves the model, runs + # the agent's own callbacks, and calls it). Closed by the response (``after_model_callback``), an error + # (``on_model_error_callback``), or the event that ends the step if a callback answered instead. + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + self.requests[key] = await self._keep() + return None + + async def after_model_callback( + self, *, callback_context: CallbackContext, llm_response: LlmResponse + ) -> LlmResponse | None: + # A streamed reply comes as partial responses, then its whole: the request is on until the whole. + if not llm_response.partial: + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + return None + + async def on_model_error_callback( + self, *, callback_context: CallbackContext, llm_request: LlmRequest, error: Exception + ) -> LlmResponse | None: + # Also called for a request that is cancelled (``error`` is the ``CancelledError``). + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + return None + + async def on_event_callback(self, *, invocation_context: Any, event: Event) -> Event | None: + # Backstop for what the callbacks miss: a response that a callback of the agent gave in place of the + # model's (no after or error callback runs then), or a tool call that one answered. An event of the + # agent that is whole ends its request, and a response to a call ends the call's scope. + if not event.partial: + await self._close(self.requests.pop(_request_key(event.invocation_id, event.branch, event.author), None)) + for response in event.get_function_responses(): + await self._close(self.local.pop(response.id or "", None)) return None async def before_tool_callback( self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext ) -> None: await tool_call_started(self.ctx) + if _local(tool): + call_id = tool_context.function_call_id or "" + await self._close(self.local.pop(call_id, None)) + self.local[call_id] = await self._keep() + + async def after_tool_callback( + self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, result: dict[str, Any] + ) -> dict[str, Any] | None: + await self._close(self.local.pop(tool_context.function_call_id or "", None)) + return None async def on_tool_error_callback( self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, error: Exception ) -> dict[str, Any] | None: + # Also called for a call that is cancelled. + await self._close(self.local.pop(tool_context.function_call_id or "", None)) if type(tool) is BaseTool: # A tool the agent does not have (ADK stands in a bare BaseTool for it): the model's to fix. Failing # the attempt would not help, as the retry resumes the same call. @@ -306,6 +403,10 @@ async def on_tool_error_callback( return None +def _request_key(invocation: str, branch: str | None, agent: str) -> tuple[str, str, str]: + return (invocation, branch or "", agent) + + # --- Tools ---------------------------------------------------------------------------------------- @@ -467,7 +568,8 @@ async def run(conversation: Conversation) -> str: sessions = _Sessions(conversation) # This package's plugin, after the app's own: a plugin of the app that answers in the model's place leaves # the messages steered to the turn to a request that goes out. - plugins = [*app.plugins, _Restate(ctx, sessions)] + restate = _Restate(ctx, sessions) + plugins = [*app.plugins, restate] runner = Runner(app=app.model_copy(update={"plugins": plugins}), session_service=sessions) answer = "" error: Event | None = None @@ -488,6 +590,10 @@ async def run(conversation: Conversation) -> str: except LlmCallsLimitExceededError as limit: # A retry would carry on with a fresh count, and so would never end an agent that keeps calling tools. raise TerminalError(f"the run reached its limit of model calls: {limit}") from limit + finally: + # The scopes the plugin's hooks left open (a callback of the agent answered in place of the model or a + # tool, a failed or stopped run): the attempt is not kept alive past the run. + await restate.release_all() if error is not None: # The model would not answer (a safety block, say), as it would not again: the prompt fails. raise TerminalError(f"the model returned {error.error_code}: {error.error_message or ''}") @@ -768,9 +874,11 @@ async def summarize(ctx: SessionContext, request: str) -> str: llm = LlmAgent(name="summarizer", model=_model(await configured(model, ctx))).canonical_model asked = LlmRequest(model=llm.model, contents=[types.Content(role="user", parts=[types.Part(text=request)])]) texts: list[str] = [] - async for response in llm.generate_content_async(asked): - if response.content is not None: - texts.extend(part.text for part in response.content.parts or [] if part.text) + # Kept alive for the request alone. + async with ctx.working(): + async for response in llm.generate_content_async(asked): + if response.content is not None: + texts.extend(part.text for part in response.content.parts or [] if part.text) return "".join(texts) return summarize diff --git a/python/restate/agents/langgraph/_agent.py b/python/restate/agents/langgraph/_agent.py index 8076f31..d52434f 100644 --- a/python/restate/agents/langgraph/_agent.py +++ b/python/restate/agents/langgraph/_agent.py @@ -351,7 +351,9 @@ async def call_model(state: State) -> dict[str, Any]: # request has them after the results, and the state keeps them for the rest of the run, which goes on. steered = typing.cast(list[BaseMessage], await take_steers(ctx, conversation)) messages = _latest([*state["messages"], *steered], window) - reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages) + # Kept alive for the request alone: not the steering before it, nor the commit after. + async with ctx.working(): + reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages) await conversation.turn.append_reply(reply) return {"messages": [*steered, reply]} @@ -651,7 +653,8 @@ def _summarizer(model: PerSession[ChatModel | None]) -> Summarize: async def summarize(ctx: SessionContext, request: str) -> str: chosen = await configured(model, ctx) chat_model = chosen if isinstance(chosen, BaseChatModel) else _chat_model(model_of(chosen)) - return _text(await chat_model.ainvoke([HumanMessage(request)])) + async with ctx.working(): + return _text(await chat_model.ainvoke([HumanMessage(request)])) return summarize diff --git a/python/restate/agents/langgraph/_graph.py b/python/restate/agents/langgraph/_graph.py index 232b8e1..8047b94 100644 --- a/python/restate/agents/langgraph/_graph.py +++ b/python/restate/agents/langgraph/_graph.py @@ -106,9 +106,12 @@ def forget(state: MessagesState) -> dict[str, Any]: import asyncio import json import typing -from collections.abc import Sequence +from collections.abc import Callable, Coroutine, Sequence +from contextlib import AsyncExitStack from typing import Any +from uuid import UUID +from langchain_core.callbacks import AsyncCallbackHandler from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage from langchain_core.runnables import RunnableConfig from langgraph.errors import InvalidUpdateError @@ -366,6 +369,78 @@ async def _stored(app: Any) -> StateSnapshot: return typing.cast(StateSnapshot, await app.aget_state(latest.config)) +class _KeepAlive(AsyncCallbackHandler): + """Keep the attempt alive (``ctx.working``) while a model request of the graph's runs, and only then. + + LangChain awaits a handler's ``on_chat_model_start`` (or ``on_llm_start``) before it sends the request, and + ``on_llm_end`` or ``on_llm_error`` (also for a cancelled request) once it is over, so the scope is the request + alone. Each is kept by the request's run id: a graph's nodes run models in parallel. It is a handler that its + runs inherit, so that the models a node, a tool or a subgraph calls are in it too. + + A scope the callbacks leave open (a request that never ended) is closed as the run's stream ends + (``release_all``). + + A sync request (``invoke``, in a sync node) has LangChain run these callbacks on a loop of its own, in a thread, + and wait there for every task they started: a scope is the attempt's, with a heartbeat on the attempt's loop, so + it is opened and closed there instead, handed over without waiting (the attempt's loop may be the one the sync + request blocks). A sync request on the attempt's own loop blocks its heartbeat anyway. + """ + + run_inline = True + """Awaited before the request, in order, rather than gathered with the other handlers.""" + + def __init__(self, ctx: SessionContext) -> None: + self.ctx = ctx + self.requests: dict[UUID, AsyncExitStack] = {} + self.loop = asyncio.get_running_loop() + """The attempt's loop: the one ``ctx.working`` runs on.""" + self._stream = 0 + """The stream of the run in progress: ``release_all`` ends one, and the run may stream again (after an + interrupt).""" + + async def _on_loop(self, work: Callable[[], Coroutine[Any, Any, None]]) -> None: + """Run ``work`` on the attempt's loop: now if this is it, or else handed over (in order) without waiting.""" + if asyncio.get_running_loop() is self.loop: + await work() + else: + asyncio.run_coroutine_threadsafe(work(), self.loop) + + async def _open(self, run_id: UUID, stream: int) -> None: + if stream != self._stream: + # A request whose start reached the attempt's loop after its stream ended: nothing to keep alive. + return + await self._close(run_id) + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + self.requests[run_id] = stack + + async def _close(self, run_id: UUID) -> None: + stack = self.requests.pop(run_id, None) + if stack is not None: + await stack.aclose() + + async def release_all(self) -> None: + self._stream += 1 + stacks = list(self.requests.values()) + self.requests.clear() + for stack in stacks: + await stack.aclose() + + async def on_chat_model_start(self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any) -> None: + stream = self._stream + await self._on_loop(lambda: self._open(run_id, stream)) + + async def on_llm_start(self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any) -> None: + stream = self._stream + await self._on_loop(lambda: self._open(run_id, stream)) + + async def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + await self._on_loop(lambda: self._close(run_id)) + + async def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + await self._on_loop(lambda: self._close(run_id)) + + def run_of(ctx: SessionContext, graph: AnyGraph, context: Any = None) -> Run: """Return the run of a prompt: the graph's, on the key's thread, from the prompt to its answer. @@ -394,7 +469,13 @@ async def _run(conversation: Conversation) -> str: before = str(metadata.get(_STRUCTURED_BEFORE, "")) if asked else await _structured_version(app) # Each checkpoint of the run names its prompt, so that a retried attempt finds the run it cut off, and the # version its structured response had before it. - config: RunnableConfig = {**THREAD, "metadata": {_PROMPT: prompt_id, _STRUCTURED_BEFORE: before}} + alive = _KeepAlive(ctx) + config: RunnableConfig = { + **THREAD, + "metadata": {_PROMPT: prompt_id, _STRUCTURED_BEFORE: before}, + # Inherited by the nodes' own runs: the models they call keep the attempt alive while they request. + "callbacks": [alive], + } if asked and not snapshot.next: # The run had finished, but its answer was not stored: that is the answer, rather than run again. await _copied(ctx, turn, known, messages, prompt_id) @@ -425,6 +506,9 @@ async def _run(conversation: Conversation) -> str: # No refusal, whatever its name: a graph's models, tools and nodes are all its own. from_tool(ctx, error) raise + finally: + # Before any wait for an interrupt: none is in a scope. Closes what a request left open. + await alive.release_all() # The run stops at its interrupts: it goes on with the signals that answer them. paused = await _stored(app) resumes = await _resumed(ctx, paused) diff --git a/python/restate/agents/openai/_agent.py b/python/restate/agents/openai/_agent.py index 4a4a10d..365d39b 100644 --- a/python/restate/agents/openai/_agent.py +++ b/python/restate/agents/openai/_agent.py @@ -114,6 +114,7 @@ import json import typing from collections.abc import Awaitable, Sequence +from contextlib import AsyncExitStack from typing import Any, Unpack from agents import ( @@ -320,9 +321,31 @@ def __init__(self, ctx: SessionContext, conversation: Conversation, config: RunC """The model request in progress, timed in the journal.""" self.calls: dict[str, Annotation] = {} """The tool calls in progress, timed in the journal, by call id.""" - - def ended(self, error: BaseException | None) -> None: - """End the timing of the model request and tool calls still in progress, as the run ends with ``error``.""" + self.alive: AsyncExitStack | None = None + """The ``ctx.working`` scope of the model request in progress: just the request, not the work around it.""" + self.local: dict[str, AsyncExitStack] = {} + """The ``ctx.working`` scopes of the built-in local tool calls in progress, by call id.""" + + async def _keep(self) -> AsyncExitStack: + """Return a scope that keeps the attempt alive until closed.""" + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + return stack + + async def _release(self) -> None: + """Close the model request's scope, if open.""" + stack, self.alive = self.alive, None + if stack is not None: + await stack.aclose() + + async def ended(self, error: BaseException | None) -> None: + """End the timing of the model request and tool calls still in progress, and close the scopes still open, as + the run ends with ``error``.""" + stacks = [*self.local.values(), *([self.alive] if self.alive is not None else [])] + self.local.clear() + self.alive = None + for stack in stacks: + await stack.aclose() timed = [*self.calls.values(), *([self.request] if self.request is not None else [])] self.request = None self.calls.clear() @@ -339,6 +362,9 @@ async def on_llm_start( # Never so: a run makes one request at a time. self.request.fail("another request started") self.request = annotate(self.ctx, f"model: {_model_name(self.model or agent.model)}") + # Kept alive for the request alone, as this hook ends: the steering and commits before it are not it. + await self._release() + self.alive = await self._keep() async def model_input(self, data: CallModelData[Any]) -> ModelInputData: """The run's ``call_model_input_filter``: store the tools' outputs before the next request, and take the @@ -381,6 +407,7 @@ async def _filtered(self, data: CallModelData[Any]) -> ModelInputData: async def on_llm_end(self, context: RunContextWrapper[Any], agent: Agent[Any], response: ModelResponse) -> None: # The request ends here, before the reply is stored. + await self._release() if self.request is not None: self.request.end(_usage(agent, response)) self.request = None @@ -392,16 +419,26 @@ async def on_tool_start(self, context: RunContextWrapper[Any], agent: Agent[Any] self.running += 1 await tool_call_started(self.ctx) # Timed from here: the settling of the earlier calls is not the tool's. - self.calls[_call_key(context, tool)] = annotate(self.ctx, f"tool: {tool.name}") + key = _call_key(context, tool) + self.calls[key] = annotate(self.ctx, f"tool: {tool.name}") + if not isinstance(tool, FunctionTool): + # The SDK's own tools (shell, apply_patch, computer...) run local work: kept alive. A function tool is + # the user's, and is not. + self.local[key] = await self._keep() async def on_handoff(self, context: RunContextWrapper[Any], from_agent: Agent[Any], to_agent: Agent[Any]) -> None: # The turn carries the agent it handed off to, even if it ends before that agent replies. self.conversation.turn.carry(agent=to_agent.name) async def on_tool_end(self, context: RunContextWrapper[Any], agent: Agent[Any], tool: SdkTool, result: Any) -> None: - if (call := self.calls.pop(_call_key(context, tool), None)) is not None: - call.end() - self.running -= 1 + key = _call_key(context, tool) + try: + if (stack := self.local.pop(key, None)) is not None: + await stack.aclose() + finally: + if (call := self.calls.pop(key, None)) is not None: + call.end() + self.running -= 1 def _call_key(context: RunContextWrapper[Any], tool: SdkTool) -> str: @@ -759,9 +796,9 @@ async def _go( raise except BaseException as error: # The model request or tool calls in progress failed with the run. - hooks.ended(error) + await hooks.ended(error) raise - hooks.ended(None) + await hooks.ended(None) # The outputs that no model request stored: those of a tool that ends the turn, say. if outputs := conversation.new_results(result.to_input_list()): await turn.append(*outputs) @@ -1102,7 +1139,8 @@ def _summarizer(model: PerSession[AnyModel | None]) -> Summarize: async def summarize(ctx: SessionContext, request: str) -> str: chosen = _model(await configured(model, ctx)) - result = await Runner.run(Agent(name="summarizer"), request, run_config=RunConfig(model=chosen)) + async with ctx.working(): + result = await Runner.run(Agent(name="summarizer"), request, run_config=RunConfig(model=chosen)) return str(result.final_output) return summarize diff --git a/python/restate/agents/pydantic_ai/_agent.py b/python/restate/agents/pydantic_ai/_agent.py index 50b4b9d..4166ffe 100644 --- a/python/restate/agents/pydantic_ai/_agent.py +++ b/python/restate/agents/pydantic_ai/_agent.py @@ -101,7 +101,8 @@ import json import sys import typing -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from contextlib import AsyncExitStack, asynccontextmanager from typing import Any, Unpack from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -118,7 +119,8 @@ ToolReturnPart, UserPromptPart, ) -from pydantic_ai.models import Model, ModelRequestContext, infer_model +from pydantic_ai.models import Model, ModelRequestContext, StreamedResponse, infer_model +from pydantic_ai.models.wrapper import WrapperModel from pydantic_ai.tools import ToolDefinition from pydantic_ai.toolsets import FunctionToolset from pydantic_ai.usage import UsageLimits @@ -139,6 +141,19 @@ estimate_context_tokens, resolve_context_window, ) +from pydantic_ai_harness.filesystem import ( + CREATE_DIRECTORY_KIND, + EDIT_FILE_KIND, + FILE_INFO_KIND, + FIND_FILES_KIND, + GREP_KIND, + LIST_DIRECTORY_KIND, + LIST_FILES_KIND, + READ_FILE_KIND, + SEARCH_FILES_KIND, + WRITE_FILE_KIND, +) +from pydantic_ai_harness.shell import SHELL_TOOL_NAMES from pydantic_core import to_json, to_jsonable_python import pydantic_ai @@ -347,6 +362,50 @@ def _feedback_of(message: ModelMessage) -> list[str]: return [to_json(p, bytes_mode="base64").decode() for p in message.parts if isinstance(p, RetryFeedbackPart)] +LOCAL_KINDS: frozenset[str] = frozenset( + kind.id + for kind in ( + READ_FILE_KIND, + WRITE_FILE_KIND, + EDIT_FILE_KIND, + LIST_DIRECTORY_KIND, + SEARCH_FILES_KIND, + FIND_FILES_KIND, + CREATE_DIRECTORY_KIND, + FILE_INFO_KIND, + LIST_FILES_KIND, + GREP_KIND, + ) +) +"""The ``tool_kind`` of the harness's built-in local tools that this package keeps alive while they run (the file +system's). The harness's shell tools declare none: they are known by the names the harness gives them +(``SHELL_TOOL_NAMES``). ``code_mode``'s ``run_code`` runs the user's tools inside it, so it is not kept alive.""" + + +def _local(tool_def: ToolDefinition) -> bool: + """Whether ``tool_def`` is a built-in local tool of the harness: its file system's, or its shell's.""" + return tool_def.tool_kind in LOCAL_KINDS or tool_def.name in SHELL_TOOL_NAMES + + +class _Alive(WrapperModel): + """A model whose requests keep the attempt alive (``ctx.working``) while they run: exactly the request, so the + stores and steering around it are outside, and a wait on Restate never is inside.""" + + def __init__(self, wrapped: Model, ctx: SessionContext) -> None: + super().__init__(wrapped) + self.ctx = ctx + + async def request(self, *args: Any, **kwargs: Any) -> ModelResponse: + async with self.ctx.working(): + return await super().request(*args, **kwargs) + + @asynccontextmanager + async def request_stream(self, *args: Any, **kwargs: Any) -> AsyncIterator[StreamedResponse]: + # In scope until the stream ends, as the caller reads it inside the ``async with``. + async with self.ctx.working(), super().request_stream(*args, **kwargs) as stream: + yield stream + + class _Restate(AbstractCapability[Any]): """Make a Pydantic AI run durable in a Restate handler. @@ -363,6 +422,8 @@ def __init__(self, ctx: SessionContext, conversation: Conversation, history: Seq self.conversation = conversation self.model_errors = ErrorSet() """The errors that the run's model requests failed with.""" + self.local: dict[str, AsyncExitStack] = {} + """The ``ctx.working`` scopes of the built-in local tool calls in progress, by call id.""" self.feedback = {feedback for m in history for feedback in _feedback_of(m)} """Pydantic AI's feedback that the conversation has (each ``RetryFeedbackPart``), as JSON.""" @@ -408,18 +469,34 @@ async def after_model_request( await self.conversation.turn.append_reply(response) return response + async def _release(self, call: ToolCallPart) -> None: + if (stack := self.local.pop(call.tool_call_id, None)) is not None: + await stack.aclose() + + async def release(self) -> None: + """Close the scopes still open, as the run ends.""" + stacks = list(self.local.values()) + self.local.clear() + for stack in stacks: + await stack.aclose() + async def before_tool_execute( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any ) -> Any: # The stores of this package write once per tool call (``_stores.py``). calling(call.tool_call_id) await tool_call_started(self.ctx) + if _local(tool_def): + # A built-in local tool (the file system's, the shell's): kept alive while it runs, from here to its end. + stack = self.local[call.tool_call_id] = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) return args async def after_tool_execute( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any, result: Any ) -> Any: calling(None) + await self._release(call) return result async def handle_deferred_tool_calls( @@ -453,6 +530,7 @@ async def on_tool_execute_error( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any, error: Exception ) -> Any: calling(None) + await self._release(call) stop = stop_signal_of(self.ctx) if stop is not None and stop.reason is not None: raise stop.reason from error @@ -794,7 +872,7 @@ async def run(conversation: Conversation) -> str: with tools_in(ctx): result = await agent.run( message_history=history, - model=model, + model=_Alive(model, ctx), toolsets=toolsets, capabilities=[restate, *chosen, *snapshots, steering], output_type=spec, @@ -824,6 +902,9 @@ async def run(conversation: Conversation) -> str: # Not a model request's: no refusal, whatever its name (an instructions function's, say). from_tool(ctx, error) raise + finally: + # Whatever scope a tool's end hook missed (a cancelled run, say). + await restate.release() # The messages that no model request stored: an output tool's return, say. await restate.store(result.all_messages()) if isinstance(result.output, DeferredToolRequests): @@ -1128,7 +1209,7 @@ async def compact_history(ctx: SessionContext, history: list[Any]) -> list[Any] compacted = await compact_now( compaction(context_window, target=BETWEEN), messages, - model=_model(named), + model=_Alive(_model(named), ctx), conversation_id=conversation_id_of(ctx), ) return None if compacted == messages else _snapshot_of(compacted) diff --git a/python/restate/sessions/_api.py b/python/restate/sessions/_api.py index 7b8970b..25a3ac7 100644 --- a/python/restate/sessions/_api.py +++ b/python/restate/sessions/_api.py @@ -21,6 +21,7 @@ import typing from collections.abc import Awaitable, Callable, Sequence +from contextlib import AbstractAsyncContextManager from datetime import timedelta from typing import Any, Protocol @@ -352,6 +353,26 @@ def cancel(self, invocation_id: str) -> None: """ ... + def working(self) -> AbstractAsyncContextManager[None]: + """Keep the attempt alive while the body of an ``async with`` runs. + + For work outside Restate that may outlast Restate's inactivity timeout + (a minute, by default), such as a model's request:: + + async with ctx.working(): + reply = await model.generate(prompt) + + Outside it, an attempt Restate hears nothing from for that long is shut + down: it suspends on the calls it waits for, and a new attempt runs the + handler from the start when one answers (or at once, if it waits for + none). So a wait of days holds no process, and work cut off that way is + done again. Inside it, the attempt sends a heartbeat every 20 seconds, + whatever it waits for: keep a long wait on Restate (a call, a signal) + out of it. The database's own calls keep the attempt alive by + themselves. + """ + ... + class ViewContext(Caller, Protocol): """The context of a session object's view. diff --git a/python/restate/sessions/_database.py b/python/restate/sessions/_database.py index 33f16d1..c194c66 100644 --- a/python/restate/sessions/_database.py +++ b/python/restate/sessions/_database.py @@ -121,7 +121,8 @@ async def open(namespace: str, store: Store, *, reader: bool = False) -> Connect vm = getattr(store, "vm", None) name = getattr(store, "database_name", None) if vm is not None else None try: - native = await _settled(sql_open(namespace, boundary.dispatch, boundary.lists_keys, reader, vm, name)) + async with boundary.alive(): + native = await _settled(sql_open(namespace, boundary.dispatch, boundary.lists_keys, reader, vm, name)) except Exception as error: if boundary.failed is not None: raise boundary.failed from error @@ -155,6 +156,11 @@ class _Boundary: def __init__(self, store: Store, loop: asyncio.AbstractEventLoop) -> None: self.store: Store = store self.loop: asyncio.AbstractEventLoop = loop + # Keeps the attempt alive while a call of the database runs: the store of an attempt on Restate has it + # (its handler's ``working``). The database's calls are local work that never waits on Restate for long. + self._working: Callable[[], contextlib.AbstractAsyncContextManager[Any]] | None = getattr( + store, "working", None + ) # The store's optional methods (see `Store`). self._keys: Callable[[], Sequence[str] | Awaitable[Sequence[str]]] | None = getattr(store, "keys", None) self.lists_keys: bool = callable(self._keys) @@ -169,6 +175,10 @@ def _keep(self, error: BaseException) -> None: if self.failed is None: self.failed = error + def alive(self) -> contextlib.AbstractAsyncContextManager[Any]: + """A scope that keeps the attempt alive, if the store has one: around each call of the database.""" + return self._working() if self._working is not None else contextlib.nullcontext() + def dispatch(self, op: int, key: Any, value: bytes | None, reply: SqlReply | None) -> None: """Handle one request of the extension; ``key`` is a store key, or empty for a sync or a commit.""" if op in (SQL_SET, SQL_CLEAR): @@ -287,7 +297,8 @@ async def _request(self, call: Callable[[], T | Awaitable[T]]) -> T: that fails right away); it returns a value or an awaitable of one. """ try: - return await _settled(call()) + async with self._boundary.alive(): + return await _settled(call()) except Exception as error: raise self._failure(error) from None @@ -374,7 +385,8 @@ async def transaction(self) -> AsyncGenerator[ConnectionTransaction, None]: # that wrote nothing does not sync, so they run afterwards instead. self._boundary.with_sync = tx.commit_runs() try: - await _settled(self._native.commit(records)) + async with self._boundary.alive(): + await _settled(self._native.commit(records)) except Exception as error: # A failed COMMIT (a deferred constraint, say) leaves the # transaction open; without a rollback, later calls would run diff --git a/python/restate/sessions/_driver.py b/python/restate/sessions/_driver.py index 07ce799..4bc9c3e 100644 --- a/python/restate/sessions/_driver.py +++ b/python/restate/sessions/_driver.py @@ -34,7 +34,7 @@ import operator import traceback import typing -from collections.abc import Awaitable, Callable, Generator +from collections.abc import AsyncIterator, Awaitable, Callable, Generator from datetime import timedelta from typing import Any @@ -63,20 +63,14 @@ T = typing.TypeVar("T") HEARTBEAT_INTERVAL = timedelta(seconds=20) -"""How often an attempt that works outside Restate tells it that it makes progress. +"""How often an attempt in ``working`` tells Restate that it makes progress. Restate shuts down an attempt it heard nothing from for its inactivity timeout -(a minute, by default), and the next one runs the handler from the start: a -model request that takes longer would be made again by every attempt, and -never answered. So an attempt heartbeats while its handler has nothing of -Restate's outstanding (it is then working on its own: a model request, a -tool), and while an annotation is open, which says the same of work that goes -on beside a call. With an answer of Restate's outstanding and no annotation, -it does not: Restate shuts the attempt down after the timeout, and starts the -next when the answer comes, so that a wait of days holds no process. An -answer is outstanding from the moment the call is made until Restate gives -it, whether or not the handler awaits it: work done after a call whose result -is left aside gets no heartbeat until that call is answered. +(a minute, by default). That is the default: the attempt suspends on what it +waits for (or, waiting for nothing, starts again at once), and a wait of days +holds no process. Work that must not be cut off (a model request) says so with +``working``: while any of it runs, the attempt heartbeats, whatever else it +waits for. A deployment that shortens the timeout must shorten this with it. """ SYNC_KEY = "v22/sync" @@ -132,9 +126,6 @@ def __init__( # The handles a cancellation leaves to settle, and is never carried to: the invocation # ids of the calls and sends, and the reads and acks of the database's store. self._left_to_settle: set[int] = set() - # The store's among them: the database waits for those, not the handler, and they come - # promptly, so they do not count as an answer of Restate's the handler waits for. - self._store_waits: set[int] = set() # A cancellation that failed nothing the handler awaits, carried to the next thing it # does: a result of Restate's (``future``), or a call of its database # (``take_carried_cancellation``). A retry of a cancelled invocation knows of it before @@ -151,8 +142,11 @@ def __init__( self._writer: asyncio.Task[None] = asyncio.ensure_future(self._write_output()) self._pump: asyncio.Task[None] | None = None self._handler: asyncio.Future[bytes] | None = None - self._open_traces: int = 0 + # The ``working`` scopes open: the attempt heartbeats while there is one. + self._scopes: int = 0 self._heartbeats: asyncio.Task[None] | None = None + # When the attempt last wrote to Restate, which restarts Restate's inactivity timer. + self._wrote_at: float = self._loop.time() self._end_listeners: list[Callable[[], None]] = [] self.finished: asyncio.Event = asyncio.Event() self.ended: BaseException | None = None @@ -193,7 +187,6 @@ async def run(self, handler: Callable[[], Awaitable[bytes]]) -> None: (a model request, say) stops, since nothing it does would count any more. """ self._handler = asyncio.ensure_future(self._cancelled_calls_first(handler)) - self._heartbeats = asyncio.ensure_future(self._heartbeat()) try: output = await self._handler self._last(lambda vm: vm.end_success(output)) @@ -304,13 +297,13 @@ def _tick(self) -> None: output = self.vm.take_output() if output: self._output.put_nowait(output) + self._wrote_at = self._loop.time() if self.ended is not None: return try: while (notification := self.vm.take_notification()) is not None: handle, value = notification self._left_to_settle.discard(handle) - self._store_waits.discard(handle) call = self._unanswered.pop(handle, None) if call is not None: call.answered = True @@ -409,7 +402,6 @@ def store_future(self, handle: int) -> asyncio.Future[Any]: with a journal codec. """ self._left_to_settle.add(handle) - self._store_waits.add(handle) return self._future(handle) def _future(self, handle: int) -> asyncio.Future[Any]: @@ -482,27 +474,46 @@ async def cancel_calls(self) -> list[str]: cancelled.append(invocation_id) return cancelled - def trace_opened(self) -> None: - """An annotation opened: the attempt heartbeats while any is, whatever else is outstanding.""" - self._open_traces += 1 + # --- Keeping the attempt alive - def trace_closed(self) -> None: - self._open_traces -= 1 + @contextlib.asynccontextmanager + async def working(self) -> AsyncIterator[None]: + """Keep the attempt alive while the body runs (see ``HEARTBEAT_INTERVAL``). - def _awaits_restate(self) -> bool: - """Whether an answer of Restate's that the handler made for is outstanding: a result or an invocation id.""" - return any(handle not in self._store_waits for handle in self._futures) + One heartbeat every interval, from the first scope opening to the last + closing. The first comes once the attempt has been silent for an + interval (at once, if it already has), so the silence never reaches more + than that; a scope that closes sooner (the database's, around each of its + calls) sends nothing. + + Raises: + RuntimeError: Entered on another event loop than the attempt's (a + thread's own loop): the heartbeat is the attempt's. + """ + if asyncio.get_running_loop() is not self._loop: + raise RuntimeError("ctx.working() must be entered on the attempt's own event loop, not another thread's") + self._scopes += 1 + if self._scopes == 1 and self.ended is None: + self._heartbeats = asyncio.ensure_future(self._heartbeat()) + try: + yield + finally: + self._scopes -= 1 + if self._scopes == 0 and self._heartbeats is not None: + self._heartbeats.cancel() + self._heartbeats = None async def _heartbeat(self) -> None: - """Tell Restate the attempt makes progress, while it works outside Restate (see ``HEARTBEAT_INTERVAL``).""" + """Tell Restate the attempt makes progress, while a ``working`` scope is open.""" + interval = HEARTBEAT_INTERVAL.total_seconds() + # The first once the silence reaches an interval: at once, if it already has. + await asyncio.sleep(max(0.0, interval - (self._loop.time() - self._wrote_at))) while True: - await asyncio.sleep(HEARTBEAT_INTERVAL.total_seconds()) - if self._open_traces == 0 and self._awaits_restate(): - continue try: self.act(lambda vm: vm.heartbeat()) except SdkInternalBaseException: return + await asyncio.sleep(interval) def _cancelled_error() -> TerminalError: @@ -678,13 +689,18 @@ def generic_send( def attach_invocation(self, invocation_id: str) -> Awaitable[bytes]: return self.driver.future(self.driver.act(lambda vm: vm.attach_invocation(invocation_id))) + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + """The handler's ``ctx.working``: the attempt is kept alive while its body runs.""" + return self.driver.working() + + def database_working(self) -> contextlib.AbstractAsyncContextManager[None]: + """The same, around each call of the database (see ``_Boundary.alive``).""" + return self.driver.working() + def open_annotation(self, name: str, parent: int | None = None) -> int | None: - trace = self.driver.act(lambda vm: vm.start_trace(name, parent)) - self.driver.trace_opened() - return trace + return self.driver.act(lambda vm: vm.start_trace(name, parent)) def close_annotation(self, handle: int, value: bytes, failure: str | None) -> None: - self.driver.trace_closed() try: if failure is None: self.driver.act(lambda vm: vm.end_trace_success(handle, value)) diff --git a/python/restate/sessions/_object.py b/python/restate/sessions/_object.py index e62a0d5..ba96a5c 100644 --- a/python/restate/sessions/_object.py +++ b/python/restate/sessions/_object.py @@ -30,6 +30,7 @@ async def list_notes(ctx: sessions.ViewContext) -> list[str]: from __future__ import annotations import asyncio +import contextlib import functools import inspect import json @@ -99,6 +100,11 @@ class AttemptContext(Calls, Sends): def cancel(self, invocation_id: str) -> None: self._cancel(invocation_id) + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + # The host of an attempt on Restate keeps it alive; a test's host may count the scopes; else nothing to do. + working = getattr(self._call_host, "working", None) + return working() if working is not None else contextlib.nullcontext() + @dataclass(frozen=True, eq=False) class AttemptViewContext(Calls): diff --git a/python/restate/sessions/_session.py b/python/restate/sessions/_session.py index d0c8517..b5ae9dd 100644 --- a/python/restate/sessions/_session.py +++ b/python/restate/sessions/_session.py @@ -28,7 +28,7 @@ import typing import uuid from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, field from datetime import timedelta from typing import Any, Protocol @@ -169,6 +169,8 @@ def __init__(self, host: SqlHost, name: str | None) -> None: self._name: str | None = name self.vm: PyExplicitRecoveryVM | None = host.vm self.database_name: str | None = name + # What keeps the attempt alive while a call of the database runs (an attempt on Restate's), if any. + self.working: Callable[[], AbstractAsyncContextManager[Any]] | None = getattr(host, "database_working", None) def _key(self, key: str) -> str: return key if self._name is None else _beside(self._name, key) @@ -207,6 +209,8 @@ def __init__(self, host: SqlHost, name: str | None) -> None: self._name: str | None = name self.vm: PyExplicitRecoveryVM | None = host.vm self.database_name: str | None = name + # What keeps the attempt alive while a call of the database runs (an attempt on Restate's), if any. + self.working: Callable[[], AbstractAsyncContextManager[Any]] | None = getattr(host, "database_working", None) def get(self, key: str) -> Awaitable[bytes | None]: key = key if self._name is None else _beside(self._name, key) diff --git a/python/restate/sessions/internal.py b/python/restate/sessions/internal.py index 5dc6cbe..fdd4a07 100644 --- a/python/restate/sessions/internal.py +++ b/python/restate/sessions/internal.py @@ -21,7 +21,8 @@ - ``cancelled`` and ``cancel_calls``: explicit cancellation, for handlers declared with ``explicit_cancellation=True``. - ``annotate``: time some work (a model request, say) in the journal, as a - run that Restate's UI shows; nothing waits for it. + run that Restate's UI shows; nothing waits for it, and it keeps nothing + alive (``ctx.working`` does). - ``hide_entries``: metadata that has Restate's UI hide the journal entries of some names (a library's own calls, say), as it hides the database's. - ``start`` and ``attach_invocation``: send, keeping the invocation id, diff --git a/tests/agents_adk.py b/tests/agents_adk.py index a318d56..9b107f7 100644 --- a/tests/agents_adk.py +++ b/tests/agents_adk.py @@ -26,6 +26,7 @@ pytest.importorskip("google.adk") from google.adk.agents import LlmAgent, RunConfig, SequentialAgent # noqa: E402 +from google.adk.agents.run_config import StreamingMode # noqa: E402 # pyright: ignore[reportPrivateImportUsage] from google.adk.models import BaseLlm # noqa: E402 from google.adk.models.llm_request import LlmRequest # noqa: E402 from google.adk.models.llm_response import LlmResponse # noqa: E402 @@ -34,7 +35,7 @@ import restate # noqa: E402 from support import RESTATE_IMAGE, Gate # noqa: E402 # pylint: disable=import-error from restate import HarnessEnvironment, sessions # noqa: E402 -from restate.agents import adk, tool # noqa: E402 +from restate.agents import adk, call, tool # noqa: E402 from restate.agents._api import Messages # noqa: E402 from restate.agents._signals import with_companion # noqa: E402 from restate.agents.internal import CANCELLED # noqa: E402 @@ -1120,3 +1121,249 @@ async def attempt() -> Any: await attempt() # The retry resumed the prompt's invocation, past the message, and answered it. assert await attempt() == "echo: in Celsius" + + +# --- Kept alive ------------------------------------------------------------------------------------ + +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseLlm): + """A model that calls the tool a prompt names (its first word), then answers, noting the scopes it sees. A prompt + that starts with ``stream`` also streams its answer; ``stuck`` waits for ``stuck`` to be set, which never is.""" + + model: str = "probing" + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + mark("model") + prompt = _prompts(list(llm_request.contents))[-1] + answered = any(part.function_response for part in llm_request.contents[-1].parts or []) + word = prompt.split(" ")[0] + if word == "stuck": + await stuck.wait() + if not answered and word in ("plain", "kept", "local", "function", "shipping_status", "wait"): + args = {"key": "k", "input": "o"} if word == "shipping_status" else {} + call = types.FunctionCall(id=f"call-{word}", name=word, args=args) + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(function_call=call)])) + return + if stream: + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(text="do")]), partial=True) + mark("partial") + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(text="done")])) + + +@tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +async def function() -> str: + """A function of the user's own, which ADK runs as its function tool.""" + mark("function") + return "ok" + + +async def hang() -> str: + mark("hang") + await stuck.wait() + return "never" + + +def local() -> Any: + """A built-in local tool of ADK (a shell: its own asks for a confirmation), as a probe.""" + from google.adk.tools.bash_tool import ExecuteBashTool # pylint: disable=import-outside-toplevel + + class Local(ExecuteBashTool): + def __init__(self, name: str = "local") -> None: + super().__init__() + self.name = name + + async def run_async(self, *, args: dict[str, Any], tool_context: Any) -> Any: + mark(self.name) + if self.name == "wait": + await stuck.wait() + return {"result": "ok"} + + return Local + + +shipping = restate.VirtualObject("shipping") + + +@shipping.handler() +async def status(ctx: restate.ObjectContext, order: str) -> str: + return "shipped" + + +def probing(*given: Any, model: Any = None, **options: Any) -> tuple[Callable[..., Awaitable[Any]], Any]: + """Return how to ask an agent with ``given`` tools, in process, and its host (the run's, once it has begun).""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + mark("remote") + return super().generic_call(*args, **kwargs) + + asker = sessions.SessionObject("asker") + llm = model or Probing() + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + if "agent" in options: + return await adk.prompt(ctx, text, **options) + return await adk.prompt(ctx, text, model=llm, tools=[*given], **options) + + backend = Backend() + + async def asked(text: str) -> Any: + scopes.clear() + now[:] = [Watching(backend, service="asker", answers={"publish": REVISION, "status": b'"shipped"'})] + return await invoke(asker, "ask", now[0], request(), text) + + return asked, now + + +async def test_a_model_request_is_kept_alive_and_the_work_around_it_is_not(): + asked, _ = probing() + assert await asked("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + # Streamed: on for the partial replies too, and released with the whole. + mine = LlmAgent(name="mine", model=Probing()) + asked, _ = probing(agent=mine, run_config=RunConfig(streaming_mode=StreamingMode.SSE)) + assert await asked("hi") == "done" + assert scopes == [("model", 1), ("partial", 1)] + assert now[0].open_scopes == 0 + + +async def test_the_summary_of_earlier_turns_is_requested_kept_alive(): + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.agents.adk import _agent # pylint: disable=import-outside-toplevel + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + summarizer = sessions.SessionObject("summarizer") + + @summarizer.handler + async def summarize(ctx: sessions.SessionContext, text: str) -> str: + return await _agent._summarizer(Probing())(ctx, text) # pyright: ignore[reportPrivateUsage] + + scopes.clear() + now[:] = [Host(Backend(), service="summarizer")] + assert await invoke(summarizer, "summarize", now[0], request(), "hi there") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_and_a_users_function_is_not(): + asked, _ = probing(plain, kept, function) + assert await asked("plain x") == "done" + assert scopes == [("model", 1), ("plain", 0), ("model", 1)] + assert await asked("kept x") == "done" + assert scopes == [("model", 1), ("kept", 1), ("model", 1)] + assert await asked("function x") == "done" + assert scopes == [("model", 1), ("function", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_built_in_local_tool_is_kept_alive_while_it_runs_and_a_restate_call_is_not(): + asked, _ = probing(local()("local")) + assert await asked("local x") == "done" + assert scopes == [("model", 1), ("local", 1), ("model", 1)] + assert now[0].open_scopes == 0 + # A call on Restate, which a tool of ours makes: no scope open. + asked, _ = probing(call(status, description="The status of a shipment.")) + assert await asked("shipping_status x") == "done" + assert scopes == [("model", 1), ("remote", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_turn_leaves_no_scope_open_whether_it_was_in_the_model_or_a_built_in_tool(): + for prompt, tools in (("stuck x", []), ("wait x", [local()("wait")])): + stuck.clear() + asked, _ = probing(*tools) + scopes.clear() + task = asyncio.ensure_future(asked(prompt)) + while not scopes or (prompt.startswith("wait") and ("wait", 1) not in scopes): + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + + +async def test_a_failed_model_request_leaves_no_scope_open(): + class Failing(BaseLlm): + model: str = "failing" + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + mark("model") + raise AuthenticationError("invalid x-api-key") + yield # pragma: no cover + + asked, _ = probing(model=Failing()) + with pytest.raises(Exception): + await asked("hi") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_response_a_callback_gives_in_place_of_the_models_leaves_no_scope_open(): + def answering(callback_context: Any, llm_request: Any) -> LlmResponse | None: + # The agent's own callback calls the tool in the model's place, once: no after-model or error callback runs. + if any(part.function_response for content in llm_request.contents for part in content.parts or []): + return None + call_ = types.FunctionCall(id="call-plain", name="plain", args={}) + return LlmResponse(content=types.Content(role="model", parts=[types.Part(function_call=call_)])) + + mine = LlmAgent(name="mine", model=Probing(), tools=adk.tools([plain]), before_model_callback=answering) + asked, _ = probing(agent=mine) + assert await asked("hi") == "done" + # The tool ran after the request the callback answered, with its scope closed; then the model was asked. + assert scopes == [("plain", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +def test_the_environments_tools_and_the_shell_are_built_in_local_tools_and_a_function_is_not(): + # The environment's tools are known by their package: ADK keeps their classes in private modules. + from google.adk.tools.function_tool import FunctionTool # pylint: disable=import-outside-toplevel + from google.adk.tools.bash_tool import ExecuteBashTool # pylint: disable=import-outside-toplevel + from google.adk.tools.environment import EnvironmentToolset # pylint: disable=import-outside-toplevel + + from restate.agents.adk._agent import _local # pylint: disable=import-outside-toplevel + + class Environment: + async def initialize(self) -> None: + pass + + toolset = EnvironmentToolset(environment=typing.cast(Any, Environment())) + tools = asyncio.run(toolset.get_tools()) + assert tools and all(_local(t) for t in tools), tools + assert _local(ExecuteBashTool()) + assert not _local(FunctionTool(lambda: None)) diff --git a/tests/agents_dead_attempt_server.py b/tests/agents_dead_attempt_server.py index 560fb63..1ccb8d1 100644 --- a/tests/agents_dead_attempt_server.py +++ b/tests/agents_dead_attempt_server.py @@ -45,7 +45,7 @@ def anyio_backend(): """How many model requests ran to their end, by the conversation's key.""" SLOW = 4.0 -"""How long a slow model request takes: longer than the inactivity timeout, shorter than a heartbeat (20 s).""" +"""How long a slow model request takes: longer than the inactivity timeout, shorter than a heartbeat's interval (20 s).""" async def model(key: str) -> str: @@ -61,12 +61,14 @@ async def model(key: str) -> str: async def answer(ctx: sessions.SessionContext, text: str) -> str: - """A scripted framework whose model request goes through the turn's stop signal, as the frameworks' do.""" + """A scripted framework whose model request goes through the turn's stop signal, in ``ctx.working``, as the + frameworks' do.""" async def work(conversation: Conversation) -> str: stop = stop_signal_of(ctx) assert stop is not None - await stop.guard(model(ctx.key)) + async with ctx.working(): + await stop.guard(model(ctx.key)) await conversation.turn.append_reply({"role": "assistant", "text": f"echo: {text}"}) return f"echo: {text}" @@ -84,8 +86,9 @@ async def harness(): async def test_a_dead_attempts_model_request_is_cancelled(harness: HarnessEnvironment): - # The attempt heartbeats every 20 s, by default: past the agent's inactivity timeout of a second, Restate - # shuts each slow attempt down in the middle of its model request, and the next attempt asks again. + # The model's request is kept alive, but the heartbeat comes every 20 s, by default: past the agent's inactivity + # timeout of a second, Restate shuts each slow attempt down in the middle of its model request, and the next + # attempt asks again. key = str(uuid.uuid4()) async with httpx2.AsyncClient(timeout=60) as ingress: sent = await ingress.post( diff --git a/tests/agents_langgraph.py b/tests/agents_langgraph.py index 4cf3654..0a00595 100644 --- a/tests/agents_langgraph.py +++ b/tests/agents_langgraph.py @@ -668,3 +668,157 @@ async def test_turns_that_outgrow_the_budget_are_summarized_once_answered(harnes assert len(summaries) == 1 and "User: hi one" in summaries[0] and "hi four" not in summaries[0] latest = summarized_prompts[-1] assert latest[0].startswith("A summary of our conversation so far:") and latest[1:] == ["hi four", "hi five"] + + +# --- Kept alive ------------------------------------------------------------------------------------- + +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseChatModel): + """A model that calls the tool a prompt names (its first word), then answers, noting the scopes it sees. A prompt + that starts with ``stuck`` waits for ``stuck`` to be set, which never is.""" + + @property + def _llm_type(self) -> str: + return "probing" + + def bind_tools(self, tools: Any, **kwargs: Any) -> Any: # pyright: ignore[reportIncompatibleMethodOverride] + return self + + def _generate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + raise AssertionError("the requests are async") + + async def _agenerate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + mark("model") + prompt = next(m for m in reversed(messages) if isinstance(m, HumanMessage)) + word = str(prompt.content).split(" ")[0] + if word == "stuck": + await stuck.wait() + if word == "fail": + raise AuthenticationError("invalid x-api-key") + if word in ("plain", "kept", "chain", "lg_shipping_status") and not isinstance(messages[-1], ToolMessage): + args_ = {"key": "k", "input": "o"} if word == "lg_shipping_status" else {} + reply = AIMessage("", tool_calls=[{"id": f"call-{word}", "name": word, "args": args_}]) + else: + reply = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=reply)]) + + +@tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +@langchain_tool +async def chain() -> str: + """A LangChain tool of the user's own.""" + mark("chain") + return "ok" + + +shipping = restate.VirtualObject("lg_shipping") + + +@shipping.handler() +async def status(ctx: restate.ObjectContext, order: str) -> str: + return "shipped" + + +def probing(*given: Any, **options: Any) -> Callable[..., Awaitable[Any]]: + """Return how to ask an agent with ``given`` tools, in process, which notes the scopes in ``scopes``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + mark("remote") + return super().generic_call(*args, **kwargs) + + asker = sessions.SessionObject("asker") + model = Probing() + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await langgraph.prompt(ctx, text, model=model, tools=[*given], **options) + + @asker.handler + async def summarize(ctx: sessions.SessionContext, text: str) -> str: + from restate.agents.langgraph import _agent # pylint: disable=import-outside-toplevel + + return await _agent._summarizer(model)(ctx, text) # pyright: ignore[reportPrivateUsage] + + backend = Backend() + + async def asked(text: str, handler: str = "ask") -> Any: + scopes.clear() + now[:] = [Watching(backend, service="asker", answers={"publish": REVISION, "status": b'"shipped"'})] + return await invoke(asker, handler, now[0], request(), text) + + return asked + + +async def test_the_built_in_agents_model_request_is_kept_alive_and_the_work_around_it_is_not(): + assert await probing()("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_the_summary_of_earlier_turns_is_requested_kept_alive(): + assert await probing()("hi there", "summarize") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_and_a_langchain_tool_is_not(): + asked = probing(plain, kept, chain) + for word, seen in (("plain", 0), ("kept", 1), ("chain", 0)): + assert await asked(f"{word} x") == "done" + assert scopes == [("model", 1), (word, seen), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_call_on_restate_that_a_tool_makes_is_outside_any_scope(): + from restate.agents import call # pylint: disable=import-outside-toplevel + + asked = probing(call(status, description="The status of a shipment.")) + assert await asked("lg_shipping_status x") == "done" + assert scopes == [("model", 1), ("remote", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_or_failed_request_leaves_no_scope_open(): + stuck.clear() + asked = probing() + scopes.clear() + task = asyncio.ensure_future(asked("stuck x")) + while not scopes: + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + with pytest.raises(Exception): + await asked("fail x") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 diff --git a/tests/agents_langgraph_graph.py b/tests/agents_langgraph_graph.py index 9ac2d9a..9c5f736 100644 --- a/tests/agents_langgraph_graph.py +++ b/tests/agents_langgraph_graph.py @@ -16,10 +16,12 @@ # pylint: disable=missing-function-docstring,missing-class-docstring import asyncio +import contextlib import collections import threading import typing import uuid +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass from datetime import timedelta from typing import Any, NotRequired, TypedDict @@ -45,6 +47,7 @@ from support import RESTATE_IMAGE # noqa: E402 # pylint: disable=import-error from restate import HarnessEnvironment, sessions # noqa: E402 from restate.agents import langgraph # noqa: E402 +from restate.agents import tool as restate_tool # noqa: E402 from restate.agents.internal import CANCELLED, ENDED # noqa: E402 from restate.agents.langgraph import nodes # noqa: E402 from restate.agents.langgraph._errors import terminal_of # noqa: E402 @@ -973,3 +976,200 @@ def test_context_is_for_a_graph_of_your_own(): make: Any = langgraph.agent with pytest.raises(TypeError, match="context is an option of your own graph: pass it with graph="): make("plain", context=lambda ctx: Caller(ctx.key)) + + +# --- Kept alive ------------------------------------------------------------------------------------- + +off_loop: list[bool] = [] +"""Each ``ctx.working`` scope entered off the attempt's event loop.""" +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseChatModel): + """A model that notes the scopes it sees, and waits on ``stuck`` (never set) for a message that says ``stuck``.""" + + @property + def _llm_type(self) -> str: + return "probing" + + def _generate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + # A sync node's request (`invoke`), in a thread of the graph's. + mark("sync model") + return ChatResult(generations=[ChatGeneration(message=AIMessage("done"))]) + + async def _agenerate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + mark("model") + if "stuck" in str(messages[-1].content): + await stuck.wait() + if "fail" in str(messages[-1].content): + raise RuntimeError("the model API is down") + return ChatResult(generations=[ChatGeneration(message=AIMessage("done"))]) + + +@restate_tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@restate_tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +def _kept_graph() -> Any: + """A graph of the user's own: a node that asks a model (without passing it the run's config, as nodes do), one + that interrupts, one that asks a model in a subgraph, and one that calls tools.""" + model = Probing() + + async def llm(state: MessagesState) -> dict[str, Any]: + return {"messages": [await model.ainvoke(state["messages"])]} + + def sync_llm(state: MessagesState) -> dict[str, Any]: + # The usual shape of LangGraph's examples: LangChain runs the callbacks of a sync request on a loop of its own. + return {"messages": [model.invoke(state["messages"])]} + + async def ask(state: MessagesState) -> dict[str, Any]: + mark("before the interrupt") + return {"messages": [AIMessage(f"approved: {interrupt('approve?')}")]} + + def calls(state: MessagesState) -> dict[str, Any]: + word = str(state["messages"][-1].content).split(" ")[0] + return {"messages": [AIMessage("", tool_calls=[{"id": f"call-{word}", "name": word, "args": {}}])]} + + inner = StateGraph(MessagesState) + inner.add_node("llm", llm) + inner.add_edge(START, "llm") + + def route(state: MessagesState) -> str: + return {"ask": "ask", "plain": "calls", "kept": "calls", "nested": "nested", "sync": "sync_llm"}.get( + str(state["messages"][-1].content).split(" ")[0], "llm" + ) + + outer = StateGraph(MessagesState) + outer.add_node("llm", llm) + outer.add_node("sync_llm", sync_llm) + outer.add_node("ask", ask) + outer.add_node("calls", calls) + outer.add_node("tools", ToolNode(langgraph.tools([plain, kept]))) + outer.add_node("nested", inner.compile()) + outer.add_conditional_edges(START, route, ["llm", "sync_llm", "ask", "calls", "nested"]) + outer.add_edge("calls", "tools") + for end in ("llm", "sync_llm", "ask", "tools", "nested"): + outer.add_edge(end, END) + return outer + + +def keeping() -> Callable[..., Awaitable[Any]]: + """Return how to ask, in process, an agent of the user's own graph, which notes its scopes in ``scopes``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.agents import AgentObject # pylint: disable=import-outside-toplevel + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + if args[1] == "wait": + mark("wait") + return super().generic_call(*args, **kwargs) + + @contextlib.asynccontextmanager + async def working(self) -> AsyncIterator[None]: + # As the driver's: a task that beats while the scope is open, on the loop the scope is entered on. A + # scope entered on another loop than the attempt's would hold that loop's caller until it closes. + if asyncio.get_running_loop() is not loop: + # LangChain swallows a handler's error: noted here instead, for the test to see. + off_loop.append(True) + async with super().working(): + beat = asyncio.ensure_future(asyncio.sleep(3600)) + try: + yield + finally: + beat.cancel() + + loop = asyncio.get_running_loop() + + graph = _kept_graph() + asker = AgentObject("kept_asker") + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await langgraph.prompt(ctx, text, graph=graph) + + backend = Backend() + + async def asked(text: str) -> Any: + scopes.clear() + off_loop.clear() + answers = {"publish": b'{"revision": 1}', "wait": b'"yes"'} + now[:] = [Watching(backend, service="kept_asker", answers=answers)] + return await invoke(asker, "ask", now[0], request(), text) + + return asked + + +async def test_a_graphs_model_request_is_kept_alive_wherever_a_node_makes_it(): + asked = keeping() + assert await asked("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + # In a subgraph the graph's nodes run: the handler is inherited. + assert await asked("nested x") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_sync_nodes_model_request_does_not_hang_and_leaves_no_scope_open(): + # LangChain runs the callbacks of a sync request (`invoke`) on a loop of its own, in a thread, and waits there for + # every task they started: a scope entered there, with its heartbeat's task, would never let the request go out. + asked = keeping() + assert await asyncio.wait_for(asked("sync now"), timeout=10) == "done" + assert [where for where, _ in scopes] == ["sync model"] + assert not off_loop, "a scope was entered off the attempt's loop" + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_in_a_graph_too(): + asked = keeping() + await asked("plain x") + assert scopes == [("plain", 0)] + await asked("kept x") + assert scopes == [("kept", 1)] + assert now[0].open_scopes == 0 + + +async def test_an_interrupts_wait_for_its_signal_is_outside_any_scope(): + asked = keeping() + assert await asked("ask x") == "approved: yes" + assert scopes == [("before the interrupt", 0), ("wait", 0), ("before the interrupt", 0)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_or_failed_request_of_a_graph_leaves_no_scope_open(): + stuck.clear() + asked = keeping() + scopes.clear() + task = asyncio.ensure_future(asked("stuck x")) + while not scopes: + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + with pytest.raises(Exception): + await asked("fail x") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 diff --git a/tests/agents_long_work_server.py b/tests/agents_long_work_server.py index b3cae99..273311e 100644 --- a/tests/agents_long_work_server.py +++ b/tests/agents_long_work_server.py @@ -43,13 +43,14 @@ def anyio_backend(): async def answer(ctx: sessions.SessionContext, text: str) -> str: """A scripted framework whose model thinks for longer than the inactivity timeout, and says nothing meanwhile. - It is what a turn of any framework is while its model request runs: local work, with nothing of Restate's - outstanding. + It is what a turn of any framework is while its model request runs: local work in ``ctx.working``, as the + frameworks' requests run. """ async def work(conversation: Conversation) -> str: requests[ctx.key] = requests.get(ctx.key, 0) + 1 - await asyncio.sleep(3) + async with ctx.working(): + await asyncio.sleep(3) await conversation.turn.append_reply({"role": "assistant", "text": f"echo: {text}"}) return f"echo: {text}" diff --git a/tests/agents_openai.py b/tests/agents_openai.py index 837377d..a3eda11 100644 --- a/tests/agents_openai.py +++ b/tests/agents_openai.py @@ -28,6 +28,7 @@ pytest.importorskip("agents") from agents import ( # noqa: E402 + LocalShellTool, Agent, GuardrailFunctionOutput, Handoff, @@ -1620,3 +1621,138 @@ async def outputs(ctx: sessions.SessionContext, _: Any = None) -> list[str]: "found two", "found three", ] + + +# --- In process: the attempt is kept alive for the model's requests and a local tool's runs only ---- + + +class Seen: + """How many ``ctx.working`` scopes were open at each moment a test's own code ran.""" + + def __init__(self) -> None: + self.host: Any = None + self.at: dict[str, int] = {} + + def mark(self, what: str) -> None: + self.at[what] = self.host.open_scopes + + +def watching_invoker(obj: sessions.SessionObject, seen: Seen) -> Callable[..., Awaitable[Any]]: + """Invoke ``obj``'s ``ask``, with the host kept in ``seen``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + backend = Backend() + + async def ask(text: str) -> Any: + seen.host = Host(backend, service="asker", answers=PUBLISHED) + try: + return await invoke(obj, "ask", seen.host, request(), text) + finally: + seen.at["after"] = seen.host.open_scopes + + return ask + + +async def test_a_model_request_is_kept_alive_and_a_tool_of_your_own_is_not(): + seen = Seen() + + @tool + async def plain(city: str) -> str: + """Plain.""" + seen.mark("plain") + return "plain done" + + @tool(keepalive=True) + async def long(city: str) -> str: + """Long.""" + seen.mark("long") + return "long done" + + @function_tool(name_override="mine") + def mine(city: str) -> str: + """The user's own function tool.""" + seen.mark("mine") + return "mine done" + + def script(system: str | None, items: list[Item]) -> list[Output]: + seen.mark("model") + word = _prompts(items)[-1].split(" ")[0] + if items[-1].get("type") == "function_call_output": + return [said("done")] + return [calls(f"call-{uuid.uuid4().hex}", word, '{"city": "Rome"}')] + + ask = watching_invoker(asker(model=Scripted(script), tools=[plain, long, mine]), seen) + for word in ("plain", "long", "mine"): + assert await ask(f"{word} now") == "done" + assert seen.at["model"] > 0 + assert seen.at["plain"] == 0 and seen.at["mine"] == 0 + assert seen.at["long"] == 1 + assert seen.at["after"] == 0 + + +async def test_the_scope_of_a_request_ends_with_the_request_not_the_steering_and_stores_around_it(): + seen = Seen() + during: list[int] = [] + + async def count_stores(data: CallModelData[Any]) -> ModelInputData: + # The input filter runs before the request: no scope is open yet. + during.append(seen.host.open_scopes) + return data.model_data + + def script(system: str | None, items: list[Item]) -> list[Output]: + seen.mark("model") + return [said("ok")] + + mine = Agent(name="mine", model=Scripted(script)) + ask = watching_invoker(asker(agent=mine, run_config=RunConfig(call_model_input_filter=count_stores)), seen) + assert await ask("hi there") == "ok" + assert during == [0] and seen.at["model"] == 1 and seen.at["after"] == 0 + + +async def test_a_failed_request_and_a_failed_tool_release_their_scopes(): + seen = Seen() + ask = watching_invoker(asker(model=Scripted(reply)), seen) + with pytest.raises(Exception): + await ask("refuse now") + assert seen.at["after"] == 0 + + @tool(keepalive=True) + async def broken(city: str) -> str: + """Fails.""" + raise restate.TerminalError("no") + + def call_it(system: str | None, items: list[Item]) -> list[Output]: + return [calls("c1", "broken", '{"city": "Rome"}')] + + ask = watching_invoker(asker(model=Scripted(call_it), tools=[broken]), seen) + with pytest.raises(restate.TerminalError): + await ask("go") + assert seen.at["after"] == 0 + + +async def test_a_built_in_local_tool_is_kept_alive_while_it_runs(): + from openai.types.responses.response_output_item import LocalShellCall # pylint: disable=import-outside-toplevel + from openai.types.responses.response_output_item import ( # pylint: disable=import-outside-toplevel + LocalShellCallAction, + ) + + seen = Seen() + + def execute(request: Any) -> str: + seen.mark("shell") + return "ran" + + def script(system: str | None, items: list[Item]) -> list[Output]: + if items[-1].get("type") in ("function_call_output", "local_shell_call_output"): + return [said("done")] + action = LocalShellCallAction(type="exec", command=["ls"], env={}) + call = LocalShellCall(id="ls1", action=action, call_id="c1", status="completed", type="local_shell_call") + return [typing.cast(Any, call)] + + mine = Agent(name="shell", model=Scripted(script), tools=[LocalShellTool(executor=execute)]) + ask = watching_invoker(asker(agent=mine), seen) + assert await ask("run it") == "done" + assert seen.at["shell"] == 1 and seen.at["after"] == 0 diff --git a/tests/agents_pydantic_ai.py b/tests/agents_pydantic_ai.py index 45f8d0b..4cf7b76 100644 --- a/tests/agents_pydantic_ai.py +++ b/tests/agents_pydantic_ai.py @@ -27,6 +27,7 @@ pytest.importorskip("pydantic_ai") from pydantic_ai import Agent, ModelRetry, RunContext # noqa: E402 +from pydantic_ai import Tool as PydanticTool # noqa: E402 from pydantic_ai.capabilities import AbstractCapability # noqa: E402 from pydantic_ai.exceptions import UnexpectedModelBehavior # noqa: E402 from pydantic_ai.messages import ( # noqa: E402 @@ -1341,3 +1342,146 @@ def lookup(about: str) -> str: if isinstance(part, ToolReturnPart) ] assert returned == ["found one", "found two", "found three"] + + +# --- In process: the attempt is kept alive for the model's requests and a local tool's runs only ---- + + +class Seen: + def __init__(self) -> None: + self.host: Any = None + self.at: dict[str, int] = {} + + def mark(self, what: str) -> None: + self.at[what] = self.host.open_scopes + + +async def run_watched(seen: Seen, ask: sessions.SessionObject, text: str, **kwargs: Any) -> Any: + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + seen.host = Host(Backend(), service="asker", **kwargs) + try: + return await invoke(ask, "ask", seen.host, request(), text) + finally: + seen.at["after"] = seen.host.open_scopes + + +async def test_a_model_request_is_kept_alive_a_tool_of_your_own_is_not_and_a_local_tool_is(): + from pydantic_ai_harness.filesystem import READ_FILE_KIND # pylint: disable=import-outside-toplevel + + seen = Seen() + + @tool + async def plain(city: str) -> str: + """Plain.""" + seen.mark("plain") + return "plain" + + @tool(keepalive=True) + async def long(city: str) -> str: + """Long.""" + seen.mark("long") + return "long" + + def local(path: str) -> str: + """Stands for a built-in local tool: its kind says so.""" + seen.mark("local") + return "read" + + def own(path: str) -> str: + """A plain function tool of the user's.""" + seen.mark("own") + return "own" + + def shell(path: str) -> str: + """Stands for the harness's shell: its name says so.""" + seen.mark("run_command") + return "ran" + + def script(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + seen.mark("model") + if isinstance(messages[-1], ModelRequest) and any(isinstance(p, ToolReturnPart) for p in messages[-1].parts): + return ModelResponse(parts=[TextPart("done")]) + word = next(str(p.content).split(" ")[0] for p in messages[-1].parts if isinstance(p, UserPromptPart)) + return ModelResponse( + parts=[ToolCallPart(word, {"city": "Rome"} if word in ("plain", "long") else {"path": "x"})] + ) + + mine = Agent( + FunctionModel(script), + tools=[ + PydanticTool(local, tool_kind=READ_FILE_KIND, name="local"), + PydanticTool(own, name="own"), + PydanticTool(shell, name="run_command"), + ], + toolsets=[pydantic_ai.toolset([plain, long])], + ) + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, agent=mine) + + for word in ("plain", "long", "local", "own", "run_command"): + assert await run_watched(seen, obj, f"{word} now") == "done" + assert seen.at["after"] == 0 + assert seen.at["model"] == 1 + assert (seen.at["plain"], seen.at["long"], seen.at["local"], seen.at["own"], seen.at["run_command"]) == ( + 0, + 1, + 1, + 0, + 1, + ) + + +async def test_a_failed_request_releases_its_scope(): + seen = Seen() + + def refuse(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + seen.mark("model") + raise AuthenticationError("invalid x-api-key") + + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, model=FunctionModel(refuse)) + + with pytest.raises(Exception): + await run_watched(seen, obj, "hi") + assert seen.at["model"] == 1 and seen.at["after"] == 0 + + +async def test_an_approval_is_waited_for_outside_any_scope(monkeypatch: pytest.MonkeyPatch): + from restate.agents.pydantic_ai import _agent # pylint: disable=import-outside-toplevel + + seen = Seen() + + async def approved(ctx: Any, call_id: str, signal: str, input: Any = None, *, approval: bool = False) -> Any: + seen.mark("approval") + return {"approved": True} + + monkeypatch.setattr(_agent, "wait_for", approved) + + def needs(city: str) -> str: + """Needs an approval.""" + return "ran" + + def script(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + if any(isinstance(p, ToolReturnPart) for m in messages if isinstance(m, ModelRequest) for p in m.parts): + return ModelResponse(parts=[TextPart("done")]) + return ModelResponse(parts=[ToolCallPart("needs", {"city": "Rome"})]) + + mine = Agent(FunctionModel(script), tools=[PydanticTool(needs, requires_approval=True)]) + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, agent=mine) + + assert await run_watched(seen, obj, "go") == "done" + assert seen.at["approval"] == 0 and seen.at["after"] == 0 diff --git a/tests/agents_pydantic_ai_snapshots.py b/tests/agents_pydantic_ai_snapshots.py index 0f84e23..87a56a7 100644 --- a/tests/agents_pydantic_ai_snapshots.py +++ b/tests/agents_pydantic_ai_snapshots.py @@ -647,3 +647,29 @@ def planner(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: stored = [p for m in steps if isinstance(m, ModelRequest) for p in m.parts] assert [type(p).__name__ for p in stored] == ["RetryFeedbackPart", "UserPromptPart", "RetryFeedbackPart"] assert [text for role, text in await chat.messages() if role == "user"] == ["plan one", "use Celsius", "plan two"] + + +async def test_the_summarizers_request_and_the_models_are_kept_alive(): + hosts: list[Host] = [] + scopes: dict[str, list[int]] = {"summary": [], "model": []} + + class Watched(Script): + def __call__(self, messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + before = self.summaries + response = super().__call__(messages, info) + scopes["summary" if self.summaries > before else "model"].append(hosts[-1].open_scopes) + return response + + script = Watched() + chat = agent(script, "pai_alive") + make = chat.host + + def watched(view: bool = False) -> Host: + hosts.append(make(view)) + return hosts[-1] + + chat.host = watched # type: ignore[method-assign] + await until_snapshot(chat, compact=False) + assert script.summaries == 1 + assert scopes["summary"] == [1] and scopes["model"] and all(n == 1 for n in scopes["model"]) + assert all(host.open_scopes == 0 for host in hosts) diff --git a/tests/agents_tools.py b/tests/agents_tools.py index 1378fba..41be1ec 100644 --- a/tests/agents_tools.py +++ b/tests/agents_tools.py @@ -351,3 +351,26 @@ async def test_a_langgraph_wait_from_a_session_object_that_is_no_agent_fails_the await internal.invoke(desk, "ask", host, request(), "refund o1") assert raised.value.message.startswith(f"{what} {NEEDS_AGENT}"), raised.value.message assert host.calls == [] + + +async def test_a_tool_runs_kept_alive_only_with_keepalive(): + host = Host(Backend(), service="runner", key="alice") + seen: dict[str, int] = {} + + @tool + async def quick() -> str: + """Quick.""" + seen["quick"] = host.open_scopes + return "done" + + @tool(keepalive=True) + async def long() -> str: + """Long.""" + seen["long"] = host.open_scopes + return "done" + + tools["quick"], tools["long"] = quick, long + await run_tool(host, "quick", {}, "c1") + await run_tool(host, "long", {}, "c2") + assert seen == {"quick": 0, "long": 1} + assert host.open_scopes == 0 diff --git a/tests/sessions_driver.py b/tests/sessions_driver.py index 3b6c175..b8b2736 100644 --- a/tests/sessions_driver.py +++ b/tests/sessions_driver.py @@ -148,18 +148,28 @@ async def commits(ctx: sessions.SessionContext) -> str: @notes.handler() -async def works_alone(ctx: sessions.SessionContext) -> str: - """Works outside Restate, with nothing of Restate's outstanding, until the test lets it go on.""" - await gate_of().wait() +async def works_alone(ctx: sessions.SessionContext, kept: bool) -> str: + """Works outside Restate, with nothing of Restate's outstanding, in ``ctx.working`` or not, until the test lets it go + on.""" + if kept: + async with ctx.working(): + await gate_of().wait() + else: + await gate_of().wait() return "done" @notes.handler() -async def works_beside_a_call(ctx: sessions.SessionContext, annotated: bool) -> str: - """Waits for a call that is never answered, working meanwhile, inside an annotation or not.""" +async def works_beside_a_call(ctx: sessions.SessionContext, how: str) -> str: + """Waits for a call that is never answered, working meanwhile: in ``ctx.working`` (``kept``), in an annotation + (``annotated``), or neither.""" pending = asyncio.ensure_future(ctx.generic_call("Callee", "echo", "A", key="k")) - annotation = internal.annotate(ctx, "tool") if annotated else None - await gate_of().wait() + annotation = internal.annotate(ctx, "tool") if how == "annotated" else None + if how == "kept": + async with ctx.working(): + await gate_of().wait() + else: + await gate_of().wait() if annotation is not None: annotation.end() pending.cancel() @@ -611,32 +621,43 @@ def quick_heartbeats(monkeypatch) -> None: monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.02)) -async def test_an_attempt_heartbeats_while_its_handler_works_with_nothing_of_restates_outstanding( - gate: Gate, quick_heartbeats: None -): - # A model request, say: Restate hears nothing from the attempt meanwhile, and shuts down one it heard nothing - # from for its inactivity timeout. The next attempt would make the request again, and never get past it. +async def test_an_attempt_does_not_heartbeat_outside_working(gate: Gate, quick_heartbeats: None): + # Suspension is the default: Restate shuts down an attempt it heard nothing from for its inactivity timeout, and + # starts the next when what it waits for answers (or at once, if nothing). + side = RestateSide() + side.start("kh1", False) + assert await beats_while_at(gate, "works_alone", side) == 0, describe(side) + assert output_of(side.sent()) == "done", describe(side) + + +async def test_an_attempt_heartbeats_in_working(gate: Gate, quick_heartbeats: None): + # A model request, say: the next attempt would make it again, and never get past it. side = RestateSide() - side.start("kh1", None) + side.start("kh2", True) assert await beats_while_at(gate, "works_alone", side) >= 3, describe(side) assert output_of(side.sent()) == "done", describe(side) -async def test_an_attempt_does_not_heartbeat_while_an_answer_of_restates_is_outstanding( +async def test_an_attempt_heartbeats_in_working_beside_an_outstanding_answer(gate: Gate, quick_heartbeats: None): + # Work beside a call it waits for: a model's request beside a tool's call, say. + side = RestateSide() + side.start("kh3", "kept") + assert await beats_while_at(gate, "works_beside_a_call", side) >= 3, describe(side) + + +async def test_an_attempt_does_not_heartbeat_while_it_waits_on_restate_outside_working( gate: Gate, quick_heartbeats: None ): - # A wait of days for an approval holds no process: Restate shuts the attempt down, and starts the next one when - # the answer comes. + # A wait of days for an approval holds no process. side = RestateSide() - side.start("kh2", False) + side.start("kh4", "neither") assert await beats_while_at(gate, "works_beside_a_call", side) == 0, describe(side) -async def test_an_open_annotation_heartbeats_beside_an_outstanding_answer(gate: Gate, quick_heartbeats: None): - # The annotation says the handler is working, on a tool beside the call it waits for. +async def test_an_open_annotation_is_a_trace_and_keeps_nothing_alive(gate: Gate, quick_heartbeats: None): side = RestateSide() - side.start("kh3", True) - assert await beats_while_at(gate, "works_beside_a_call", side) >= 3, describe(side) + side.start("kh5", "annotated") + assert await beats_while_at(gate, "works_beside_a_call", side) == 0, describe(side) async def test_a_read_that_asks_restate_nothing_sees_a_carried_cancellation_too_once(gate: Gate): diff --git a/tests/sessions_harness.py b/tests/sessions_harness.py index d525709..24900a6 100644 --- a/tests/sessions_harness.py +++ b/tests/sessions_harness.py @@ -260,17 +260,18 @@ async def waits_for_a_crawl(ctx: sessions.SessionContext) -> list[Any]: @slowpoke.handler() async def thinks(ctx: sessions.SessionContext) -> int: - """Works locally longer than the inactivity timeout, inside an annotation.""" + """Works locally longer than the inactivity timeout, in ``ctx.working`` (and an annotation, which only times it).""" n = attempt(ctx) timed = internal.annotate(ctx, "model: thinking") - await asyncio.sleep(2.5) + async with ctx.working(): + await asyncio.sleep(2.5) timed.end() return n @slowpoke.handler() async def daydreams(ctx: sessions.SessionContext) -> int: - """Works locally longer than the inactivity timeout, telling Restate nothing.""" + """Works locally longer than the inactivity timeout, outside ``ctx.working``: telling Restate nothing.""" n = attempt(ctx) await asyncio.sleep(2.5) daydreamt[ctx.key] = daydreamt.get(ctx.key, 0) + 1 @@ -449,30 +450,20 @@ async def test_a_shut_down_attempt_resumes_when_its_call_completes(harness: Harn assert n >= 2 -async def test_an_open_annotation_keeps_the_attempt_alive(harness: HarnessEnvironment, monkeypatch): +async def test_work_in_working_keeps_the_attempt_alive(harness: HarnessEnvironment, monkeypatch): + # The attempt heartbeats in `ctx.working`: here faster than the object's inactivity timeout of a second, so the + # work is done in the first attempt. monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.3)) key = str(uuid.uuid4()) assert await harness.client.object_call(thinks, key=key, arg=None) == 1 -async def test_local_work_keeps_its_attempt_alive_with_nothing_of_restates_outstanding( - harness: HarnessEnvironment, monkeypatch -): - # The attempt heartbeats while its handler works on its own, annotation or not: here faster than the object's - # inactivity timeout of a second, so the work is done in the first attempt. +async def test_local_work_outside_working_is_shut_down_and_restarted(harness: HarnessEnvironment, monkeypatch): + # However quick the heartbeat, none is sent outside `ctx.working`: each attempt is shut down after a second with + # nothing heard from it, and restarted right away. monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.3)) - key = str(uuid.uuid4()) - assert await harness.client.object_call(daydreams, key=key, arg=None) == 1 - assert daydreamt.get(key, 0) == 1 - - -async def test_local_work_is_shut_down_and_restarted_when_the_heartbeat_is_slower_than_the_inactivity_timeout( - harness: HarnessEnvironment, -): key = str(uuid.uuid4()) sent = await harness.client.object_send(daydreams, key=key, arg=None) - # The heartbeat comes every 20 seconds, and the object's inactivity timeout is a second: each attempt is shut - # down after a second with nothing heard from it, and restarted right away. for _ in range(300): if attempts.get(key, 0) >= 4: break diff --git a/tests/sessions_object.py b/tests/sessions_object.py index e409993..a848dfb 100644 --- a/tests/sessions_object.py +++ b/tests/sessions_object.py @@ -13,6 +13,7 @@ # pylint: disable=missing-function-docstring,missing-class-docstring import asyncio +import contextlib import itertools import json import typing @@ -114,6 +115,17 @@ class Host: cancelled: list[str] = field(default_factory=list) # How many more syncs go through before the process crashes: every sync after fails, losing its writes. _syncs_left: int | None = None + # The handler's ``ctx.working`` scopes open now: an attempt on Restate heartbeats while there is one. + open_scopes: int = 0 + + @contextlib.asynccontextmanager + async def working(self): + """The handler's ``ctx.working``, counted: a test reads ``open_scopes`` while some work of its own runs.""" + self.open_scopes += 1 + try: + yield + finally: + self.open_scopes -= 1 def crash_after(self, syncs: int) -> None: """Let ``syncs`` more syncs through, then crash: every later sync fails.""" @@ -648,6 +660,9 @@ def generic_send(self, service: str, handler: str, arg: Any = None, **options: A def cancel(self, invocation_id: str) -> None: pass + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + return contextlib.nullcontext() + async def test_a_handler_runs_over_a_fake_context_of_a_test(): fake = FakeContext(db=FakeDatabase())