Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
20 changes: 11 additions & 9 deletions docs/agents/turns.md
Original file line number Diff line number Diff line change
Expand Up @@ -457,15 +457,17 @@ sequenceDiagram
message starts afresh: the cancelled prompt is over.
- The cancellation reaches the turn's work at once. Restate shuts down an
attempt it has heard nothing from for the inactivity timeout (a minute by
default), and the next attempt runs the turn from its start. So while the
turn works on its own (a model request, a tool), its attempt tells Restate
every 20 seconds that it makes progress, and Restate keeps it however long
the step takes. While the turn waits for an answer of Restate's instead (a
tool waiting for an approval, a call), it tells Restate nothing: Restate
shuts the attempt down after the timeout, so that a wait of days holds no
process, and the next attempt gets the cancellation as it starts. A
deployment that sets an inactivity timeout under 20 seconds is shut down
between two of those messages.
default), and the next attempt runs the turn from its start. So while a
model request runs, or a framework's built-in local tool (a shell, file
edits), the attempt tells Restate every 20 seconds that it makes progress
(`ctx.working`), and Restate keeps it however long the step takes. Nothing
else of a turn does: a tool of your own only with `tool(keepalive=True)`.
While the turn waits for an answer of Restate's (a tool waiting for an
approval, a call), it tells Restate nothing: Restate shuts the attempt down
after the timeout, so that a wait of days holds no process, and the next
attempt gets the cancellation as it starts. A deployment that sets an
inactivity timeout under 20 seconds is shut down between two of those
messages.
- A cancellation that comes once the turn answered changes nothing: the
answer stands.
- A tool cut short by the stop raises the stop's reason, so the model never
Expand Down
29 changes: 25 additions & 4 deletions python/restate/agents/_tools.py
Original file line number Diff line number Diff line change
Expand Up @@ -126,13 +126,15 @@ class Tool:
description: What the tool does, for the model.
input_schema: The JSON schema of the tool's input, which the
framework passes to the model.
keepalive: Whether the tool runs in ``ctx.working``: see ``tool``.
"""

name: str
description: str
input_schema: dict[str, Any]
_run: Callable[[dict[str, Any], ToolCall], Awaitable[Any]]
_validate: Callable[[dict[str, Any]], Any]
keepalive: bool = False

async def run(self, args: object, call: ToolCall) -> str:
"""Run the tool with the model's raw arguments, and return its result as text for the model.
Expand All @@ -157,6 +159,9 @@ async def run(self, args: object, call: ToolCall) -> str:
except Exception as error: # pydantic's ValidationError, for the model to fix
return f"Error: the arguments do not fit the tool's input: {error}"
try:
if self.keepalive:
async with call.ctx.working():
return _text(await self._run(arguments, call))
return _text(await self._run(arguments, call))
except BaseException as error:
# The turn stopped (its invocation was cancelled, or its attempt
Expand Down Expand Up @@ -219,15 +224,23 @@ def _text(value: object) -> str:


@typing.overload
def tool(fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None) -> Tool: ...
def tool(
fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None, keepalive: bool = False
) -> Tool: ...


@typing.overload
def tool(*, name: str | None = None, description: str | None = None) -> Callable[[Callable[..., Any]], Tool]: ...
def tool(
*, name: str | None = None, description: str | None = None, keepalive: bool = False
) -> Callable[[Callable[..., Any]], Tool]: ...


def tool(
fn: Callable[..., Any] | None = None, *, name: str | None = None, description: str | None = None
fn: Callable[..., Any] | None = None,
*,
name: str | None = None,
description: str | None = None,
keepalive: bool = False,
) -> Tool | Callable[[Callable[..., Any]], Tool]:
"""Create a tool from a function, sync or async.

Expand All @@ -251,12 +264,19 @@ async def get_weather(city: str) -> str:
result, and can act on it. Any other exception fails the attempt, and
Restate retries it.

A tool does not keep its attempt alive: one that works for longer than
Restate's inactivity timeout (a minute, by default) is cut off, as by a
crash. One that must work that long sets ``keepalive``, and runs in
``ctx.working``; it keeps long waits on Restate (a call, a signal) out of
it, or holds its process for as long as they take.

Args:
fn: The function. Omit it to use ``tool(name=..., description=...)``
as a decorator.
name: The tool's name. Defaults to the function's name.
description: What the tool does, for the model. Defaults to the
function's docstring.
keepalive: Keep the attempt alive while the tool runs.

Returns:
The tool, or a decorator that creates it.
Expand All @@ -267,7 +287,7 @@ async def get_weather(city: str) -> str:
if fn is None:

def decorate(fn: Callable[..., Any]) -> Tool:
return tool(fn, name=name, description=description)
return tool(fn, name=name, description=description, keepalive=keepalive)

return decorate
pydantic = _pydantic()
Expand Down Expand Up @@ -295,6 +315,7 @@ def validate(args: dict[str, Any]) -> dict[str, Any]:
input_schema=typing.cast(dict[str, Any], model.model_json_schema()),
_run=run,
_validate=validate,
keepalive=keepalive,
)


Expand Down
120 changes: 114 additions & 6 deletions python/restate/agents/adk/_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -47,6 +47,10 @@
with ``include_contents='none'`` starts each request at the latest message
of the user: the requests after the one that takes a steered message start
at it, without the prompt and the tools' results before it.
- The attempt is kept alive (``ctx.working``) while a model request runs (not the steering and commits around it)
and while one of ADK's built-in local tools does (the shell and the environment's tools: ``_LOCAL_TOOLS``),
not while one of yours does: a ``FunctionTool``, or a ``tool`` of ours unless it has ``keepalive``. A code
executor of an agent (``code_executor``) is not covered.
- A run of more model calls than its ``RunConfig``'s ``max_llm_calls`` (500 by
default) fails its prompt for good: a retry would carry on with a fresh count.
- An agent or ``App`` of your own runs as you configured it, with the
Expand Down Expand Up @@ -75,6 +79,7 @@
import time
import typing
from collections.abc import Awaitable, Callable, Sequence
from contextlib import AsyncExitStack
from typing import Any, Unpack

from google.adk.agents import BaseAgent, LlmAgent, RunConfig
Expand All @@ -90,6 +95,8 @@
from google.adk.sessions import BaseSessionService, Session
from google.adk.sessions.base_session_service import GetSessionConfig, ListSessionsResponse
from google.adk.tools.base_tool import BaseTool
from google.adk.tools.bash_tool import ExecuteBashTool
from google.adk.tools.skill_toolset import RunSkillScriptTool
from google.adk.tools.tool_context import ToolContext
from google.adk.workflow import BaseNode
from google.genai import types
Expand Down Expand Up @@ -265,14 +272,59 @@ async def add_stored(self, session: Session, event: Event) -> None:
await super().append_event(session, event)


_LOCAL_TOOLS = (ExecuteBashTool, RunSkillScriptTool)
"""ADK's built-in tools that work on the local machine (a shell, scripts), which run kept alive; and the
environment's (``EnvironmentToolset``: execute, read, write and edit a file), whose classes ADK keeps in private
modules, so they are known by their package (``_ENVIRONMENT_TOOLS``).

Only these: not ours (``_AdkTool``), a user's own function (``FunctionTool``, which includes
``LongRunningFunctionTool``), a sub-agent (``AgentTool``, whose model requests are kept alive by the plugin), nor
the tools that call a service (MCP, OpenAPI, search, ...). Code executors (``LlmAgent.code_executor``) run as a
processor of the flow, not as a tool."""

_ENVIRONMENT_TOOLS = "google.adk.tools.environment."
"""The package of the environment's tools (``EnvironmentToolset``)."""


def _local(tool: BaseTool) -> bool:
"""Whether ``tool`` is one of ADK's built-in local tools (see ``_LOCAL_TOOLS``)."""
return isinstance(tool, _LOCAL_TOOLS) or type(tool).__module__.startswith(_ENVIRONMENT_TOOLS)


class _Restate(BasePlugin):
"""Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, and give the model the
messages steered to the turn."""
"""Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, give the model the
messages steered to the turn, and keep the attempt alive while a model request or a built-in local tool runs.

A scope (``ctx.working``) is opened by one hook and closed by another, so each is kept per request or per call:
the run closes what is left when it ends (``release_all``)."""

def __init__(self, ctx: SessionContext, sessions: _Sessions) -> None:
super().__init__(name="restate")
self.ctx = ctx
self.sessions = sessions
self.requests: dict[tuple[str, str, str], AsyncExitStack] = {}
"""The scopes of the model requests in progress, by invocation, branch and agent: parallel agents each
have one."""
self.local: dict[str, AsyncExitStack] = {}
"""The scopes of the built-in local tool calls in progress, by call id: tools run in parallel."""

async def _keep(self) -> AsyncExitStack:
stack = AsyncExitStack()
await stack.enter_async_context(self.ctx.working())
return stack

@staticmethod
async def _close(stack: AsyncExitStack | None) -> None:
if stack is not None:
await stack.aclose()

async def release_all(self) -> None:
"""Close the scopes still open, as the run ends: whatever way, success, error or a stopped turn."""
stacks = [*self.requests.values(), *self.local.values()]
self.requests.clear()
self.local.clear()
for stack in stacks:
await self._close(stack)

async def before_model_callback(
self, *, callback_context: CallbackContext, llm_request: LlmRequest
Expand All @@ -287,16 +339,61 @@ async def before_model_callback(
await self.sessions.add_stored(callback_context.session, event)
if event.content is not None:
llm_request.contents.append(event.content.model_copy(deep=True))
# Kept alive from here, once the steering is done: the request is what follows (ADK resolves the model, runs
# the agent's own callbacks, and calls it). Closed by the response (``after_model_callback``), an error
# (``on_model_error_callback``), or the event that ends the step if a callback answered instead.
key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name)
await self._close(self.requests.pop(key, None))
self.requests[key] = await self._keep()
return None

async def after_model_callback(
self, *, callback_context: CallbackContext, llm_response: LlmResponse
) -> LlmResponse | None:
# A streamed reply comes as partial responses, then its whole: the request is on until the whole.
if not llm_response.partial:
key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name)
await self._close(self.requests.pop(key, None))
return None

async def on_model_error_callback(
self, *, callback_context: CallbackContext, llm_request: LlmRequest, error: Exception
) -> LlmResponse | None:
# Also called for a request that is cancelled (``error`` is the ``CancelledError``).
key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name)
await self._close(self.requests.pop(key, None))
return None

async def on_event_callback(self, *, invocation_context: Any, event: Event) -> Event | None:
# Backstop for what the callbacks miss: a response that a callback of the agent gave in place of the
# model's (no after or error callback runs then), or a tool call that one answered. An event of the
# agent that is whole ends its request, and a response to a call ends the call's scope.
if not event.partial:
await self._close(self.requests.pop(_request_key(event.invocation_id, event.branch, event.author), None))
for response in event.get_function_responses():
await self._close(self.local.pop(response.id or "", None))
return None

async def before_tool_callback(
self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext
) -> None:
await tool_call_started(self.ctx)
if _local(tool):
call_id = tool_context.function_call_id or ""
await self._close(self.local.pop(call_id, None))
self.local[call_id] = await self._keep()

async def after_tool_callback(
self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, result: dict[str, Any]
) -> dict[str, Any] | None:
await self._close(self.local.pop(tool_context.function_call_id or "", None))
return None

async def on_tool_error_callback(
self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, error: Exception
) -> dict[str, Any] | None:
# Also called for a call that is cancelled.
await self._close(self.local.pop(tool_context.function_call_id or "", None))
if type(tool) is BaseTool:
# A tool the agent does not have (ADK stands in a bare BaseTool for it): the model's to fix. Failing
# the attempt would not help, as the retry resumes the same call.
Expand All @@ -306,6 +403,10 @@ async def on_tool_error_callback(
return None


def _request_key(invocation: str, branch: str | None, agent: str) -> tuple[str, str, str]:
return (invocation, branch or "", agent)


# --- Tools ----------------------------------------------------------------------------------------


Expand Down Expand Up @@ -467,7 +568,8 @@ async def run(conversation: Conversation) -> str:
sessions = _Sessions(conversation)
# This package's plugin, after the app's own: a plugin of the app that answers in the model's place leaves
# the messages steered to the turn to a request that goes out.
plugins = [*app.plugins, _Restate(ctx, sessions)]
restate = _Restate(ctx, sessions)
plugins = [*app.plugins, restate]
runner = Runner(app=app.model_copy(update={"plugins": plugins}), session_service=sessions)
answer = ""
error: Event | None = None
Expand All @@ -488,6 +590,10 @@ async def run(conversation: Conversation) -> str:
except LlmCallsLimitExceededError as limit:
# A retry would carry on with a fresh count, and so would never end an agent that keeps calling tools.
raise TerminalError(f"the run reached its limit of model calls: {limit}") from limit
finally:
# The scopes the plugin's hooks left open (a callback of the agent answered in place of the model or a
# tool, a failed or stopped run): the attempt is not kept alive past the run.
await restate.release_all()
if error is not None:
# The model would not answer (a safety block, say), as it would not again: the prompt fails.
raise TerminalError(f"the model returned {error.error_code}: {error.error_message or ''}")
Expand Down Expand Up @@ -768,9 +874,11 @@ async def summarize(ctx: SessionContext, request: str) -> str:
llm = LlmAgent(name="summarizer", model=_model(await configured(model, ctx))).canonical_model
asked = LlmRequest(model=llm.model, contents=[types.Content(role="user", parts=[types.Part(text=request)])])
texts: list[str] = []
async for response in llm.generate_content_async(asked):
if response.content is not None:
texts.extend(part.text for part in response.content.parts or [] if part.text)
# Kept alive for the request alone.
async with ctx.working():
async for response in llm.generate_content_async(asked):
if response.content is not None:
texts.extend(part.text for part in response.content.parts or [] if part.text)
return "".join(texts)

return summarize
7 changes: 5 additions & 2 deletions python/restate/agents/langgraph/_agent.py
Original file line number Diff line number Diff line change
Expand Up @@ -351,7 +351,9 @@ async def call_model(state: State) -> dict[str, Any]:
# request has them after the results, and the state keeps them for the rest of the run, which goes on.
steered = typing.cast(list[BaseMessage], await take_steers(ctx, conversation))
messages = _latest([*state["messages"], *steered], window)
reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages)
# Kept alive for the request alone: not the steering before it, nor the commit after.
async with ctx.working():
reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages)
await conversation.turn.append_reply(reply)
return {"messages": [*steered, reply]}

Expand Down Expand Up @@ -651,7 +653,8 @@ def _summarizer(model: PerSession[ChatModel | None]) -> Summarize:
async def summarize(ctx: SessionContext, request: str) -> str:
chosen = await configured(model, ctx)
chat_model = chosen if isinstance(chosen, BaseChatModel) else _chat_model(model_of(chosen))
return _text(await chat_model.ainvoke([HumanMessage(request)]))
async with ctx.working():
return _text(await chat_model.ainvoke([HumanMessage(request)]))

return summarize

Expand Down
Loading
Loading