From 6d9941e7f86117a3aba82043a1d8cedd47b3aab5 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:11:37 +0000 Subject: [PATCH 1/6] Sessions: heartbeats only inside ctx.working; a shutdown is the default An attempt heartbeated on its own while nothing of Restate's was outstanding, or while an annotation was open. That kept a handler stuck on a promise of its own alive forever, and an annotation around a tool that waits for a signal held its process for the whole wait. - `ctx.working()`: an async context manager. While any scope is open the attempt heartbeats, whatever it waits for; outside, Restate shuts it down at its inactivity timeout and it suspends on its calls (or starts again at once, waiting for none). - The driver counts the scopes. A scope's first heartbeat comes at once only if the attempt wrote nothing for an interval, so the database's scope per call costs no message. - The database's own calls (open, statements, commits) run in a scope. - Annotations are traces only: they keep nothing alive. - `tool(keepalive=True)` runs a tool of your own in `ctx.working()`; by default a tool is not kept alive. - Tests: the driver's and the server's heartbeat tests follow the new rule; the scripted frameworks of the long-work and dead-attempt tests run their model in `ctx.working()`; the test `Host` counts open scopes. --- docs/agents/turns.md | 20 ++++--- python/restate/agents/_tools.py | 29 ++++++++-- python/restate/sessions/_api.py | 21 +++++++ python/restate/sessions/_database.py | 18 +++++- python/restate/sessions/_driver.py | 82 ++++++++++++++++------------ python/restate/sessions/_object.py | 6 ++ python/restate/sessions/_session.py | 6 +- python/restate/sessions/internal.py | 3 +- tests/agents_dead_attempt_server.py | 13 +++-- tests/agents_long_work_server.py | 7 ++- tests/agents_tools.py | 23 ++++++++ tests/sessions_driver.py | 63 ++++++++++++++------- tests/sessions_harness.py | 29 ++++------ tests/sessions_object.py | 12 ++++ 14 files changed, 230 insertions(+), 102 deletions(-) diff --git a/docs/agents/turns.md b/docs/agents/turns.md index 2224e44..1af853a 100644 --- a/docs/agents/turns.md +++ b/docs/agents/turns.md @@ -457,15 +457,17 @@ sequenceDiagram message starts afresh: the cancelled prompt is over. - The cancellation reaches the turn's work at once. Restate shuts down an attempt it has heard nothing from for the inactivity timeout (a minute by - default), and the next attempt runs the turn from its start. So while the - turn works on its own (a model request, a tool), its attempt tells Restate - every 20 seconds that it makes progress, and Restate keeps it however long - the step takes. While the turn waits for an answer of Restate's instead (a - tool waiting for an approval, a call), it tells Restate nothing: Restate - shuts the attempt down after the timeout, so that a wait of days holds no - process, and the next attempt gets the cancellation as it starts. A - deployment that sets an inactivity timeout under 20 seconds is shut down - between two of those messages. + default), and the next attempt runs the turn from its start. So while a + model request runs, or a framework's built-in local tool (a shell, file + edits), the attempt tells Restate every 20 seconds that it makes progress + (`ctx.working`), and Restate keeps it however long the step takes. Nothing + else of a turn does: a tool of your own only with `tool(keepalive=True)`. + While the turn waits for an answer of Restate's (a tool waiting for an + approval, a call), it tells Restate nothing: Restate shuts the attempt down + after the timeout, so that a wait of days holds no process, and the next + attempt gets the cancellation as it starts. A deployment that sets an + inactivity timeout under 20 seconds is shut down between two of those + messages. - A cancellation that comes once the turn answered changes nothing: the answer stands. - A tool cut short by the stop raises the stop's reason, so the model never diff --git a/python/restate/agents/_tools.py b/python/restate/agents/_tools.py index de02906..c892a01 100644 --- a/python/restate/agents/_tools.py +++ b/python/restate/agents/_tools.py @@ -126,6 +126,7 @@ class Tool: description: What the tool does, for the model. input_schema: The JSON schema of the tool's input, which the framework passes to the model. + keepalive: Whether the tool runs in ``ctx.working``: see ``tool``. """ name: str @@ -133,6 +134,7 @@ class Tool: input_schema: dict[str, Any] _run: Callable[[dict[str, Any], ToolCall], Awaitable[Any]] _validate: Callable[[dict[str, Any]], Any] + keepalive: bool = False async def run(self, args: object, call: ToolCall) -> str: """Run the tool with the model's raw arguments, and return its result as text for the model. @@ -157,6 +159,9 @@ async def run(self, args: object, call: ToolCall) -> str: except Exception as error: # pydantic's ValidationError, for the model to fix return f"Error: the arguments do not fit the tool's input: {error}" try: + if self.keepalive: + async with call.ctx.working(): + return _text(await self._run(arguments, call)) return _text(await self._run(arguments, call)) except BaseException as error: # The turn stopped (its invocation was cancelled, or its attempt @@ -219,15 +224,23 @@ def _text(value: object) -> str: @typing.overload -def tool(fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None) -> Tool: ... +def tool( + fn: Callable[..., Any], /, *, name: str | None = None, description: str | None = None, keepalive: bool = False +) -> Tool: ... @typing.overload -def tool(*, name: str | None = None, description: str | None = None) -> Callable[[Callable[..., Any]], Tool]: ... +def tool( + *, name: str | None = None, description: str | None = None, keepalive: bool = False +) -> Callable[[Callable[..., Any]], Tool]: ... def tool( - fn: Callable[..., Any] | None = None, *, name: str | None = None, description: str | None = None + fn: Callable[..., Any] | None = None, + *, + name: str | None = None, + description: str | None = None, + keepalive: bool = False, ) -> Tool | Callable[[Callable[..., Any]], Tool]: """Create a tool from a function, sync or async. @@ -251,12 +264,19 @@ async def get_weather(city: str) -> str: result, and can act on it. Any other exception fails the attempt, and Restate retries it. + A tool does not keep its attempt alive: one that works for longer than + Restate's inactivity timeout (a minute, by default) is cut off, as by a + crash. One that must work that long sets ``keepalive``, and runs in + ``ctx.working``; it keeps long waits on Restate (a call, a signal) out of + it, or holds its process for as long as they take. + Args: fn: The function. Omit it to use ``tool(name=..., description=...)`` as a decorator. name: The tool's name. Defaults to the function's name. description: What the tool does, for the model. Defaults to the function's docstring. + keepalive: Keep the attempt alive while the tool runs. Returns: The tool, or a decorator that creates it. @@ -267,7 +287,7 @@ async def get_weather(city: str) -> str: if fn is None: def decorate(fn: Callable[..., Any]) -> Tool: - return tool(fn, name=name, description=description) + return tool(fn, name=name, description=description, keepalive=keepalive) return decorate pydantic = _pydantic() @@ -295,6 +315,7 @@ def validate(args: dict[str, Any]) -> dict[str, Any]: input_schema=typing.cast(dict[str, Any], model.model_json_schema()), _run=run, _validate=validate, + keepalive=keepalive, ) diff --git a/python/restate/sessions/_api.py b/python/restate/sessions/_api.py index 7b8970b..25a3ac7 100644 --- a/python/restate/sessions/_api.py +++ b/python/restate/sessions/_api.py @@ -21,6 +21,7 @@ import typing from collections.abc import Awaitable, Callable, Sequence +from contextlib import AbstractAsyncContextManager from datetime import timedelta from typing import Any, Protocol @@ -352,6 +353,26 @@ def cancel(self, invocation_id: str) -> None: """ ... + def working(self) -> AbstractAsyncContextManager[None]: + """Keep the attempt alive while the body of an ``async with`` runs. + + For work outside Restate that may outlast Restate's inactivity timeout + (a minute, by default), such as a model's request:: + + async with ctx.working(): + reply = await model.generate(prompt) + + Outside it, an attempt Restate hears nothing from for that long is shut + down: it suspends on the calls it waits for, and a new attempt runs the + handler from the start when one answers (or at once, if it waits for + none). So a wait of days holds no process, and work cut off that way is + done again. Inside it, the attempt sends a heartbeat every 20 seconds, + whatever it waits for: keep a long wait on Restate (a call, a signal) + out of it. The database's own calls keep the attempt alive by + themselves. + """ + ... + class ViewContext(Caller, Protocol): """The context of a session object's view. diff --git a/python/restate/sessions/_database.py b/python/restate/sessions/_database.py index 33f16d1..c194c66 100644 --- a/python/restate/sessions/_database.py +++ b/python/restate/sessions/_database.py @@ -121,7 +121,8 @@ async def open(namespace: str, store: Store, *, reader: bool = False) -> Connect vm = getattr(store, "vm", None) name = getattr(store, "database_name", None) if vm is not None else None try: - native = await _settled(sql_open(namespace, boundary.dispatch, boundary.lists_keys, reader, vm, name)) + async with boundary.alive(): + native = await _settled(sql_open(namespace, boundary.dispatch, boundary.lists_keys, reader, vm, name)) except Exception as error: if boundary.failed is not None: raise boundary.failed from error @@ -155,6 +156,11 @@ class _Boundary: def __init__(self, store: Store, loop: asyncio.AbstractEventLoop) -> None: self.store: Store = store self.loop: asyncio.AbstractEventLoop = loop + # Keeps the attempt alive while a call of the database runs: the store of an attempt on Restate has it + # (its handler's ``working``). The database's calls are local work that never waits on Restate for long. + self._working: Callable[[], contextlib.AbstractAsyncContextManager[Any]] | None = getattr( + store, "working", None + ) # The store's optional methods (see `Store`). self._keys: Callable[[], Sequence[str] | Awaitable[Sequence[str]]] | None = getattr(store, "keys", None) self.lists_keys: bool = callable(self._keys) @@ -169,6 +175,10 @@ def _keep(self, error: BaseException) -> None: if self.failed is None: self.failed = error + def alive(self) -> contextlib.AbstractAsyncContextManager[Any]: + """A scope that keeps the attempt alive, if the store has one: around each call of the database.""" + return self._working() if self._working is not None else contextlib.nullcontext() + def dispatch(self, op: int, key: Any, value: bytes | None, reply: SqlReply | None) -> None: """Handle one request of the extension; ``key`` is a store key, or empty for a sync or a commit.""" if op in (SQL_SET, SQL_CLEAR): @@ -287,7 +297,8 @@ async def _request(self, call: Callable[[], T | Awaitable[T]]) -> T: that fails right away); it returns a value or an awaitable of one. """ try: - return await _settled(call()) + async with self._boundary.alive(): + return await _settled(call()) except Exception as error: raise self._failure(error) from None @@ -374,7 +385,8 @@ async def transaction(self) -> AsyncGenerator[ConnectionTransaction, None]: # that wrote nothing does not sync, so they run afterwards instead. self._boundary.with_sync = tx.commit_runs() try: - await _settled(self._native.commit(records)) + async with self._boundary.alive(): + await _settled(self._native.commit(records)) except Exception as error: # A failed COMMIT (a deferred constraint, say) leaves the # transaction open; without a rollback, later calls would run diff --git a/python/restate/sessions/_driver.py b/python/restate/sessions/_driver.py index 07ce799..68cfbb6 100644 --- a/python/restate/sessions/_driver.py +++ b/python/restate/sessions/_driver.py @@ -34,7 +34,7 @@ import operator import traceback import typing -from collections.abc import Awaitable, Callable, Generator +from collections.abc import AsyncIterator, Awaitable, Callable, Generator from datetime import timedelta from typing import Any @@ -63,20 +63,14 @@ T = typing.TypeVar("T") HEARTBEAT_INTERVAL = timedelta(seconds=20) -"""How often an attempt that works outside Restate tells it that it makes progress. +"""How often an attempt in ``working`` tells Restate that it makes progress. Restate shuts down an attempt it heard nothing from for its inactivity timeout -(a minute, by default), and the next one runs the handler from the start: a -model request that takes longer would be made again by every attempt, and -never answered. So an attempt heartbeats while its handler has nothing of -Restate's outstanding (it is then working on its own: a model request, a -tool), and while an annotation is open, which says the same of work that goes -on beside a call. With an answer of Restate's outstanding and no annotation, -it does not: Restate shuts the attempt down after the timeout, and starts the -next when the answer comes, so that a wait of days holds no process. An -answer is outstanding from the moment the call is made until Restate gives -it, whether or not the handler awaits it: work done after a call whose result -is left aside gets no heartbeat until that call is answered. +(a minute, by default). That is the default: the attempt suspends on what it +waits for (or, waiting for nothing, starts again at once), and a wait of days +holds no process. Work that must not be cut off (a model request) says so with +``working``: while any of it runs, the attempt heartbeats, whatever else it +waits for. A deployment that shortens the timeout must shorten this with it. """ SYNC_KEY = "v22/sync" @@ -132,9 +126,6 @@ def __init__( # The handles a cancellation leaves to settle, and is never carried to: the invocation # ids of the calls and sends, and the reads and acks of the database's store. self._left_to_settle: set[int] = set() - # The store's among them: the database waits for those, not the handler, and they come - # promptly, so they do not count as an answer of Restate's the handler waits for. - self._store_waits: set[int] = set() # A cancellation that failed nothing the handler awaits, carried to the next thing it # does: a result of Restate's (``future``), or a call of its database # (``take_carried_cancellation``). A retry of a cancelled invocation knows of it before @@ -151,8 +142,11 @@ def __init__( self._writer: asyncio.Task[None] = asyncio.ensure_future(self._write_output()) self._pump: asyncio.Task[None] | None = None self._handler: asyncio.Future[bytes] | None = None - self._open_traces: int = 0 + # The ``working`` scopes open: the attempt heartbeats while there is one. + self._scopes: int = 0 self._heartbeats: asyncio.Task[None] | None = None + # When the attempt last wrote to Restate, which restarts Restate's inactivity timer. + self._wrote_at: float = self._loop.time() self._end_listeners: list[Callable[[], None]] = [] self.finished: asyncio.Event = asyncio.Event() self.ended: BaseException | None = None @@ -193,7 +187,6 @@ async def run(self, handler: Callable[[], Awaitable[bytes]]) -> None: (a model request, say) stops, since nothing it does would count any more. """ self._handler = asyncio.ensure_future(self._cancelled_calls_first(handler)) - self._heartbeats = asyncio.ensure_future(self._heartbeat()) try: output = await self._handler self._last(lambda vm: vm.end_success(output)) @@ -304,13 +297,13 @@ def _tick(self) -> None: output = self.vm.take_output() if output: self._output.put_nowait(output) + self._wrote_at = self._loop.time() if self.ended is not None: return try: while (notification := self.vm.take_notification()) is not None: handle, value = notification self._left_to_settle.discard(handle) - self._store_waits.discard(handle) call = self._unanswered.pop(handle, None) if call is not None: call.answered = True @@ -409,7 +402,6 @@ def store_future(self, handle: int) -> asyncio.Future[Any]: with a journal codec. """ self._left_to_settle.add(handle) - self._store_waits.add(handle) return self._future(handle) def _future(self, handle: int) -> asyncio.Future[Any]: @@ -482,27 +474,40 @@ async def cancel_calls(self) -> list[str]: cancelled.append(invocation_id) return cancelled - def trace_opened(self) -> None: - """An annotation opened: the attempt heartbeats while any is, whatever else is outstanding.""" - self._open_traces += 1 + # --- Keeping the attempt alive - def trace_closed(self) -> None: - self._open_traces -= 1 + @contextlib.asynccontextmanager + async def working(self) -> AsyncIterator[None]: + """Keep the attempt alive while the body runs (see ``HEARTBEAT_INTERVAL``). - def _awaits_restate(self) -> bool: - """Whether an answer of Restate's that the handler made for is outstanding: a result or an invocation id.""" - return any(handle not in self._store_waits for handle in self._futures) + One heartbeat every interval, from the first scope opening to the last + closing. The first comes at once if the attempt has been silent for an + interval already (most of the timeout may be gone), so the silence never + reaches twice that; otherwise the scopes the database opens for each of + its calls would each send one. + """ + self._scopes += 1 + if self._scopes == 1 and self.ended is None: + self._heartbeats = asyncio.ensure_future(self._heartbeat()) + try: + yield + finally: + self._scopes -= 1 + if self._scopes == 0 and self._heartbeats is not None: + self._heartbeats.cancel() + self._heartbeats = None async def _heartbeat(self) -> None: - """Tell Restate the attempt makes progress, while it works outside Restate (see ``HEARTBEAT_INTERVAL``).""" + """Tell Restate the attempt makes progress, while a ``working`` scope is open.""" + interval = HEARTBEAT_INTERVAL.total_seconds() + if self._loop.time() - self._wrote_at < interval: + await asyncio.sleep(interval) while True: - await asyncio.sleep(HEARTBEAT_INTERVAL.total_seconds()) - if self._open_traces == 0 and self._awaits_restate(): - continue try: self.act(lambda vm: vm.heartbeat()) except SdkInternalBaseException: return + await asyncio.sleep(interval) def _cancelled_error() -> TerminalError: @@ -678,13 +683,18 @@ def generic_send( def attach_invocation(self, invocation_id: str) -> Awaitable[bytes]: return self.driver.future(self.driver.act(lambda vm: vm.attach_invocation(invocation_id))) + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + """The handler's ``ctx.working``: the attempt is kept alive while its body runs.""" + return self.driver.working() + + def database_working(self) -> contextlib.AbstractAsyncContextManager[None]: + """The same, around each call of the database (see ``_Boundary.alive``).""" + return self.driver.working() + def open_annotation(self, name: str, parent: int | None = None) -> int | None: - trace = self.driver.act(lambda vm: vm.start_trace(name, parent)) - self.driver.trace_opened() - return trace + return self.driver.act(lambda vm: vm.start_trace(name, parent)) def close_annotation(self, handle: int, value: bytes, failure: str | None) -> None: - self.driver.trace_closed() try: if failure is None: self.driver.act(lambda vm: vm.end_trace_success(handle, value)) diff --git a/python/restate/sessions/_object.py b/python/restate/sessions/_object.py index e62a0d5..ba96a5c 100644 --- a/python/restate/sessions/_object.py +++ b/python/restate/sessions/_object.py @@ -30,6 +30,7 @@ async def list_notes(ctx: sessions.ViewContext) -> list[str]: from __future__ import annotations import asyncio +import contextlib import functools import inspect import json @@ -99,6 +100,11 @@ class AttemptContext(Calls, Sends): def cancel(self, invocation_id: str) -> None: self._cancel(invocation_id) + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + # The host of an attempt on Restate keeps it alive; a test's host may count the scopes; else nothing to do. + working = getattr(self._call_host, "working", None) + return working() if working is not None else contextlib.nullcontext() + @dataclass(frozen=True, eq=False) class AttemptViewContext(Calls): diff --git a/python/restate/sessions/_session.py b/python/restate/sessions/_session.py index d0c8517..b5ae9dd 100644 --- a/python/restate/sessions/_session.py +++ b/python/restate/sessions/_session.py @@ -28,7 +28,7 @@ import typing import uuid from collections.abc import AsyncGenerator, Awaitable, Callable, Sequence -from contextlib import asynccontextmanager +from contextlib import AbstractAsyncContextManager, asynccontextmanager from dataclasses import dataclass, field from datetime import timedelta from typing import Any, Protocol @@ -169,6 +169,8 @@ def __init__(self, host: SqlHost, name: str | None) -> None: self._name: str | None = name self.vm: PyExplicitRecoveryVM | None = host.vm self.database_name: str | None = name + # What keeps the attempt alive while a call of the database runs (an attempt on Restate's), if any. + self.working: Callable[[], AbstractAsyncContextManager[Any]] | None = getattr(host, "database_working", None) def _key(self, key: str) -> str: return key if self._name is None else _beside(self._name, key) @@ -207,6 +209,8 @@ def __init__(self, host: SqlHost, name: str | None) -> None: self._name: str | None = name self.vm: PyExplicitRecoveryVM | None = host.vm self.database_name: str | None = name + # What keeps the attempt alive while a call of the database runs (an attempt on Restate's), if any. + self.working: Callable[[], AbstractAsyncContextManager[Any]] | None = getattr(host, "database_working", None) def get(self, key: str) -> Awaitable[bytes | None]: key = key if self._name is None else _beside(self._name, key) diff --git a/python/restate/sessions/internal.py b/python/restate/sessions/internal.py index 5dc6cbe..fdd4a07 100644 --- a/python/restate/sessions/internal.py +++ b/python/restate/sessions/internal.py @@ -21,7 +21,8 @@ - ``cancelled`` and ``cancel_calls``: explicit cancellation, for handlers declared with ``explicit_cancellation=True``. - ``annotate``: time some work (a model request, say) in the journal, as a - run that Restate's UI shows; nothing waits for it. + run that Restate's UI shows; nothing waits for it, and it keeps nothing + alive (``ctx.working`` does). - ``hide_entries``: metadata that has Restate's UI hide the journal entries of some names (a library's own calls, say), as it hides the database's. - ``start`` and ``attach_invocation``: send, keeping the invocation id, diff --git a/tests/agents_dead_attempt_server.py b/tests/agents_dead_attempt_server.py index 560fb63..1ccb8d1 100644 --- a/tests/agents_dead_attempt_server.py +++ b/tests/agents_dead_attempt_server.py @@ -45,7 +45,7 @@ def anyio_backend(): """How many model requests ran to their end, by the conversation's key.""" SLOW = 4.0 -"""How long a slow model request takes: longer than the inactivity timeout, shorter than a heartbeat (20 s).""" +"""How long a slow model request takes: longer than the inactivity timeout, shorter than a heartbeat's interval (20 s).""" async def model(key: str) -> str: @@ -61,12 +61,14 @@ async def model(key: str) -> str: async def answer(ctx: sessions.SessionContext, text: str) -> str: - """A scripted framework whose model request goes through the turn's stop signal, as the frameworks' do.""" + """A scripted framework whose model request goes through the turn's stop signal, in ``ctx.working``, as the + frameworks' do.""" async def work(conversation: Conversation) -> str: stop = stop_signal_of(ctx) assert stop is not None - await stop.guard(model(ctx.key)) + async with ctx.working(): + await stop.guard(model(ctx.key)) await conversation.turn.append_reply({"role": "assistant", "text": f"echo: {text}"}) return f"echo: {text}" @@ -84,8 +86,9 @@ async def harness(): async def test_a_dead_attempts_model_request_is_cancelled(harness: HarnessEnvironment): - # The attempt heartbeats every 20 s, by default: past the agent's inactivity timeout of a second, Restate - # shuts each slow attempt down in the middle of its model request, and the next attempt asks again. + # The model's request is kept alive, but the heartbeat comes every 20 s, by default: past the agent's inactivity + # timeout of a second, Restate shuts each slow attempt down in the middle of its model request, and the next + # attempt asks again. key = str(uuid.uuid4()) async with httpx2.AsyncClient(timeout=60) as ingress: sent = await ingress.post( diff --git a/tests/agents_long_work_server.py b/tests/agents_long_work_server.py index b3cae99..273311e 100644 --- a/tests/agents_long_work_server.py +++ b/tests/agents_long_work_server.py @@ -43,13 +43,14 @@ def anyio_backend(): async def answer(ctx: sessions.SessionContext, text: str) -> str: """A scripted framework whose model thinks for longer than the inactivity timeout, and says nothing meanwhile. - It is what a turn of any framework is while its model request runs: local work, with nothing of Restate's - outstanding. + It is what a turn of any framework is while its model request runs: local work in ``ctx.working``, as the + frameworks' requests run. """ async def work(conversation: Conversation) -> str: requests[ctx.key] = requests.get(ctx.key, 0) + 1 - await asyncio.sleep(3) + async with ctx.working(): + await asyncio.sleep(3) await conversation.turn.append_reply({"role": "assistant", "text": f"echo: {text}"}) return f"echo: {text}" diff --git a/tests/agents_tools.py b/tests/agents_tools.py index 1378fba..41be1ec 100644 --- a/tests/agents_tools.py +++ b/tests/agents_tools.py @@ -351,3 +351,26 @@ async def test_a_langgraph_wait_from_a_session_object_that_is_no_agent_fails_the await internal.invoke(desk, "ask", host, request(), "refund o1") assert raised.value.message.startswith(f"{what} {NEEDS_AGENT}"), raised.value.message assert host.calls == [] + + +async def test_a_tool_runs_kept_alive_only_with_keepalive(): + host = Host(Backend(), service="runner", key="alice") + seen: dict[str, int] = {} + + @tool + async def quick() -> str: + """Quick.""" + seen["quick"] = host.open_scopes + return "done" + + @tool(keepalive=True) + async def long() -> str: + """Long.""" + seen["long"] = host.open_scopes + return "done" + + tools["quick"], tools["long"] = quick, long + await run_tool(host, "quick", {}, "c1") + await run_tool(host, "long", {}, "c2") + assert seen == {"quick": 0, "long": 1} + assert host.open_scopes == 0 diff --git a/tests/sessions_driver.py b/tests/sessions_driver.py index 3b6c175..b8b2736 100644 --- a/tests/sessions_driver.py +++ b/tests/sessions_driver.py @@ -148,18 +148,28 @@ async def commits(ctx: sessions.SessionContext) -> str: @notes.handler() -async def works_alone(ctx: sessions.SessionContext) -> str: - """Works outside Restate, with nothing of Restate's outstanding, until the test lets it go on.""" - await gate_of().wait() +async def works_alone(ctx: sessions.SessionContext, kept: bool) -> str: + """Works outside Restate, with nothing of Restate's outstanding, in ``ctx.working`` or not, until the test lets it go + on.""" + if kept: + async with ctx.working(): + await gate_of().wait() + else: + await gate_of().wait() return "done" @notes.handler() -async def works_beside_a_call(ctx: sessions.SessionContext, annotated: bool) -> str: - """Waits for a call that is never answered, working meanwhile, inside an annotation or not.""" +async def works_beside_a_call(ctx: sessions.SessionContext, how: str) -> str: + """Waits for a call that is never answered, working meanwhile: in ``ctx.working`` (``kept``), in an annotation + (``annotated``), or neither.""" pending = asyncio.ensure_future(ctx.generic_call("Callee", "echo", "A", key="k")) - annotation = internal.annotate(ctx, "tool") if annotated else None - await gate_of().wait() + annotation = internal.annotate(ctx, "tool") if how == "annotated" else None + if how == "kept": + async with ctx.working(): + await gate_of().wait() + else: + await gate_of().wait() if annotation is not None: annotation.end() pending.cancel() @@ -611,32 +621,43 @@ def quick_heartbeats(monkeypatch) -> None: monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.02)) -async def test_an_attempt_heartbeats_while_its_handler_works_with_nothing_of_restates_outstanding( - gate: Gate, quick_heartbeats: None -): - # A model request, say: Restate hears nothing from the attempt meanwhile, and shuts down one it heard nothing - # from for its inactivity timeout. The next attempt would make the request again, and never get past it. +async def test_an_attempt_does_not_heartbeat_outside_working(gate: Gate, quick_heartbeats: None): + # Suspension is the default: Restate shuts down an attempt it heard nothing from for its inactivity timeout, and + # starts the next when what it waits for answers (or at once, if nothing). + side = RestateSide() + side.start("kh1", False) + assert await beats_while_at(gate, "works_alone", side) == 0, describe(side) + assert output_of(side.sent()) == "done", describe(side) + + +async def test_an_attempt_heartbeats_in_working(gate: Gate, quick_heartbeats: None): + # A model request, say: the next attempt would make it again, and never get past it. side = RestateSide() - side.start("kh1", None) + side.start("kh2", True) assert await beats_while_at(gate, "works_alone", side) >= 3, describe(side) assert output_of(side.sent()) == "done", describe(side) -async def test_an_attempt_does_not_heartbeat_while_an_answer_of_restates_is_outstanding( +async def test_an_attempt_heartbeats_in_working_beside_an_outstanding_answer(gate: Gate, quick_heartbeats: None): + # Work beside a call it waits for: a model's request beside a tool's call, say. + side = RestateSide() + side.start("kh3", "kept") + assert await beats_while_at(gate, "works_beside_a_call", side) >= 3, describe(side) + + +async def test_an_attempt_does_not_heartbeat_while_it_waits_on_restate_outside_working( gate: Gate, quick_heartbeats: None ): - # A wait of days for an approval holds no process: Restate shuts the attempt down, and starts the next one when - # the answer comes. + # A wait of days for an approval holds no process. side = RestateSide() - side.start("kh2", False) + side.start("kh4", "neither") assert await beats_while_at(gate, "works_beside_a_call", side) == 0, describe(side) -async def test_an_open_annotation_heartbeats_beside_an_outstanding_answer(gate: Gate, quick_heartbeats: None): - # The annotation says the handler is working, on a tool beside the call it waits for. +async def test_an_open_annotation_is_a_trace_and_keeps_nothing_alive(gate: Gate, quick_heartbeats: None): side = RestateSide() - side.start("kh3", True) - assert await beats_while_at(gate, "works_beside_a_call", side) >= 3, describe(side) + side.start("kh5", "annotated") + assert await beats_while_at(gate, "works_beside_a_call", side) == 0, describe(side) async def test_a_read_that_asks_restate_nothing_sees_a_carried_cancellation_too_once(gate: Gate): diff --git a/tests/sessions_harness.py b/tests/sessions_harness.py index d525709..24900a6 100644 --- a/tests/sessions_harness.py +++ b/tests/sessions_harness.py @@ -260,17 +260,18 @@ async def waits_for_a_crawl(ctx: sessions.SessionContext) -> list[Any]: @slowpoke.handler() async def thinks(ctx: sessions.SessionContext) -> int: - """Works locally longer than the inactivity timeout, inside an annotation.""" + """Works locally longer than the inactivity timeout, in ``ctx.working`` (and an annotation, which only times it).""" n = attempt(ctx) timed = internal.annotate(ctx, "model: thinking") - await asyncio.sleep(2.5) + async with ctx.working(): + await asyncio.sleep(2.5) timed.end() return n @slowpoke.handler() async def daydreams(ctx: sessions.SessionContext) -> int: - """Works locally longer than the inactivity timeout, telling Restate nothing.""" + """Works locally longer than the inactivity timeout, outside ``ctx.working``: telling Restate nothing.""" n = attempt(ctx) await asyncio.sleep(2.5) daydreamt[ctx.key] = daydreamt.get(ctx.key, 0) + 1 @@ -449,30 +450,20 @@ async def test_a_shut_down_attempt_resumes_when_its_call_completes(harness: Harn assert n >= 2 -async def test_an_open_annotation_keeps_the_attempt_alive(harness: HarnessEnvironment, monkeypatch): +async def test_work_in_working_keeps_the_attempt_alive(harness: HarnessEnvironment, monkeypatch): + # The attempt heartbeats in `ctx.working`: here faster than the object's inactivity timeout of a second, so the + # work is done in the first attempt. monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.3)) key = str(uuid.uuid4()) assert await harness.client.object_call(thinks, key=key, arg=None) == 1 -async def test_local_work_keeps_its_attempt_alive_with_nothing_of_restates_outstanding( - harness: HarnessEnvironment, monkeypatch -): - # The attempt heartbeats while its handler works on its own, annotation or not: here faster than the object's - # inactivity timeout of a second, so the work is done in the first attempt. +async def test_local_work_outside_working_is_shut_down_and_restarted(harness: HarnessEnvironment, monkeypatch): + # However quick the heartbeat, none is sent outside `ctx.working`: each attempt is shut down after a second with + # nothing heard from it, and restarted right away. monkeypatch.setattr(_driver, "HEARTBEAT_INTERVAL", timedelta(seconds=0.3)) - key = str(uuid.uuid4()) - assert await harness.client.object_call(daydreams, key=key, arg=None) == 1 - assert daydreamt.get(key, 0) == 1 - - -async def test_local_work_is_shut_down_and_restarted_when_the_heartbeat_is_slower_than_the_inactivity_timeout( - harness: HarnessEnvironment, -): key = str(uuid.uuid4()) sent = await harness.client.object_send(daydreams, key=key, arg=None) - # The heartbeat comes every 20 seconds, and the object's inactivity timeout is a second: each attempt is shut - # down after a second with nothing heard from it, and restarted right away. for _ in range(300): if attempts.get(key, 0) >= 4: break diff --git a/tests/sessions_object.py b/tests/sessions_object.py index e409993..cfecee1 100644 --- a/tests/sessions_object.py +++ b/tests/sessions_object.py @@ -13,6 +13,7 @@ # pylint: disable=missing-function-docstring,missing-class-docstring import asyncio +import contextlib import itertools import json import typing @@ -114,6 +115,17 @@ class Host: cancelled: list[str] = field(default_factory=list) # How many more syncs go through before the process crashes: every sync after fails, losing its writes. _syncs_left: int | None = None + # The handler's ``ctx.working`` scopes open now: an attempt on Restate heartbeats while there is one. + open_scopes: int = 0 + + @contextlib.asynccontextmanager + async def working(self): + """The handler's ``ctx.working``, counted: a test reads ``open_scopes`` while some work of its own runs.""" + self.open_scopes += 1 + try: + yield + finally: + self.open_scopes -= 1 def crash_after(self, syncs: int) -> None: """Let ``syncs`` more syncs through, then crash: every later sync fails.""" From daa129eb176805d54adac2948d921eef10e0c4be Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:16:08 +0000 Subject: [PATCH 2/6] Agents: OpenAI's and Pydantic AI's model requests and built-in local tools run kept alive - OpenAI: each model request is kept alive from the end of `on_llm_start` to the top of `on_llm_end`, so the steering and commits around it are not. The SDK's own tools (shell, local shell, apply_patch, computer) are kept alive per call; a function tool is the user's, and is not. The summarizer's request too. Whatever a run leaves open is closed as it ends. - Pydantic AI: the model the run gets is wrapped, so exactly each request (streamed or not) is kept alive; a compaction's too. The harness's file system tools (by their kind) and its shell (by the names it gives its tools) are kept alive per call; `code_mode` is not, since it runs the user's tools. Deferred and approval waits stay outside. Known gap: the model requests of an OpenAI agent used as a tool of another (`Agent.as_tool`) are not kept alive: the SDK gives its nested run no hooks. --- python/restate/agents/openai/_agent.py | 58 ++++++-- python/restate/agents/pydantic_ai/_agent.py | 89 +++++++++++- tests/agents_openai.py | 136 ++++++++++++++++++ tests/agents_pydantic_ai.py | 144 ++++++++++++++++++++ tests/agents_pydantic_ai_snapshots.py | 26 ++++ 5 files changed, 439 insertions(+), 14 deletions(-) diff --git a/python/restate/agents/openai/_agent.py b/python/restate/agents/openai/_agent.py index 4a4a10d..365d39b 100644 --- a/python/restate/agents/openai/_agent.py +++ b/python/restate/agents/openai/_agent.py @@ -114,6 +114,7 @@ import json import typing from collections.abc import Awaitable, Sequence +from contextlib import AsyncExitStack from typing import Any, Unpack from agents import ( @@ -320,9 +321,31 @@ def __init__(self, ctx: SessionContext, conversation: Conversation, config: RunC """The model request in progress, timed in the journal.""" self.calls: dict[str, Annotation] = {} """The tool calls in progress, timed in the journal, by call id.""" - - def ended(self, error: BaseException | None) -> None: - """End the timing of the model request and tool calls still in progress, as the run ends with ``error``.""" + self.alive: AsyncExitStack | None = None + """The ``ctx.working`` scope of the model request in progress: just the request, not the work around it.""" + self.local: dict[str, AsyncExitStack] = {} + """The ``ctx.working`` scopes of the built-in local tool calls in progress, by call id.""" + + async def _keep(self) -> AsyncExitStack: + """Return a scope that keeps the attempt alive until closed.""" + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + return stack + + async def _release(self) -> None: + """Close the model request's scope, if open.""" + stack, self.alive = self.alive, None + if stack is not None: + await stack.aclose() + + async def ended(self, error: BaseException | None) -> None: + """End the timing of the model request and tool calls still in progress, and close the scopes still open, as + the run ends with ``error``.""" + stacks = [*self.local.values(), *([self.alive] if self.alive is not None else [])] + self.local.clear() + self.alive = None + for stack in stacks: + await stack.aclose() timed = [*self.calls.values(), *([self.request] if self.request is not None else [])] self.request = None self.calls.clear() @@ -339,6 +362,9 @@ async def on_llm_start( # Never so: a run makes one request at a time. self.request.fail("another request started") self.request = annotate(self.ctx, f"model: {_model_name(self.model or agent.model)}") + # Kept alive for the request alone, as this hook ends: the steering and commits before it are not it. + await self._release() + self.alive = await self._keep() async def model_input(self, data: CallModelData[Any]) -> ModelInputData: """The run's ``call_model_input_filter``: store the tools' outputs before the next request, and take the @@ -381,6 +407,7 @@ async def _filtered(self, data: CallModelData[Any]) -> ModelInputData: async def on_llm_end(self, context: RunContextWrapper[Any], agent: Agent[Any], response: ModelResponse) -> None: # The request ends here, before the reply is stored. + await self._release() if self.request is not None: self.request.end(_usage(agent, response)) self.request = None @@ -392,16 +419,26 @@ async def on_tool_start(self, context: RunContextWrapper[Any], agent: Agent[Any] self.running += 1 await tool_call_started(self.ctx) # Timed from here: the settling of the earlier calls is not the tool's. - self.calls[_call_key(context, tool)] = annotate(self.ctx, f"tool: {tool.name}") + key = _call_key(context, tool) + self.calls[key] = annotate(self.ctx, f"tool: {tool.name}") + if not isinstance(tool, FunctionTool): + # The SDK's own tools (shell, apply_patch, computer...) run local work: kept alive. A function tool is + # the user's, and is not. + self.local[key] = await self._keep() async def on_handoff(self, context: RunContextWrapper[Any], from_agent: Agent[Any], to_agent: Agent[Any]) -> None: # The turn carries the agent it handed off to, even if it ends before that agent replies. self.conversation.turn.carry(agent=to_agent.name) async def on_tool_end(self, context: RunContextWrapper[Any], agent: Agent[Any], tool: SdkTool, result: Any) -> None: - if (call := self.calls.pop(_call_key(context, tool), None)) is not None: - call.end() - self.running -= 1 + key = _call_key(context, tool) + try: + if (stack := self.local.pop(key, None)) is not None: + await stack.aclose() + finally: + if (call := self.calls.pop(key, None)) is not None: + call.end() + self.running -= 1 def _call_key(context: RunContextWrapper[Any], tool: SdkTool) -> str: @@ -759,9 +796,9 @@ async def _go( raise except BaseException as error: # The model request or tool calls in progress failed with the run. - hooks.ended(error) + await hooks.ended(error) raise - hooks.ended(None) + await hooks.ended(None) # The outputs that no model request stored: those of a tool that ends the turn, say. if outputs := conversation.new_results(result.to_input_list()): await turn.append(*outputs) @@ -1102,7 +1139,8 @@ def _summarizer(model: PerSession[AnyModel | None]) -> Summarize: async def summarize(ctx: SessionContext, request: str) -> str: chosen = _model(await configured(model, ctx)) - result = await Runner.run(Agent(name="summarizer"), request, run_config=RunConfig(model=chosen)) + async with ctx.working(): + result = await Runner.run(Agent(name="summarizer"), request, run_config=RunConfig(model=chosen)) return str(result.final_output) return summarize diff --git a/python/restate/agents/pydantic_ai/_agent.py b/python/restate/agents/pydantic_ai/_agent.py index 50b4b9d..4166ffe 100644 --- a/python/restate/agents/pydantic_ai/_agent.py +++ b/python/restate/agents/pydantic_ai/_agent.py @@ -101,7 +101,8 @@ import json import sys import typing -from collections.abc import Awaitable, Callable, Sequence +from collections.abc import AsyncIterator, Awaitable, Callable, Sequence +from contextlib import AsyncExitStack, asynccontextmanager from typing import Any, Unpack from pydantic import BaseModel, ConfigDict, TypeAdapter, ValidationError @@ -118,7 +119,8 @@ ToolReturnPart, UserPromptPart, ) -from pydantic_ai.models import Model, ModelRequestContext, infer_model +from pydantic_ai.models import Model, ModelRequestContext, StreamedResponse, infer_model +from pydantic_ai.models.wrapper import WrapperModel from pydantic_ai.tools import ToolDefinition from pydantic_ai.toolsets import FunctionToolset from pydantic_ai.usage import UsageLimits @@ -139,6 +141,19 @@ estimate_context_tokens, resolve_context_window, ) +from pydantic_ai_harness.filesystem import ( + CREATE_DIRECTORY_KIND, + EDIT_FILE_KIND, + FILE_INFO_KIND, + FIND_FILES_KIND, + GREP_KIND, + LIST_DIRECTORY_KIND, + LIST_FILES_KIND, + READ_FILE_KIND, + SEARCH_FILES_KIND, + WRITE_FILE_KIND, +) +from pydantic_ai_harness.shell import SHELL_TOOL_NAMES from pydantic_core import to_json, to_jsonable_python import pydantic_ai @@ -347,6 +362,50 @@ def _feedback_of(message: ModelMessage) -> list[str]: return [to_json(p, bytes_mode="base64").decode() for p in message.parts if isinstance(p, RetryFeedbackPart)] +LOCAL_KINDS: frozenset[str] = frozenset( + kind.id + for kind in ( + READ_FILE_KIND, + WRITE_FILE_KIND, + EDIT_FILE_KIND, + LIST_DIRECTORY_KIND, + SEARCH_FILES_KIND, + FIND_FILES_KIND, + CREATE_DIRECTORY_KIND, + FILE_INFO_KIND, + LIST_FILES_KIND, + GREP_KIND, + ) +) +"""The ``tool_kind`` of the harness's built-in local tools that this package keeps alive while they run (the file +system's). The harness's shell tools declare none: they are known by the names the harness gives them +(``SHELL_TOOL_NAMES``). ``code_mode``'s ``run_code`` runs the user's tools inside it, so it is not kept alive.""" + + +def _local(tool_def: ToolDefinition) -> bool: + """Whether ``tool_def`` is a built-in local tool of the harness: its file system's, or its shell's.""" + return tool_def.tool_kind in LOCAL_KINDS or tool_def.name in SHELL_TOOL_NAMES + + +class _Alive(WrapperModel): + """A model whose requests keep the attempt alive (``ctx.working``) while they run: exactly the request, so the + stores and steering around it are outside, and a wait on Restate never is inside.""" + + def __init__(self, wrapped: Model, ctx: SessionContext) -> None: + super().__init__(wrapped) + self.ctx = ctx + + async def request(self, *args: Any, **kwargs: Any) -> ModelResponse: + async with self.ctx.working(): + return await super().request(*args, **kwargs) + + @asynccontextmanager + async def request_stream(self, *args: Any, **kwargs: Any) -> AsyncIterator[StreamedResponse]: + # In scope until the stream ends, as the caller reads it inside the ``async with``. + async with self.ctx.working(), super().request_stream(*args, **kwargs) as stream: + yield stream + + class _Restate(AbstractCapability[Any]): """Make a Pydantic AI run durable in a Restate handler. @@ -363,6 +422,8 @@ def __init__(self, ctx: SessionContext, conversation: Conversation, history: Seq self.conversation = conversation self.model_errors = ErrorSet() """The errors that the run's model requests failed with.""" + self.local: dict[str, AsyncExitStack] = {} + """The ``ctx.working`` scopes of the built-in local tool calls in progress, by call id.""" self.feedback = {feedback for m in history for feedback in _feedback_of(m)} """Pydantic AI's feedback that the conversation has (each ``RetryFeedbackPart``), as JSON.""" @@ -408,18 +469,34 @@ async def after_model_request( await self.conversation.turn.append_reply(response) return response + async def _release(self, call: ToolCallPart) -> None: + if (stack := self.local.pop(call.tool_call_id, None)) is not None: + await stack.aclose() + + async def release(self) -> None: + """Close the scopes still open, as the run ends.""" + stacks = list(self.local.values()) + self.local.clear() + for stack in stacks: + await stack.aclose() + async def before_tool_execute( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any ) -> Any: # The stores of this package write once per tool call (``_stores.py``). calling(call.tool_call_id) await tool_call_started(self.ctx) + if _local(tool_def): + # A built-in local tool (the file system's, the shell's): kept alive while it runs, from here to its end. + stack = self.local[call.tool_call_id] = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) return args async def after_tool_execute( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any, result: Any ) -> Any: calling(None) + await self._release(call) return result async def handle_deferred_tool_calls( @@ -453,6 +530,7 @@ async def on_tool_execute_error( self, ctx: RunContext[Any], *, call: ToolCallPart, tool_def: ToolDefinition, args: Any, error: Exception ) -> Any: calling(None) + await self._release(call) stop = stop_signal_of(self.ctx) if stop is not None and stop.reason is not None: raise stop.reason from error @@ -794,7 +872,7 @@ async def run(conversation: Conversation) -> str: with tools_in(ctx): result = await agent.run( message_history=history, - model=model, + model=_Alive(model, ctx), toolsets=toolsets, capabilities=[restate, *chosen, *snapshots, steering], output_type=spec, @@ -824,6 +902,9 @@ async def run(conversation: Conversation) -> str: # Not a model request's: no refusal, whatever its name (an instructions function's, say). from_tool(ctx, error) raise + finally: + # Whatever scope a tool's end hook missed (a cancelled run, say). + await restate.release() # The messages that no model request stored: an output tool's return, say. await restate.store(result.all_messages()) if isinstance(result.output, DeferredToolRequests): @@ -1128,7 +1209,7 @@ async def compact_history(ctx: SessionContext, history: list[Any]) -> list[Any] compacted = await compact_now( compaction(context_window, target=BETWEEN), messages, - model=_model(named), + model=_Alive(_model(named), ctx), conversation_id=conversation_id_of(ctx), ) return None if compacted == messages else _snapshot_of(compacted) diff --git a/tests/agents_openai.py b/tests/agents_openai.py index 837377d..a3eda11 100644 --- a/tests/agents_openai.py +++ b/tests/agents_openai.py @@ -28,6 +28,7 @@ pytest.importorskip("agents") from agents import ( # noqa: E402 + LocalShellTool, Agent, GuardrailFunctionOutput, Handoff, @@ -1620,3 +1621,138 @@ async def outputs(ctx: sessions.SessionContext, _: Any = None) -> list[str]: "found two", "found three", ] + + +# --- In process: the attempt is kept alive for the model's requests and a local tool's runs only ---- + + +class Seen: + """How many ``ctx.working`` scopes were open at each moment a test's own code ran.""" + + def __init__(self) -> None: + self.host: Any = None + self.at: dict[str, int] = {} + + def mark(self, what: str) -> None: + self.at[what] = self.host.open_scopes + + +def watching_invoker(obj: sessions.SessionObject, seen: Seen) -> Callable[..., Awaitable[Any]]: + """Invoke ``obj``'s ``ask``, with the host kept in ``seen``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + backend = Backend() + + async def ask(text: str) -> Any: + seen.host = Host(backend, service="asker", answers=PUBLISHED) + try: + return await invoke(obj, "ask", seen.host, request(), text) + finally: + seen.at["after"] = seen.host.open_scopes + + return ask + + +async def test_a_model_request_is_kept_alive_and_a_tool_of_your_own_is_not(): + seen = Seen() + + @tool + async def plain(city: str) -> str: + """Plain.""" + seen.mark("plain") + return "plain done" + + @tool(keepalive=True) + async def long(city: str) -> str: + """Long.""" + seen.mark("long") + return "long done" + + @function_tool(name_override="mine") + def mine(city: str) -> str: + """The user's own function tool.""" + seen.mark("mine") + return "mine done" + + def script(system: str | None, items: list[Item]) -> list[Output]: + seen.mark("model") + word = _prompts(items)[-1].split(" ")[0] + if items[-1].get("type") == "function_call_output": + return [said("done")] + return [calls(f"call-{uuid.uuid4().hex}", word, '{"city": "Rome"}')] + + ask = watching_invoker(asker(model=Scripted(script), tools=[plain, long, mine]), seen) + for word in ("plain", "long", "mine"): + assert await ask(f"{word} now") == "done" + assert seen.at["model"] > 0 + assert seen.at["plain"] == 0 and seen.at["mine"] == 0 + assert seen.at["long"] == 1 + assert seen.at["after"] == 0 + + +async def test_the_scope_of_a_request_ends_with_the_request_not_the_steering_and_stores_around_it(): + seen = Seen() + during: list[int] = [] + + async def count_stores(data: CallModelData[Any]) -> ModelInputData: + # The input filter runs before the request: no scope is open yet. + during.append(seen.host.open_scopes) + return data.model_data + + def script(system: str | None, items: list[Item]) -> list[Output]: + seen.mark("model") + return [said("ok")] + + mine = Agent(name="mine", model=Scripted(script)) + ask = watching_invoker(asker(agent=mine, run_config=RunConfig(call_model_input_filter=count_stores)), seen) + assert await ask("hi there") == "ok" + assert during == [0] and seen.at["model"] == 1 and seen.at["after"] == 0 + + +async def test_a_failed_request_and_a_failed_tool_release_their_scopes(): + seen = Seen() + ask = watching_invoker(asker(model=Scripted(reply)), seen) + with pytest.raises(Exception): + await ask("refuse now") + assert seen.at["after"] == 0 + + @tool(keepalive=True) + async def broken(city: str) -> str: + """Fails.""" + raise restate.TerminalError("no") + + def call_it(system: str | None, items: list[Item]) -> list[Output]: + return [calls("c1", "broken", '{"city": "Rome"}')] + + ask = watching_invoker(asker(model=Scripted(call_it), tools=[broken]), seen) + with pytest.raises(restate.TerminalError): + await ask("go") + assert seen.at["after"] == 0 + + +async def test_a_built_in_local_tool_is_kept_alive_while_it_runs(): + from openai.types.responses.response_output_item import LocalShellCall # pylint: disable=import-outside-toplevel + from openai.types.responses.response_output_item import ( # pylint: disable=import-outside-toplevel + LocalShellCallAction, + ) + + seen = Seen() + + def execute(request: Any) -> str: + seen.mark("shell") + return "ran" + + def script(system: str | None, items: list[Item]) -> list[Output]: + if items[-1].get("type") in ("function_call_output", "local_shell_call_output"): + return [said("done")] + action = LocalShellCallAction(type="exec", command=["ls"], env={}) + call = LocalShellCall(id="ls1", action=action, call_id="c1", status="completed", type="local_shell_call") + return [typing.cast(Any, call)] + + mine = Agent(name="shell", model=Scripted(script), tools=[LocalShellTool(executor=execute)]) + ask = watching_invoker(asker(agent=mine), seen) + assert await ask("run it") == "done" + assert seen.at["shell"] == 1 and seen.at["after"] == 0 diff --git a/tests/agents_pydantic_ai.py b/tests/agents_pydantic_ai.py index 45f8d0b..4cf7b76 100644 --- a/tests/agents_pydantic_ai.py +++ b/tests/agents_pydantic_ai.py @@ -27,6 +27,7 @@ pytest.importorskip("pydantic_ai") from pydantic_ai import Agent, ModelRetry, RunContext # noqa: E402 +from pydantic_ai import Tool as PydanticTool # noqa: E402 from pydantic_ai.capabilities import AbstractCapability # noqa: E402 from pydantic_ai.exceptions import UnexpectedModelBehavior # noqa: E402 from pydantic_ai.messages import ( # noqa: E402 @@ -1341,3 +1342,146 @@ def lookup(about: str) -> str: if isinstance(part, ToolReturnPart) ] assert returned == ["found one", "found two", "found three"] + + +# --- In process: the attempt is kept alive for the model's requests and a local tool's runs only ---- + + +class Seen: + def __init__(self) -> None: + self.host: Any = None + self.at: dict[str, int] = {} + + def mark(self, what: str) -> None: + self.at[what] = self.host.open_scopes + + +async def run_watched(seen: Seen, ask: sessions.SessionObject, text: str, **kwargs: Any) -> Any: + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + seen.host = Host(Backend(), service="asker", **kwargs) + try: + return await invoke(ask, "ask", seen.host, request(), text) + finally: + seen.at["after"] = seen.host.open_scopes + + +async def test_a_model_request_is_kept_alive_a_tool_of_your_own_is_not_and_a_local_tool_is(): + from pydantic_ai_harness.filesystem import READ_FILE_KIND # pylint: disable=import-outside-toplevel + + seen = Seen() + + @tool + async def plain(city: str) -> str: + """Plain.""" + seen.mark("plain") + return "plain" + + @tool(keepalive=True) + async def long(city: str) -> str: + """Long.""" + seen.mark("long") + return "long" + + def local(path: str) -> str: + """Stands for a built-in local tool: its kind says so.""" + seen.mark("local") + return "read" + + def own(path: str) -> str: + """A plain function tool of the user's.""" + seen.mark("own") + return "own" + + def shell(path: str) -> str: + """Stands for the harness's shell: its name says so.""" + seen.mark("run_command") + return "ran" + + def script(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + seen.mark("model") + if isinstance(messages[-1], ModelRequest) and any(isinstance(p, ToolReturnPart) for p in messages[-1].parts): + return ModelResponse(parts=[TextPart("done")]) + word = next(str(p.content).split(" ")[0] for p in messages[-1].parts if isinstance(p, UserPromptPart)) + return ModelResponse( + parts=[ToolCallPart(word, {"city": "Rome"} if word in ("plain", "long") else {"path": "x"})] + ) + + mine = Agent( + FunctionModel(script), + tools=[ + PydanticTool(local, tool_kind=READ_FILE_KIND, name="local"), + PydanticTool(own, name="own"), + PydanticTool(shell, name="run_command"), + ], + toolsets=[pydantic_ai.toolset([plain, long])], + ) + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, agent=mine) + + for word in ("plain", "long", "local", "own", "run_command"): + assert await run_watched(seen, obj, f"{word} now") == "done" + assert seen.at["after"] == 0 + assert seen.at["model"] == 1 + assert (seen.at["plain"], seen.at["long"], seen.at["local"], seen.at["own"], seen.at["run_command"]) == ( + 0, + 1, + 1, + 0, + 1, + ) + + +async def test_a_failed_request_releases_its_scope(): + seen = Seen() + + def refuse(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + seen.mark("model") + raise AuthenticationError("invalid x-api-key") + + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, model=FunctionModel(refuse)) + + with pytest.raises(Exception): + await run_watched(seen, obj, "hi") + assert seen.at["model"] == 1 and seen.at["after"] == 0 + + +async def test_an_approval_is_waited_for_outside_any_scope(monkeypatch: pytest.MonkeyPatch): + from restate.agents.pydantic_ai import _agent # pylint: disable=import-outside-toplevel + + seen = Seen() + + async def approved(ctx: Any, call_id: str, signal: str, input: Any = None, *, approval: bool = False) -> Any: + seen.mark("approval") + return {"approved": True} + + monkeypatch.setattr(_agent, "wait_for", approved) + + def needs(city: str) -> str: + """Needs an approval.""" + return "ran" + + def script(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + if any(isinstance(p, ToolReturnPart) for m in messages if isinstance(m, ModelRequest) for p in m.parts): + return ModelResponse(parts=[TextPart("done")]) + return ModelResponse(parts=[ToolCallPart("needs", {"city": "Rome"})]) + + mine = Agent(FunctionModel(script), tools=[PydanticTool(needs, requires_approval=True)]) + obj = sessions.SessionObject("asker") + + @obj.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await pydantic_ai.prompt(ctx, text, agent=mine) + + assert await run_watched(seen, obj, "go") == "done" + assert seen.at["approval"] == 0 and seen.at["after"] == 0 diff --git a/tests/agents_pydantic_ai_snapshots.py b/tests/agents_pydantic_ai_snapshots.py index 0f84e23..87a56a7 100644 --- a/tests/agents_pydantic_ai_snapshots.py +++ b/tests/agents_pydantic_ai_snapshots.py @@ -647,3 +647,29 @@ def planner(messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: stored = [p for m in steps if isinstance(m, ModelRequest) for p in m.parts] assert [type(p).__name__ for p in stored] == ["RetryFeedbackPart", "UserPromptPart", "RetryFeedbackPart"] assert [text for role, text in await chat.messages() if role == "user"] == ["plan one", "use Celsius", "plan two"] + + +async def test_the_summarizers_request_and_the_models_are_kept_alive(): + hosts: list[Host] = [] + scopes: dict[str, list[int]] = {"summary": [], "model": []} + + class Watched(Script): + def __call__(self, messages: list[ModelMessage], info: AgentInfo) -> ModelResponse: + before = self.summaries + response = super().__call__(messages, info) + scopes["summary" if self.summaries > before else "model"].append(hosts[-1].open_scopes) + return response + + script = Watched() + chat = agent(script, "pai_alive") + make = chat.host + + def watched(view: bool = False) -> Host: + hosts.append(make(view)) + return hosts[-1] + + chat.host = watched # type: ignore[method-assign] + await until_snapshot(chat, compact=False) + assert script.summaries == 1 + assert scopes["summary"] == [1] and scopes["model"] and all(n == 1 for n in scopes["model"]) + assert all(host.open_scopes == 0 for host in hosts) From 27a6dbc6e3c58601057f6891419df8864b0147a9 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:19:58 +0000 Subject: [PATCH 3/6] Agents: ADK's and LangGraph's model requests run kept alive, and ADK's built-in local tools - ADK: the plugin keeps each model request alive from the end of its `before_model_callback` (after the steering) to the whole response, an error, or the event that ends the step if a callback answered instead. The shell, the skills' scripts and the environment's tools (known by their package, as ADK keeps their classes private) are kept alive per call. The summarizer's request too. A run closes what is left as it ends. Not covered: an agent's code executor, which runs as a processor of the flow. - LangGraph: the built-in agent's request and its summarizer's run in `ctx.working()`. A graph of your own gets a callback handler, inherited by its nodes' runs, that keeps each chat model request alive from its start (awaited before it is sent) to its end or error, and releases what is left before any wait for an interrupt. Known gap: the shell of LangChain's `ShellToolMiddleware` is a plain tool with nothing that tells it apart, so it is not kept alive. --- python/restate/agents/adk/_agent.py | 120 ++++++++++- python/restate/agents/langgraph/_agent.py | 7 +- python/restate/agents/langgraph/_graph.py | 63 +++++- tests/agents_adk.py | 249 +++++++++++++++++++++- tests/agents_langgraph.py | 154 +++++++++++++ tests/agents_langgraph_graph.py | 163 ++++++++++++++ 6 files changed, 746 insertions(+), 10 deletions(-) diff --git a/python/restate/agents/adk/_agent.py b/python/restate/agents/adk/_agent.py index 8aad3f6..c79dc2a 100644 --- a/python/restate/agents/adk/_agent.py +++ b/python/restate/agents/adk/_agent.py @@ -47,6 +47,10 @@ with ``include_contents='none'`` starts each request at the latest message of the user: the requests after the one that takes a steered message start at it, without the prompt and the tools' results before it. +- The attempt is kept alive (``ctx.working``) while a model request runs (not the steering and commits around it) + and while one of ADK's built-in local tools does (the shell and the environment's tools: ``_LOCAL_TOOLS``), + not while one of yours does: a ``FunctionTool``, or a ``tool`` of ours unless it has ``keepalive``. A code + executor of an agent (``code_executor``) is not covered. - A run of more model calls than its ``RunConfig``'s ``max_llm_calls`` (500 by default) fails its prompt for good: a retry would carry on with a fresh count. - An agent or ``App`` of your own runs as you configured it, with the @@ -75,6 +79,7 @@ import time import typing from collections.abc import Awaitable, Callable, Sequence +from contextlib import AsyncExitStack from typing import Any, Unpack from google.adk.agents import BaseAgent, LlmAgent, RunConfig @@ -90,6 +95,8 @@ from google.adk.sessions import BaseSessionService, Session from google.adk.sessions.base_session_service import GetSessionConfig, ListSessionsResponse from google.adk.tools.base_tool import BaseTool +from google.adk.tools.bash_tool import ExecuteBashTool +from google.adk.tools.skill_toolset import RunSkillScriptTool from google.adk.tools.tool_context import ToolContext from google.adk.workflow import BaseNode from google.genai import types @@ -265,14 +272,59 @@ async def add_stored(self, session: Session, event: Event) -> None: await super().append_event(session, event) +_LOCAL_TOOLS = (ExecuteBashTool, RunSkillScriptTool) +"""ADK's built-in tools that work on the local machine (a shell, scripts), which run kept alive; and the +environment's (``EnvironmentToolset``: execute, read, write and edit a file), whose classes ADK keeps in private +modules, so they are known by their package (``_ENVIRONMENT_TOOLS``). + +Only these: not ours (``_AdkTool``), a user's own function (``FunctionTool``, which includes +``LongRunningFunctionTool``), a sub-agent (``AgentTool``, whose model requests are kept alive by the plugin), nor +the tools that call a service (MCP, OpenAPI, search, ...). Code executors (``LlmAgent.code_executor``) run as a +processor of the flow, not as a tool.""" + +_ENVIRONMENT_TOOLS = "google.adk.tools.environment." +"""The package of the environment's tools (``EnvironmentToolset``).""" + + +def _local(tool: BaseTool) -> bool: + """Whether ``tool`` is one of ADK's built-in local tools (see ``_LOCAL_TOOLS``).""" + return isinstance(tool, _LOCAL_TOOLS) or type(tool).__module__.startswith(_ENVIRONMENT_TOOLS) + + class _Restate(BasePlugin): - """Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, and give the model the - messages steered to the turn.""" + """Settle the earlier calls as each tool call starts, mark a tool's error as the tool's, give the model the + messages steered to the turn, and keep the attempt alive while a model request or a built-in local tool runs. + + A scope (``ctx.working``) is opened by one hook and closed by another, so each is kept per request or per call: + the run closes what is left when it ends (``release_all``).""" def __init__(self, ctx: SessionContext, sessions: _Sessions) -> None: super().__init__(name="restate") self.ctx = ctx self.sessions = sessions + self.requests: dict[tuple[str, str, str], AsyncExitStack] = {} + """The scopes of the model requests in progress, by invocation, branch and agent: parallel agents each + have one.""" + self.local: dict[str, AsyncExitStack] = {} + """The scopes of the built-in local tool calls in progress, by call id: tools run in parallel.""" + + async def _keep(self) -> AsyncExitStack: + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + return stack + + @staticmethod + async def _close(stack: AsyncExitStack | None) -> None: + if stack is not None: + await stack.aclose() + + async def release_all(self) -> None: + """Close the scopes still open, as the run ends: whatever way, success, error or a stopped turn.""" + stacks = [*self.requests.values(), *self.local.values()] + self.requests.clear() + self.local.clear() + for stack in stacks: + await self._close(stack) async def before_model_callback( self, *, callback_context: CallbackContext, llm_request: LlmRequest @@ -287,16 +339,61 @@ async def before_model_callback( await self.sessions.add_stored(callback_context.session, event) if event.content is not None: llm_request.contents.append(event.content.model_copy(deep=True)) + # Kept alive from here, once the steering is done: the request is what follows (ADK resolves the model, runs + # the agent's own callbacks, and calls it). Closed by the response (``after_model_callback``), an error + # (``on_model_error_callback``), or the event that ends the step if a callback answered instead. + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + self.requests[key] = await self._keep() + return None + + async def after_model_callback( + self, *, callback_context: CallbackContext, llm_response: LlmResponse + ) -> LlmResponse | None: + # A streamed reply comes as partial responses, then its whole: the request is on until the whole. + if not llm_response.partial: + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + return None + + async def on_model_error_callback( + self, *, callback_context: CallbackContext, llm_request: LlmRequest, error: Exception + ) -> LlmResponse | None: + # Also called for a request that is cancelled (``error`` is the ``CancelledError``). + key = _request_key(callback_context.invocation_id, callback_context.branch, callback_context.agent_name) + await self._close(self.requests.pop(key, None)) + return None + + async def on_event_callback(self, *, invocation_context: Any, event: Event) -> Event | None: + # Backstop for what the callbacks miss: a response that a callback of the agent gave in place of the + # model's (no after or error callback runs then), or a tool call that one answered. An event of the + # agent that is whole ends its request, and a response to a call ends the call's scope. + if not event.partial: + await self._close(self.requests.pop(_request_key(event.invocation_id, event.branch, event.author), None)) + for response in event.get_function_responses(): + await self._close(self.local.pop(response.id or "", None)) return None async def before_tool_callback( self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext ) -> None: await tool_call_started(self.ctx) + if _local(tool): + call_id = tool_context.function_call_id or "" + await self._close(self.local.pop(call_id, None)) + self.local[call_id] = await self._keep() + + async def after_tool_callback( + self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, result: dict[str, Any] + ) -> dict[str, Any] | None: + await self._close(self.local.pop(tool_context.function_call_id or "", None)) + return None async def on_tool_error_callback( self, *, tool: BaseTool, tool_args: dict[str, Any], tool_context: ToolContext, error: Exception ) -> dict[str, Any] | None: + # Also called for a call that is cancelled. + await self._close(self.local.pop(tool_context.function_call_id or "", None)) if type(tool) is BaseTool: # A tool the agent does not have (ADK stands in a bare BaseTool for it): the model's to fix. Failing # the attempt would not help, as the retry resumes the same call. @@ -306,6 +403,10 @@ async def on_tool_error_callback( return None +def _request_key(invocation: str, branch: str | None, agent: str) -> tuple[str, str, str]: + return (invocation, branch or "", agent) + + # --- Tools ---------------------------------------------------------------------------------------- @@ -467,7 +568,8 @@ async def run(conversation: Conversation) -> str: sessions = _Sessions(conversation) # This package's plugin, after the app's own: a plugin of the app that answers in the model's place leaves # the messages steered to the turn to a request that goes out. - plugins = [*app.plugins, _Restate(ctx, sessions)] + restate = _Restate(ctx, sessions) + plugins = [*app.plugins, restate] runner = Runner(app=app.model_copy(update={"plugins": plugins}), session_service=sessions) answer = "" error: Event | None = None @@ -488,6 +590,10 @@ async def run(conversation: Conversation) -> str: except LlmCallsLimitExceededError as limit: # A retry would carry on with a fresh count, and so would never end an agent that keeps calling tools. raise TerminalError(f"the run reached its limit of model calls: {limit}") from limit + finally: + # The scopes the plugin's hooks left open (a callback of the agent answered in place of the model or a + # tool, a failed or stopped run): the attempt is not kept alive past the run. + await restate.release_all() if error is not None: # The model would not answer (a safety block, say), as it would not again: the prompt fails. raise TerminalError(f"the model returned {error.error_code}: {error.error_message or ''}") @@ -768,9 +874,11 @@ async def summarize(ctx: SessionContext, request: str) -> str: llm = LlmAgent(name="summarizer", model=_model(await configured(model, ctx))).canonical_model asked = LlmRequest(model=llm.model, contents=[types.Content(role="user", parts=[types.Part(text=request)])]) texts: list[str] = [] - async for response in llm.generate_content_async(asked): - if response.content is not None: - texts.extend(part.text for part in response.content.parts or [] if part.text) + # Kept alive for the request alone. + async with ctx.working(): + async for response in llm.generate_content_async(asked): + if response.content is not None: + texts.extend(part.text for part in response.content.parts or [] if part.text) return "".join(texts) return summarize diff --git a/python/restate/agents/langgraph/_agent.py b/python/restate/agents/langgraph/_agent.py index 8076f31..d52434f 100644 --- a/python/restate/agents/langgraph/_agent.py +++ b/python/restate/agents/langgraph/_agent.py @@ -351,7 +351,9 @@ async def call_model(state: State) -> dict[str, Any]: # request has them after the results, and the state keeps them for the rest of the run, which goes on. steered = typing.cast(list[BaseMessage], await take_steers(ctx, conversation)) messages = _latest([*state["messages"], *steered], window) - reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages) + # Kept alive for the request alone: not the steering before it, nor the commit after. + async with ctx.working(): + reply = await bound.ainvoke([SystemMessage(system), *messages] if system else messages) await conversation.turn.append_reply(reply) return {"messages": [*steered, reply]} @@ -651,7 +653,8 @@ def _summarizer(model: PerSession[ChatModel | None]) -> Summarize: async def summarize(ctx: SessionContext, request: str) -> str: chosen = await configured(model, ctx) chat_model = chosen if isinstance(chosen, BaseChatModel) else _chat_model(model_of(chosen)) - return _text(await chat_model.ainvoke([HumanMessage(request)])) + async with ctx.working(): + return _text(await chat_model.ainvoke([HumanMessage(request)])) return summarize diff --git a/python/restate/agents/langgraph/_graph.py b/python/restate/agents/langgraph/_graph.py index 232b8e1..6d8e42a 100644 --- a/python/restate/agents/langgraph/_graph.py +++ b/python/restate/agents/langgraph/_graph.py @@ -107,8 +107,11 @@ def forget(state: MessagesState) -> dict[str, Any]: import json import typing from collections.abc import Sequence +from contextlib import AsyncExitStack from typing import Any +from uuid import UUID +from langchain_core.callbacks import AsyncCallbackHandler from langchain_core.messages import AIMessage, BaseMessage, HumanMessage, ToolMessage from langchain_core.runnables import RunnableConfig from langgraph.errors import InvalidUpdateError @@ -366,6 +369,55 @@ async def _stored(app: Any) -> StateSnapshot: return typing.cast(StateSnapshot, await app.aget_state(latest.config)) +class _KeepAlive(AsyncCallbackHandler): + """Keep the attempt alive (``ctx.working``) while a model request of the graph's runs, and only then. + + LangChain awaits a handler's ``on_chat_model_start`` (or ``on_llm_start``) before it sends the request, and + ``on_llm_end`` or ``on_llm_error`` (also for a cancelled request) once it is over, so the scope is the request + alone. Each is kept by the request's run id: a graph's nodes run models in parallel. It is a handler that its + runs inherit, so that the models a node, a tool or a subgraph calls are in it too. + + A scope the callbacks leave open (a request that never ended) is closed as the run's stream ends + (``release_all``). + """ + + run_inline = True + """Awaited before the request, in order, rather than gathered with the other handlers.""" + + def __init__(self, ctx: SessionContext) -> None: + self.ctx = ctx + self.requests: dict[UUID, AsyncExitStack] = {} + + async def _open(self, run_id: UUID) -> None: + await self._close(run_id) + stack = AsyncExitStack() + await stack.enter_async_context(self.ctx.working()) + self.requests[run_id] = stack + + async def _close(self, run_id: UUID) -> None: + stack = self.requests.pop(run_id, None) + if stack is not None: + await stack.aclose() + + async def release_all(self) -> None: + stacks = list(self.requests.values()) + self.requests.clear() + for stack in stacks: + await stack.aclose() + + async def on_chat_model_start(self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any) -> None: + await self._open(run_id) + + async def on_llm_start(self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any) -> None: + await self._open(run_id) + + async def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: + await self._close(run_id) + + async def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: + await self._close(run_id) + + def run_of(ctx: SessionContext, graph: AnyGraph, context: Any = None) -> Run: """Return the run of a prompt: the graph's, on the key's thread, from the prompt to its answer. @@ -394,7 +446,13 @@ async def _run(conversation: Conversation) -> str: before = str(metadata.get(_STRUCTURED_BEFORE, "")) if asked else await _structured_version(app) # Each checkpoint of the run names its prompt, so that a retried attempt finds the run it cut off, and the # version its structured response had before it. - config: RunnableConfig = {**THREAD, "metadata": {_PROMPT: prompt_id, _STRUCTURED_BEFORE: before}} + alive = _KeepAlive(ctx) + config: RunnableConfig = { + **THREAD, + "metadata": {_PROMPT: prompt_id, _STRUCTURED_BEFORE: before}, + # Inherited by the nodes' own runs: the models they call keep the attempt alive while they request. + "callbacks": [alive], + } if asked and not snapshot.next: # The run had finished, but its answer was not stored: that is the answer, rather than run again. await _copied(ctx, turn, known, messages, prompt_id) @@ -425,6 +483,9 @@ async def _run(conversation: Conversation) -> str: # No refusal, whatever its name: a graph's models, tools and nodes are all its own. from_tool(ctx, error) raise + finally: + # Before any wait for an interrupt: none is in a scope. Closes what a request left open. + await alive.release_all() # The run stops at its interrupts: it goes on with the signals that answer them. paused = await _stored(app) resumes = await _resumed(ctx, paused) diff --git a/tests/agents_adk.py b/tests/agents_adk.py index a318d56..9b107f7 100644 --- a/tests/agents_adk.py +++ b/tests/agents_adk.py @@ -26,6 +26,7 @@ pytest.importorskip("google.adk") from google.adk.agents import LlmAgent, RunConfig, SequentialAgent # noqa: E402 +from google.adk.agents.run_config import StreamingMode # noqa: E402 # pyright: ignore[reportPrivateImportUsage] from google.adk.models import BaseLlm # noqa: E402 from google.adk.models.llm_request import LlmRequest # noqa: E402 from google.adk.models.llm_response import LlmResponse # noqa: E402 @@ -34,7 +35,7 @@ import restate # noqa: E402 from support import RESTATE_IMAGE, Gate # noqa: E402 # pylint: disable=import-error from restate import HarnessEnvironment, sessions # noqa: E402 -from restate.agents import adk, tool # noqa: E402 +from restate.agents import adk, call, tool # noqa: E402 from restate.agents._api import Messages # noqa: E402 from restate.agents._signals import with_companion # noqa: E402 from restate.agents.internal import CANCELLED # noqa: E402 @@ -1120,3 +1121,249 @@ async def attempt() -> Any: await attempt() # The retry resumed the prompt's invocation, past the message, and answered it. assert await attempt() == "echo: in Celsius" + + +# --- Kept alive ------------------------------------------------------------------------------------ + +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseLlm): + """A model that calls the tool a prompt names (its first word), then answers, noting the scopes it sees. A prompt + that starts with ``stream`` also streams its answer; ``stuck`` waits for ``stuck`` to be set, which never is.""" + + model: str = "probing" + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + mark("model") + prompt = _prompts(list(llm_request.contents))[-1] + answered = any(part.function_response for part in llm_request.contents[-1].parts or []) + word = prompt.split(" ")[0] + if word == "stuck": + await stuck.wait() + if not answered and word in ("plain", "kept", "local", "function", "shipping_status", "wait"): + args = {"key": "k", "input": "o"} if word == "shipping_status" else {} + call = types.FunctionCall(id=f"call-{word}", name=word, args=args) + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(function_call=call)])) + return + if stream: + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(text="do")]), partial=True) + mark("partial") + yield LlmResponse(content=types.Content(role="model", parts=[types.Part(text="done")])) + + +@tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +async def function() -> str: + """A function of the user's own, which ADK runs as its function tool.""" + mark("function") + return "ok" + + +async def hang() -> str: + mark("hang") + await stuck.wait() + return "never" + + +def local() -> Any: + """A built-in local tool of ADK (a shell: its own asks for a confirmation), as a probe.""" + from google.adk.tools.bash_tool import ExecuteBashTool # pylint: disable=import-outside-toplevel + + class Local(ExecuteBashTool): + def __init__(self, name: str = "local") -> None: + super().__init__() + self.name = name + + async def run_async(self, *, args: dict[str, Any], tool_context: Any) -> Any: + mark(self.name) + if self.name == "wait": + await stuck.wait() + return {"result": "ok"} + + return Local + + +shipping = restate.VirtualObject("shipping") + + +@shipping.handler() +async def status(ctx: restate.ObjectContext, order: str) -> str: + return "shipped" + + +def probing(*given: Any, model: Any = None, **options: Any) -> tuple[Callable[..., Awaitable[Any]], Any]: + """Return how to ask an agent with ``given`` tools, in process, and its host (the run's, once it has begun).""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + mark("remote") + return super().generic_call(*args, **kwargs) + + asker = sessions.SessionObject("asker") + llm = model or Probing() + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + if "agent" in options: + return await adk.prompt(ctx, text, **options) + return await adk.prompt(ctx, text, model=llm, tools=[*given], **options) + + backend = Backend() + + async def asked(text: str) -> Any: + scopes.clear() + now[:] = [Watching(backend, service="asker", answers={"publish": REVISION, "status": b'"shipped"'})] + return await invoke(asker, "ask", now[0], request(), text) + + return asked, now + + +async def test_a_model_request_is_kept_alive_and_the_work_around_it_is_not(): + asked, _ = probing() + assert await asked("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + # Streamed: on for the partial replies too, and released with the whole. + mine = LlmAgent(name="mine", model=Probing()) + asked, _ = probing(agent=mine, run_config=RunConfig(streaming_mode=StreamingMode.SSE)) + assert await asked("hi") == "done" + assert scopes == [("model", 1), ("partial", 1)] + assert now[0].open_scopes == 0 + + +async def test_the_summary_of_earlier_turns_is_requested_kept_alive(): + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.agents.adk import _agent # pylint: disable=import-outside-toplevel + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + summarizer = sessions.SessionObject("summarizer") + + @summarizer.handler + async def summarize(ctx: sessions.SessionContext, text: str) -> str: + return await _agent._summarizer(Probing())(ctx, text) # pyright: ignore[reportPrivateUsage] + + scopes.clear() + now[:] = [Host(Backend(), service="summarizer")] + assert await invoke(summarizer, "summarize", now[0], request(), "hi there") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_and_a_users_function_is_not(): + asked, _ = probing(plain, kept, function) + assert await asked("plain x") == "done" + assert scopes == [("model", 1), ("plain", 0), ("model", 1)] + assert await asked("kept x") == "done" + assert scopes == [("model", 1), ("kept", 1), ("model", 1)] + assert await asked("function x") == "done" + assert scopes == [("model", 1), ("function", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_built_in_local_tool_is_kept_alive_while_it_runs_and_a_restate_call_is_not(): + asked, _ = probing(local()("local")) + assert await asked("local x") == "done" + assert scopes == [("model", 1), ("local", 1), ("model", 1)] + assert now[0].open_scopes == 0 + # A call on Restate, which a tool of ours makes: no scope open. + asked, _ = probing(call(status, description="The status of a shipment.")) + assert await asked("shipping_status x") == "done" + assert scopes == [("model", 1), ("remote", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_turn_leaves_no_scope_open_whether_it_was_in_the_model_or_a_built_in_tool(): + for prompt, tools in (("stuck x", []), ("wait x", [local()("wait")])): + stuck.clear() + asked, _ = probing(*tools) + scopes.clear() + task = asyncio.ensure_future(asked(prompt)) + while not scopes or (prompt.startswith("wait") and ("wait", 1) not in scopes): + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + + +async def test_a_failed_model_request_leaves_no_scope_open(): + class Failing(BaseLlm): + model: str = "failing" + + async def generate_content_async( + self, llm_request: LlmRequest, stream: bool = False + ) -> AsyncGenerator[LlmResponse, None]: + mark("model") + raise AuthenticationError("invalid x-api-key") + yield # pragma: no cover + + asked, _ = probing(model=Failing()) + with pytest.raises(Exception): + await asked("hi") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_response_a_callback_gives_in_place_of_the_models_leaves_no_scope_open(): + def answering(callback_context: Any, llm_request: Any) -> LlmResponse | None: + # The agent's own callback calls the tool in the model's place, once: no after-model or error callback runs. + if any(part.function_response for content in llm_request.contents for part in content.parts or []): + return None + call_ = types.FunctionCall(id="call-plain", name="plain", args={}) + return LlmResponse(content=types.Content(role="model", parts=[types.Part(function_call=call_)])) + + mine = LlmAgent(name="mine", model=Probing(), tools=adk.tools([plain]), before_model_callback=answering) + asked, _ = probing(agent=mine) + assert await asked("hi") == "done" + # The tool ran after the request the callback answered, with its scope closed; then the model was asked. + assert scopes == [("plain", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +def test_the_environments_tools_and_the_shell_are_built_in_local_tools_and_a_function_is_not(): + # The environment's tools are known by their package: ADK keeps their classes in private modules. + from google.adk.tools.function_tool import FunctionTool # pylint: disable=import-outside-toplevel + from google.adk.tools.bash_tool import ExecuteBashTool # pylint: disable=import-outside-toplevel + from google.adk.tools.environment import EnvironmentToolset # pylint: disable=import-outside-toplevel + + from restate.agents.adk._agent import _local # pylint: disable=import-outside-toplevel + + class Environment: + async def initialize(self) -> None: + pass + + toolset = EnvironmentToolset(environment=typing.cast(Any, Environment())) + tools = asyncio.run(toolset.get_tools()) + assert tools and all(_local(t) for t in tools), tools + assert _local(ExecuteBashTool()) + assert not _local(FunctionTool(lambda: None)) diff --git a/tests/agents_langgraph.py b/tests/agents_langgraph.py index 4cf3654..0a00595 100644 --- a/tests/agents_langgraph.py +++ b/tests/agents_langgraph.py @@ -668,3 +668,157 @@ async def test_turns_that_outgrow_the_budget_are_summarized_once_answered(harnes assert len(summaries) == 1 and "User: hi one" in summaries[0] and "hi four" not in summaries[0] latest = summarized_prompts[-1] assert latest[0].startswith("A summary of our conversation so far:") and latest[1:] == ["hi four", "hi five"] + + +# --- Kept alive ------------------------------------------------------------------------------------- + +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseChatModel): + """A model that calls the tool a prompt names (its first word), then answers, noting the scopes it sees. A prompt + that starts with ``stuck`` waits for ``stuck`` to be set, which never is.""" + + @property + def _llm_type(self) -> str: + return "probing" + + def bind_tools(self, tools: Any, **kwargs: Any) -> Any: # pyright: ignore[reportIncompatibleMethodOverride] + return self + + def _generate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + raise AssertionError("the requests are async") + + async def _agenerate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + mark("model") + prompt = next(m for m in reversed(messages) if isinstance(m, HumanMessage)) + word = str(prompt.content).split(" ")[0] + if word == "stuck": + await stuck.wait() + if word == "fail": + raise AuthenticationError("invalid x-api-key") + if word in ("plain", "kept", "chain", "lg_shipping_status") and not isinstance(messages[-1], ToolMessage): + args_ = {"key": "k", "input": "o"} if word == "lg_shipping_status" else {} + reply = AIMessage("", tool_calls=[{"id": f"call-{word}", "name": word, "args": args_}]) + else: + reply = AIMessage("done") + return ChatResult(generations=[ChatGeneration(message=reply)]) + + +@tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +@langchain_tool +async def chain() -> str: + """A LangChain tool of the user's own.""" + mark("chain") + return "ok" + + +shipping = restate.VirtualObject("lg_shipping") + + +@shipping.handler() +async def status(ctx: restate.ObjectContext, order: str) -> str: + return "shipped" + + +def probing(*given: Any, **options: Any) -> Callable[..., Awaitable[Any]]: + """Return how to ask an agent with ``given`` tools, in process, which notes the scopes in ``scopes``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + mark("remote") + return super().generic_call(*args, **kwargs) + + asker = sessions.SessionObject("asker") + model = Probing() + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await langgraph.prompt(ctx, text, model=model, tools=[*given], **options) + + @asker.handler + async def summarize(ctx: sessions.SessionContext, text: str) -> str: + from restate.agents.langgraph import _agent # pylint: disable=import-outside-toplevel + + return await _agent._summarizer(model)(ctx, text) # pyright: ignore[reportPrivateUsage] + + backend = Backend() + + async def asked(text: str, handler: str = "ask") -> Any: + scopes.clear() + now[:] = [Watching(backend, service="asker", answers={"publish": REVISION, "status": b'"shipped"'})] + return await invoke(asker, handler, now[0], request(), text) + + return asked + + +async def test_the_built_in_agents_model_request_is_kept_alive_and_the_work_around_it_is_not(): + assert await probing()("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_the_summary_of_earlier_turns_is_requested_kept_alive(): + assert await probing()("hi there", "summarize") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_and_a_langchain_tool_is_not(): + asked = probing(plain, kept, chain) + for word, seen in (("plain", 0), ("kept", 1), ("chain", 0)): + assert await asked(f"{word} x") == "done" + assert scopes == [("model", 1), (word, seen), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_call_on_restate_that_a_tool_makes_is_outside_any_scope(): + from restate.agents import call # pylint: disable=import-outside-toplevel + + asked = probing(call(status, description="The status of a shipment.")) + assert await asked("lg_shipping_status x") == "done" + assert scopes == [("model", 1), ("remote", 0), ("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_or_failed_request_leaves_no_scope_open(): + stuck.clear() + asked = probing() + scopes.clear() + task = asyncio.ensure_future(asked("stuck x")) + while not scopes: + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + with pytest.raises(Exception): + await asked("fail x") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 diff --git a/tests/agents_langgraph_graph.py b/tests/agents_langgraph_graph.py index 9ac2d9a..f817f1e 100644 --- a/tests/agents_langgraph_graph.py +++ b/tests/agents_langgraph_graph.py @@ -20,6 +20,7 @@ import threading import typing import uuid +from collections.abc import Awaitable, Callable from dataclasses import dataclass from datetime import timedelta from typing import Any, NotRequired, TypedDict @@ -45,6 +46,7 @@ from support import RESTATE_IMAGE # noqa: E402 # pylint: disable=import-error from restate import HarnessEnvironment, sessions # noqa: E402 from restate.agents import langgraph # noqa: E402 +from restate.agents import tool as restate_tool # noqa: E402 from restate.agents.internal import CANCELLED, ENDED # noqa: E402 from restate.agents.langgraph import nodes # noqa: E402 from restate.agents.langgraph._errors import terminal_of # noqa: E402 @@ -973,3 +975,164 @@ def test_context_is_for_a_graph_of_your_own(): make: Any = langgraph.agent with pytest.raises(TypeError, match="context is an option of your own graph: pass it with graph="): make("plain", context=lambda ctx: Caller(ctx.key)) + + +# --- Kept alive ------------------------------------------------------------------------------------- + +scopes: list[tuple[str, int]] = [] +"""The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" +now: list[Any] = [] +"""The host of the run in progress.""" +stuck = asyncio.Event() + + +def mark(where: str) -> None: + scopes.append((where, now[0].open_scopes)) + + +class Probing(BaseChatModel): + """A model that notes the scopes it sees, and waits on ``stuck`` (never set) for a message that says ``stuck``.""" + + @property + def _llm_type(self) -> str: + return "probing" + + def _generate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + raise AssertionError("the requests are async") + + async def _agenerate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: + mark("model") + if "stuck" in str(messages[-1].content): + await stuck.wait() + if "fail" in str(messages[-1].content): + raise RuntimeError("the model API is down") + return ChatResult(generations=[ChatGeneration(message=AIMessage("done"))]) + + +@restate_tool +async def plain() -> str: + """A tool of ours.""" + mark("plain") + return "ok" + + +@restate_tool(keepalive=True) +async def kept() -> str: + """A tool of ours that is kept alive.""" + mark("kept") + return "ok" + + +def _kept_graph() -> Any: + """A graph of the user's own: a node that asks a model (without passing it the run's config, as nodes do), one + that interrupts, one that asks a model in a subgraph, and one that calls tools.""" + model = Probing() + + async def llm(state: MessagesState) -> dict[str, Any]: + return {"messages": [await model.ainvoke(state["messages"])]} + + async def ask(state: MessagesState) -> dict[str, Any]: + mark("before the interrupt") + return {"messages": [AIMessage(f"approved: {interrupt('approve?')}")]} + + def calls(state: MessagesState) -> dict[str, Any]: + word = str(state["messages"][-1].content).split(" ")[0] + return {"messages": [AIMessage("", tool_calls=[{"id": f"call-{word}", "name": word, "args": {}}])]} + + inner = StateGraph(MessagesState) + inner.add_node("llm", llm) + inner.add_edge(START, "llm") + + def route(state: MessagesState) -> str: + return {"ask": "ask", "plain": "calls", "kept": "calls", "nested": "nested"}.get( + str(state["messages"][-1].content).split(" ")[0], "llm" + ) + + outer = StateGraph(MessagesState) + outer.add_node("llm", llm) + outer.add_node("ask", ask) + outer.add_node("calls", calls) + outer.add_node("tools", ToolNode(langgraph.tools([plain, kept]))) + outer.add_node("nested", inner.compile()) + outer.add_conditional_edges(START, route, ["llm", "ask", "calls", "nested"]) + outer.add_edge("calls", "tools") + for end in ("llm", "ask", "tools", "nested"): + outer.add_edge(end, END) + return outer + + +def keeping() -> Callable[..., Awaitable[Any]]: + """Return how to ask, in process, an agent of the user's own graph, which notes its scopes in ``scopes``.""" + from sessions_database import Backend # pylint: disable=import-error,import-outside-toplevel + from sessions_object import Host, request # pylint: disable=import-error,import-outside-toplevel + + from restate.agents import AgentObject # pylint: disable=import-outside-toplevel + from restate.sessions.internal import invoke # pylint: disable=import-outside-toplevel + + class Watching(Host): + def generic_call(self, *args: Any, **kwargs: Any) -> Any: + if args[1] == "wait": + mark("wait") + return super().generic_call(*args, **kwargs) + + graph = _kept_graph() + asker = AgentObject("kept_asker") + + @asker.handler + async def ask(ctx: sessions.SessionContext, text: str) -> str: + return await langgraph.prompt(ctx, text, graph=graph) + + backend = Backend() + + async def asked(text: str) -> Any: + scopes.clear() + answers = {"publish": b'{"revision": 1}', "wait": b'"yes"'} + now[:] = [Watching(backend, service="kept_asker", answers=answers)] + return await invoke(asker, "ask", now[0], request(), text) + + return asked + + +async def test_a_graphs_model_request_is_kept_alive_wherever_a_node_makes_it(): + asked = keeping() + assert await asked("hi") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + # In a subgraph the graph's nodes run: the handler is inherited. + assert await asked("nested x") == "done" + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 + + +async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_in_a_graph_too(): + asked = keeping() + await asked("plain x") + assert scopes == [("plain", 0)] + await asked("kept x") + assert scopes == [("kept", 1)] + assert now[0].open_scopes == 0 + + +async def test_an_interrupts_wait_for_its_signal_is_outside_any_scope(): + asked = keeping() + assert await asked("ask x") == "approved: yes" + assert scopes == [("before the interrupt", 0), ("wait", 0), ("before the interrupt", 0)] + assert now[0].open_scopes == 0 + + +async def test_a_stopped_or_failed_request_of_a_graph_leaves_no_scope_open(): + stuck.clear() + asked = keeping() + scopes.clear() + task = asyncio.ensure_future(asked("stuck x")) + while not scopes: + await asyncio.sleep(0.01) + assert now[0].open_scopes == 1 + task.cancel() + with pytest.raises(asyncio.CancelledError): + await task + assert now[0].open_scopes == 0 + with pytest.raises(Exception): + await asked("fail x") + assert scopes == [("model", 1)] + assert now[0].open_scopes == 0 From e5f94ac90e69f20cb07819d2013542dc363c7506 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:27:03 +0000 Subject: [PATCH 4/6] Tests: a context of a test's own has working, as SessionContext declares --- tests/sessions_object.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/sessions_object.py b/tests/sessions_object.py index cfecee1..a848dfb 100644 --- a/tests/sessions_object.py +++ b/tests/sessions_object.py @@ -660,6 +660,9 @@ def generic_send(self, service: str, handler: str, arg: Any = None, **options: A def cancel(self, invocation_id: str) -> None: pass + def working(self) -> contextlib.AbstractAsyncContextManager[None]: + return contextlib.nullcontext() + async def test_a_handler_runs_over_a_fake_context_of_a_test(): fake = FakeContext(db=FakeDatabase()) From 23cb6bbeb7597fbb6a107bb6ed85d5e27a87f0b4 Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 13:41:54 +0000 Subject: [PATCH 5/6] Keep-alive: a sync LangGraph request does not hang, and the first heartbeat comes within an interval - LangChain runs the callbacks of a sync request (`invoke`, in a sync node) on a loop of its own, in a thread, and waits there for every task they start. The handler entered `ctx.working()` there, whose heartbeat task never ends: the request never went out. It now opens and closes its scopes on the attempt's loop, handed over without waiting; one that arrives after its stream ended is dropped. A test with a sync node, whose `working()` starts a task as the driver's does, hung before. - `ctx.working()` refuses to be entered on another loop than the attempt's. - A scope's first heartbeat waited a whole interval after any recent write, so the silence could reach almost two intervals; it now comes once the silence reaches one. --- python/restate/agents/langgraph/_graph.py | 35 ++++++++++++++--- python/restate/sessions/_driver.py | 18 ++++++--- tests/agents_langgraph_graph.py | 47 ++++++++++++++++++++--- 3 files changed, 83 insertions(+), 17 deletions(-) diff --git a/python/restate/agents/langgraph/_graph.py b/python/restate/agents/langgraph/_graph.py index 6d8e42a..8047b94 100644 --- a/python/restate/agents/langgraph/_graph.py +++ b/python/restate/agents/langgraph/_graph.py @@ -106,7 +106,7 @@ def forget(state: MessagesState) -> dict[str, Any]: import asyncio import json import typing -from collections.abc import Sequence +from collections.abc import Callable, Coroutine, Sequence from contextlib import AsyncExitStack from typing import Any from uuid import UUID @@ -379,6 +379,11 @@ class _KeepAlive(AsyncCallbackHandler): A scope the callbacks leave open (a request that never ended) is closed as the run's stream ends (``release_all``). + + A sync request (``invoke``, in a sync node) has LangChain run these callbacks on a loop of its own, in a thread, + and wait there for every task they started: a scope is the attempt's, with a heartbeat on the attempt's loop, so + it is opened and closed there instead, handed over without waiting (the attempt's loop may be the one the sync + request blocks). A sync request on the attempt's own loop blocks its heartbeat anyway. """ run_inline = True @@ -387,8 +392,23 @@ class _KeepAlive(AsyncCallbackHandler): def __init__(self, ctx: SessionContext) -> None: self.ctx = ctx self.requests: dict[UUID, AsyncExitStack] = {} + self.loop = asyncio.get_running_loop() + """The attempt's loop: the one ``ctx.working`` runs on.""" + self._stream = 0 + """The stream of the run in progress: ``release_all`` ends one, and the run may stream again (after an + interrupt).""" + + async def _on_loop(self, work: Callable[[], Coroutine[Any, Any, None]]) -> None: + """Run ``work`` on the attempt's loop: now if this is it, or else handed over (in order) without waiting.""" + if asyncio.get_running_loop() is self.loop: + await work() + else: + asyncio.run_coroutine_threadsafe(work(), self.loop) - async def _open(self, run_id: UUID) -> None: + async def _open(self, run_id: UUID, stream: int) -> None: + if stream != self._stream: + # A request whose start reached the attempt's loop after its stream ended: nothing to keep alive. + return await self._close(run_id) stack = AsyncExitStack() await stack.enter_async_context(self.ctx.working()) @@ -400,22 +420,25 @@ async def _close(self, run_id: UUID) -> None: await stack.aclose() async def release_all(self) -> None: + self._stream += 1 stacks = list(self.requests.values()) self.requests.clear() for stack in stacks: await stack.aclose() async def on_chat_model_start(self, serialized: Any, messages: Any, *, run_id: UUID, **kwargs: Any) -> None: - await self._open(run_id) + stream = self._stream + await self._on_loop(lambda: self._open(run_id, stream)) async def on_llm_start(self, serialized: Any, prompts: Any, *, run_id: UUID, **kwargs: Any) -> None: - await self._open(run_id) + stream = self._stream + await self._on_loop(lambda: self._open(run_id, stream)) async def on_llm_end(self, response: Any, *, run_id: UUID, **kwargs: Any) -> None: - await self._close(run_id) + await self._on_loop(lambda: self._close(run_id)) async def on_llm_error(self, error: BaseException, *, run_id: UUID, **kwargs: Any) -> None: - await self._close(run_id) + await self._on_loop(lambda: self._close(run_id)) def run_of(ctx: SessionContext, graph: AnyGraph, context: Any = None) -> Run: diff --git a/python/restate/sessions/_driver.py b/python/restate/sessions/_driver.py index 68cfbb6..265b7f6 100644 --- a/python/restate/sessions/_driver.py +++ b/python/restate/sessions/_driver.py @@ -481,11 +481,17 @@ async def working(self) -> AsyncIterator[None]: """Keep the attempt alive while the body runs (see ``HEARTBEAT_INTERVAL``). One heartbeat every interval, from the first scope opening to the last - closing. The first comes at once if the attempt has been silent for an - interval already (most of the timeout may be gone), so the silence never - reaches twice that; otherwise the scopes the database opens for each of - its calls would each send one. + closing. The first comes once the attempt has been silent for an + interval (at once, if it already has), so the silence never reaches more + than that; a scope that closes sooner (the database's, around each of its + calls) sends nothing. + + Raises: + RuntimeError: Entered on another event loop than the attempt's (a + thread's own loop): the heartbeat is the attempt's. """ + if asyncio.get_running_loop() is not self._loop: + raise RuntimeError("ctx.working() is entered on the attempt's own event loop, not another thread's") self._scopes += 1 if self._scopes == 1 and self.ended is None: self._heartbeats = asyncio.ensure_future(self._heartbeat()) @@ -500,8 +506,8 @@ async def working(self) -> AsyncIterator[None]: async def _heartbeat(self) -> None: """Tell Restate the attempt makes progress, while a ``working`` scope is open.""" interval = HEARTBEAT_INTERVAL.total_seconds() - if self._loop.time() - self._wrote_at < interval: - await asyncio.sleep(interval) + # The first once the silence reaches an interval: at once, if it already has. + await asyncio.sleep(max(0.0, interval - (self._loop.time() - self._wrote_at))) while True: try: self.act(lambda vm: vm.heartbeat()) diff --git a/tests/agents_langgraph_graph.py b/tests/agents_langgraph_graph.py index f817f1e..9c5f736 100644 --- a/tests/agents_langgraph_graph.py +++ b/tests/agents_langgraph_graph.py @@ -16,11 +16,12 @@ # pylint: disable=missing-function-docstring,missing-class-docstring import asyncio +import contextlib import collections import threading import typing import uuid -from collections.abc import Awaitable, Callable +from collections.abc import AsyncIterator, Awaitable, Callable from dataclasses import dataclass from datetime import timedelta from typing import Any, NotRequired, TypedDict @@ -979,6 +980,8 @@ def test_context_is_for_a_graph_of_your_own(): # --- Kept alive ------------------------------------------------------------------------------------- +off_loop: list[bool] = [] +"""Each ``ctx.working`` scope entered off the attempt's event loop.""" scopes: list[tuple[str, int]] = [] """The ``ctx.working`` scopes open (``Host.open_scopes``) when each part of a run saw it, in order.""" now: list[Any] = [] @@ -998,7 +1001,9 @@ def _llm_type(self) -> str: return "probing" def _generate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: - raise AssertionError("the requests are async") + # A sync node's request (`invoke`), in a thread of the graph's. + mark("sync model") + return ChatResult(generations=[ChatGeneration(message=AIMessage("done"))]) async def _agenerate(self, messages: list[BaseMessage], *args: Any, **kwargs: Any) -> ChatResult: mark("model") @@ -1031,6 +1036,10 @@ def _kept_graph() -> Any: async def llm(state: MessagesState) -> dict[str, Any]: return {"messages": [await model.ainvoke(state["messages"])]} + def sync_llm(state: MessagesState) -> dict[str, Any]: + # The usual shape of LangGraph's examples: LangChain runs the callbacks of a sync request on a loop of its own. + return {"messages": [model.invoke(state["messages"])]} + async def ask(state: MessagesState) -> dict[str, Any]: mark("before the interrupt") return {"messages": [AIMessage(f"approved: {interrupt('approve?')}")]} @@ -1044,19 +1053,20 @@ def calls(state: MessagesState) -> dict[str, Any]: inner.add_edge(START, "llm") def route(state: MessagesState) -> str: - return {"ask": "ask", "plain": "calls", "kept": "calls", "nested": "nested"}.get( + return {"ask": "ask", "plain": "calls", "kept": "calls", "nested": "nested", "sync": "sync_llm"}.get( str(state["messages"][-1].content).split(" ")[0], "llm" ) outer = StateGraph(MessagesState) outer.add_node("llm", llm) + outer.add_node("sync_llm", sync_llm) outer.add_node("ask", ask) outer.add_node("calls", calls) outer.add_node("tools", ToolNode(langgraph.tools([plain, kept]))) outer.add_node("nested", inner.compile()) - outer.add_conditional_edges(START, route, ["llm", "ask", "calls", "nested"]) + outer.add_conditional_edges(START, route, ["llm", "sync_llm", "ask", "calls", "nested"]) outer.add_edge("calls", "tools") - for end in ("llm", "ask", "tools", "nested"): + for end in ("llm", "sync_llm", "ask", "tools", "nested"): outer.add_edge(end, END) return outer @@ -1075,6 +1085,22 @@ def generic_call(self, *args: Any, **kwargs: Any) -> Any: mark("wait") return super().generic_call(*args, **kwargs) + @contextlib.asynccontextmanager + async def working(self) -> AsyncIterator[None]: + # As the driver's: a task that beats while the scope is open, on the loop the scope is entered on. A + # scope entered on another loop than the attempt's would hold that loop's caller until it closes. + if asyncio.get_running_loop() is not loop: + # LangChain swallows a handler's error: noted here instead, for the test to see. + off_loop.append(True) + async with super().working(): + beat = asyncio.ensure_future(asyncio.sleep(3600)) + try: + yield + finally: + beat.cancel() + + loop = asyncio.get_running_loop() + graph = _kept_graph() asker = AgentObject("kept_asker") @@ -1086,6 +1112,7 @@ async def ask(ctx: sessions.SessionContext, text: str) -> str: async def asked(text: str) -> Any: scopes.clear() + off_loop.clear() answers = {"publish": b'{"revision": 1}', "wait": b'"yes"'} now[:] = [Watching(backend, service="kept_asker", answers=answers)] return await invoke(asker, "ask", now[0], request(), text) @@ -1104,6 +1131,16 @@ async def test_a_graphs_model_request_is_kept_alive_wherever_a_node_makes_it(): assert now[0].open_scopes == 0 +async def test_a_sync_nodes_model_request_does_not_hang_and_leaves_no_scope_open(): + # LangChain runs the callbacks of a sync request (`invoke`) on a loop of its own, in a thread, and waits there for + # every task they started: a scope entered there, with its heartbeat's task, would never let the request go out. + asked = keeping() + assert await asyncio.wait_for(asked("sync now"), timeout=10) == "done" + assert [where for where, _ in scopes] == ["sync model"] + assert not off_loop, "a scope was entered off the attempt's loop" + assert now[0].open_scopes == 0 + + async def test_a_tool_of_ours_is_kept_alive_only_with_keepalive_in_a_graph_too(): asked = keeping() await asked("plain x") From 1c01bee79743d422d76f65d62998b5c2892cbc5c Mon Sep 17 00:00:00 2001 From: Claude Date: Thu, 19 Nov 2026 16:03:12 +0000 Subject: [PATCH 6/6] Sessions: ctx.working()'s error off the attempt's loop states the requirement --- python/restate/sessions/_driver.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python/restate/sessions/_driver.py b/python/restate/sessions/_driver.py index 265b7f6..4bc9c3e 100644 --- a/python/restate/sessions/_driver.py +++ b/python/restate/sessions/_driver.py @@ -491,7 +491,7 @@ async def working(self) -> AsyncIterator[None]: thread's own loop): the heartbeat is the attempt's. """ if asyncio.get_running_loop() is not self._loop: - raise RuntimeError("ctx.working() is entered on the attempt's own event loop, not another thread's") + raise RuntimeError("ctx.working() must be entered on the attempt's own event loop, not another thread's") self._scopes += 1 if self._scopes == 1 and self.ended is None: self._heartbeats = asyncio.ensure_future(self._heartbeat())