Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 3 additions & 2 deletions Cargo.lock

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ tracing-subscriber = { version = "0.3", features = ["fmt", "env-filter"] }
# checkpoints may outgrow Restate into (RESTATE_SQL_OBJECT_STORE: S3, GCS, Azure, or a
# directory); `tunnel` is Restate Cloud's. `aws_lc_rs` is the crypto of all of them (the
# tunnel's TLS, request identity's signatures), as the object stores' must be aws-lc.
restate-sdk-shared-core = { version = "8.6.0", features = [
restate-sdk-shared-core = { version = "8.7.0", features = [
"request_identity",
"sha2_random_seed",
"aws_lc_rs",
Expand Down
19 changes: 11 additions & 8 deletions docs/agents/turns.md
Original file line number Diff line number Diff line change
Expand Up @@ -455,14 +455,17 @@ sequenceDiagram
does nothing), never the message that started after it.
- The turn ends cancelled whenever the cancellation comes, and the next
message starts afresh: the cancelled prompt is over.
- The cancellation reaches the turn's work at once while Restate keeps the
turn's input open. Restate closes it once the turn has sent and received
nothing for the inactivity timeout (a minute by default: a long model
request, or a slow tool). The turn works on, as watching for the
cancellation holds nothing, and a cancellation then reaches it at its next
step, when it next commits or calls. For turns whose steps take longer,
raise Restate's inactivity timeout (the server's
`worker.invoker.inactivity-timeout`).
- The cancellation reaches the turn's work at once. Restate shuts down an
attempt it has heard nothing from for the inactivity timeout (a minute by
default), and the next attempt runs the turn from its start. So while the
turn works on its own (a model request, a tool), its attempt tells Restate
every 20 seconds that it makes progress, and Restate keeps it however long
the step takes. While the turn waits for an answer of Restate's instead (a
tool waiting for an approval, a call), it tells Restate nothing: Restate
shuts the attempt down after the timeout, so that a wait of days holds no
process, and the next attempt gets the cancellation as it starts. A
deployment that sets an inactivity timeout under 20 seconds is shut down
between two of those messages.
- A cancellation that comes once the turn answered changes nothing: the
answer stands.
- A tool cut short by the stop raises the stop's reason, so the model never
Expand Down
67 changes: 65 additions & 2 deletions python/restate/_internal.pyi
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,69 @@ _SqlValue = Union[None, int, float, str, bytes]

def __getattr__(name: str) -> Any: ...

class AttemptShutDownException(Exception):
"""Restate shut the attempt down: a new attempt will run the handler from the start."""

class PyInput:
invocation_id: str
random_seed: int
key: str
headers: list[Any]
input: bytes
scope: str | None
limit_key: str | None
idempotency_key: str | None

class PyExplicitRecoveryVM:
"""The VM of a session object's attempt (``src/explicit_recovery.rs``)."""

def __init__(self, headers: Sequence[tuple[str, str]], bidirectional: bool) -> None: ...
def set_scheduler(self, scheduler: Callable[[], None]) -> None: ...
def get_response_head(self) -> Any: ...
def notify_input(self, buffer: bytes) -> None: ...
def notify_input_closed(self) -> None: ...
def take_output(self) -> bytes: ...
def is_ready_to_execute(self) -> bool: ...
def input(self) -> PyInput: ...
def take_notification(self) -> tuple[int, Any] | None: ...
def get_state(self, key: str, name: str | None = None) -> int: ...
def get_state_keys(self, name: str | None = None) -> int: ...
def call(
self,
service: str,
handler: str,
buffer: bytes,
key: str | None = None,
idempotency_key: str | None = None,
headers: Sequence[Any] | None = None,
name: str | None = None,
) -> Any: ...
def attach_invocation(self, invocation_id: str, name: str | None = None) -> int: ...
def set_state(self, key: str, buffer: bytes, name: str | None = None, ack: bool = False) -> int: ...
def clear_state(self, key: str, name: str | None = None, ack: bool = False) -> int: ...
def clear_all_state(self, name: str | None = None, ack: bool = False) -> int: ...
def cancel_invocation(self, invocation_id: str, name: str | None = None, ack: bool = False) -> int: ...
def send(
self,
service: str,
handler: str,
buffer: bytes,
key: str | None = None,
delay: int | None = None,
idempotency_key: str | None = None,
headers: Sequence[Any] | None = None,
name: str | None = None,
) -> int: ...
def start_trace(self, name: str, parent: int | None = None) -> int: ...
def end_trace_success(self, trace: int, buffer: bytes) -> None: ...
def end_trace_failure(self, trace: int, failure: Any) -> None: ...
def heartbeat(self) -> None: ...
def end_success(self, buffer: bytes) -> None: ...
def end_failure(self, failure: Any) -> None: ...
def notify_error(self, error: str, stacktrace: str | None = None, delay_override_ms: int | None = None) -> None: ...
def is_cancelled(self) -> bool: ...
def state(self) -> str: ...

class PyVM:
"""The invocation's VM (``restate.vm.VMWrapper`` wraps it)."""

Expand All @@ -50,7 +113,7 @@ SQL_SET: int
SQL_CLEAR: int
SQL_SYNC: int
SQL_KEYS: int
SQL_WAIT: int
SQL_COMMIT: int

class SqlRefusedException(Exception):
"""A failure of the database that a retry would repeat (shared-core's ``sql::Refused``)."""
Expand Down Expand Up @@ -79,6 +142,6 @@ def sql_open(
dispatch: Callable[[int, Any, bytes | None, SqlReply | None], None],
lists_keys: bool = False,
reader: bool = False,
vm: PyVM | None = None,
vm: PyExplicitRecoveryVM | None = None,
name: str | None = None,
) -> _Answer[PySqlDatabase]: ...
25 changes: 18 additions & 7 deletions python/restate/agents/_turn.py
Original file line number Diff line number Diff line change
Expand Up @@ -112,12 +112,18 @@ async def guard(self, work: Awaitable[T]) -> T:

If it stops first, ``work`` is cancelled and the stop reason is
raised. Use it for a framework request that stopping the turn must cut
short.
short. Cancelling the task that awaits it (as the end of an attempt
cancels its handler) cancels ``work`` too.
"""
task = asyncio.ensure_future(work)
stopping = asyncio.ensure_future(self._event.wait())
try:
await asyncio.wait({task, stopping}, return_when=asyncio.FIRST_COMPLETED)
except BaseException:
# The guard itself is cancelled: so is the work, whose end nobody awaits.
task.cancel()
task.add_done_callback(_retrieve)
raise
finally:
stopping.cancel()
if task.done():
Expand All @@ -129,6 +135,12 @@ async def guard(self, work: Awaitable[T]) -> T:
raise self.reason


def _retrieve(task: asyncio.Future[typing.Any]) -> None:
"""Retrieve the end of a task nobody awaits, so that asyncio logs no "exception was never retrieved"."""
if not task.cancelled():
task.exception()


_STOP: ScopeKey[Stop] = ScopeKey("stop_signal_of")


Expand Down Expand Up @@ -371,12 +383,11 @@ async def chat_turn(ctx: AgentContext, text: str, answer: Callable[[AgentContext
While it answers, the object's ``status`` reports the turn. Cancelling the
invocation stops the turn (through ``stop_signal_of``) whenever it comes,
and cancels the calls its tools made; the handler then fails with the
cancellation. The cancellation reaches the turn's work at once while
Restate keeps the turn's input open; once Restate closed it (nothing
sent or received for its inactivity timeout: a long model request, say),
at the turn's next step, when it next commits or calls (see
``cancelled`` of ``restate.sessions.internal``). Watching for it holds
nothing: the turn works on. If the attempt ends first, the turn stops with
cancellation. The cancellation reaches the turn's work at once (see
``cancelled`` of ``restate.sessions.internal``): watching for it holds
nothing, and while the turn works on its own (a model request, a tool)
its attempt tells Restate that it makes progress, so Restate keeps it
past its inactivity timeout. If the attempt ends first, the turn stops with
``AttemptEnded`` and the next attempt carries on. The turn publishes when
it starts (the companion's ``start`` does), after each write of its steps
that commits (a reply with tool calls, the tools' results: ``answer_once``
Expand Down
2 changes: 2 additions & 0 deletions python/restate/handler.py
Original file line number Diff line number Diff line change
Expand Up @@ -283,6 +283,8 @@ class Handler(Generic[I, O]):
context_managers: Optional[List[Callable[[], AsyncContextManager[None]]]] = None
explicit_recovery: bool = False
explicit_cancellation: bool = False
attempt_runner: Optional[Callable[..., Awaitable[None]]] = None
"""What runs an attempt instead of the SDK's context, on a VM of its own (a session object's)."""


# disable too many arguments warning
Expand Down
12 changes: 12 additions & 0 deletions python/restate/server.py
Original file line number Diff line number Diff line change
Expand Up @@ -410,6 +410,18 @@ async def app(scope: Scope, receive: Receive, send: Send):
active_channels.add(receive_channel)
try:
journal_codec = await get_codec()
if handler.attempt_runner is not None:
await handler.attempt_runner(
handler,
request_headers=request_headers,
attempt_headers=dict(request_headers),
receive=receive_channel,
send=send,
bidirectional=scope["http_version"] != "1.1" and endpoint.protocol != "request_response",
server_headers=X_RESTATE_SERVER,
journal_codec=journal_codec,
)
return
await process_invocation_to_completion(
VMWrapper(
request_headers,
Expand Down
34 changes: 34 additions & 0 deletions python/restate/sessions/_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -266,6 +266,37 @@ class Database(DatabaseWriter, Protocol):

Each statement outside ``transaction()`` is a transaction of its own,
durable once its call returns. Calls run one at a time, in call order.

A cancellation of the invocation reaches the handler once, as
``TerminalError("cancelled", 409)`` (unless the handler has
``explicit_cancellation``). When it comes, the calls Restate has not
answered are cancelled, and every await of Restate then pending fails with
it: a call, an attached invocation. If none was pending, it is carried to
the handler's next such await, or to its next call of the database; a
handler that awaits nothing more does not see it. A call made while it is
carried is made all the same: its result fails, and the callee runs on.

A call of the database is treated differently from an await of Restate, so
that its failure means one thing. A call already under way when the
cancellation comes runs to its end. A call of the database that the
carried cancellation fails was refused before it began, and did nothing:
it wrote nothing, and a ``transaction()`` neither committed nor made its
sends. A transaction that returns committed, and its sends are made, once.
(A transaction also raises the cancellation when its body awaits a call
that the cancellation fails, unless the body catches it: it rolls back, as
for any error of its body.)

What the handler awaits after the cancellation works, so it can
compensate, or make the call again::

try:
async with ctx.db.transaction() as tx:
...
except restate.TerminalError as error:
if error.status_code != 409 or error.message != "cancelled":
raise
await ctx.db.execute("UPDATE orders SET status = 'cancelled' WHERE id = ?", [order])
raise
"""

def transaction(self) -> typing.AsyncContextManager[Transaction]:
Expand All @@ -292,6 +323,9 @@ class SessionContext(Caller, Sender, Protocol):
suspension, Restate starts a new attempt, which runs the handler from the
start and finds what earlier attempts committed. Its operations may be
awaited together, with ``asyncio.gather`` or ``asyncio.wait``.

A cancellation reaches the handler once (see ``Database``). An attempt
that follows a cancelled one gets it at its first await.
"""

@property
Expand Down
25 changes: 13 additions & 12 deletions python/restate/sessions/_database.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,11 @@

from restate._internal import ( # pylint: disable=import-error,no-name-in-module
SQL_CLEAR,
SQL_COMMIT,
SQL_GET,
SQL_KEYS,
SQL_SET,
SQL_SYNC,
SQL_WAIT,
PySqlDatabase,
SqlRefusedException,
SqlReply,
Expand Down Expand Up @@ -96,11 +96,9 @@ class Store(Protocol):
- ``keys()`` returns every key, as ``ctx.state_keys()`` does, or an
awaitable of them. With it, a commit that writes a checkpoint deletes
the packs no checkpoint needs any more; without it, they are kept.
- ``vm`` is the invocation's VM (``restate._internal.PyVM``), for a store
over a Restate context. The extension then reads and writes the VM
directly instead of calling ``get``, ``set`` and ``clear``, and calls
``wait(handle)`` (an awaitable that completes once the VM has the read
for ``handle``) only when it must wait for a read.
- ``vm`` is the attempt's VM (``restate._internal.PyExplicitRecoveryVM``),
for a store over Restate. The extension then reads, writes, lists and
syncs with the VM directly, instead of calling the store's methods.
- ``database_name`` names a database of the key other than the object's
own (see ``_session.database``).
"""
Expand Down Expand Up @@ -159,7 +157,6 @@ def __init__(self, store: Store, loop: asyncio.AbstractEventLoop) -> None:
self.loop: asyncio.AbstractEventLoop = loop
# The store's optional methods (see `Store`).
self._keys: Callable[[], Sequence[str] | Awaitable[Sequence[str]]] | None = getattr(store, "keys", None)
self._wait: Callable[[int], Awaitable[None]] | None = getattr(store, "wait", None)
self.lists_keys: bool = callable(self._keys)
# The store's first error.
self.failed: BaseException | None = None
Expand All @@ -173,7 +170,7 @@ def _keep(self, error: BaseException) -> None:
self.failed = error

def dispatch(self, op: int, key: Any, value: bytes | None, reply: SqlReply | None) -> None:
"""Handle one request of the extension; ``key`` is a store key, or a VM handle for ``SQL_WAIT``."""
"""Handle one request of the extension; ``key`` is a store key, or empty for a sync or a commit."""
if op in (SQL_SET, SQL_CLEAR):
try:
if op == SQL_SET:
Expand All @@ -193,9 +190,13 @@ def dispatch(self, op: int, key: Any, value: bytes | None, reply: SqlReply | Non
elif op == SQL_KEYS:
assert self._keys is not None
result = self._keys()
elif op == SQL_WAIT:
assert self._wait is not None
result = self._wait(key)
elif op == SQL_COMMIT:
# A store over the VM syncs with an acked write of the head: what the transaction
# makes at its commit runs between that write and the wait for its ack.
run, self.with_sync = self.with_sync, None
if run is not None:
run()
result = None
elif self._write_failed is not None:
reply(None, str(self._write_failed))
return
Expand Down Expand Up @@ -230,7 +231,7 @@ def done(future: asyncio.Future[object]) -> None:

def _reply_value(op: int, result: object) -> object:
"""Convert a request's result to what the extension expects: a value, a list of keys, or ``None``."""
if op in (SQL_SYNC, SQL_WAIT):
if op in (SQL_SYNC, SQL_COMMIT):
return None
if op == SQL_KEYS:
return list(typing.cast("Sequence[str]", result))
Expand Down
Loading
Loading