feat(inworld): Inworld Realtime Plugin Support

examples/voice_agents/inworld_realtime_api.py

@@ -0,0 +1,72 @@
+from dotenv import load_dotenv
+
+from livekit.agents import (
+    Agent,
+    AgentServer,
+    AgentSession,
+    JobContext,
+    cli,
+    function_tool,
+    llm,
+    room_io,
+)
+from livekit.plugins.inworld.realtime import RealtimeModel
+
+load_dotenv()
+
+
+class MyAgent(Agent):
+    def __init__(self) -> None:
+        super().__init__(
+            instructions="You are Jessica, a helpful assistant",
+            llm=RealtimeModel(
+                model="google-ai-studio/gemini-3.1-flash-lite",
+                voice="Ashley",
+                tts_model="inworld-tts-2",
+                stt_model="inworld/inworld-stt-1",
+                modalities=["audio"],
+                provider_data={"auto_tool_response": False},
+            ),
+        )
+
+    async def on_enter(self):
+        chat_history = [
+            {
+                "role": "user",
+                "content": "Hello. I'm just picking up.",
+            },
+        ]
+        # Google models require a user item to generate a response, so pass it through chat_ctx.
+        chat_ctx = llm.ChatContext.empty()
+        for item in chat_history:
+            chat_ctx.add_message(role=item["role"], content=item["content"])
+
+        self.session.generate_reply(
+            instructions="introduce yourself very briefly and ask about the user's day",
+            chat_ctx=chat_ctx,
+        )
+
+    @function_tool
+    async def get_weather(self, city: str):
+        """Get the weather for a given city"""
+        return f"The weather in {city} is sunny and 70 degrees"
+
+
+server = AgentServer()
+
+
+@server.rtc_session()
+async def entrypoint(ctx: JobContext):
+    session = AgentSession()
+
+    await session.start(
+        agent=MyAgent(),
+        room=ctx.room,
+        room_options=room_io.RoomOptions(
+            text_output=room_io.TextOutputOptions(transcription_speed_factor=1.5),
+        ),
+    )
+
+
+if __name__ == "__main__":
+    cli.run_app(server)

livekit-plugins/livekit-plugins-inworld/README.md

@@ -204,3 +204,76 @@ session = AgentSession(
    # ... llm, etc.
 )
 ```
+
+### Realtime (speech-to-speech)
+
+Inworld's Realtime API is a single WebSocket that runs STT, LLM, and TTS server-side. It is
+wire-compatible with the OpenAI Realtime spec, so it plugs into `AgentSession` as the `llm`:
+
+```python
+from livekit.plugins import inworld
+
+session = AgentSession(
+    llm=inworld.realtime.RealtimeModel(
+        model="openai/gpt-4o-mini",   # LLM (or a router like "inworld/auto")
+        voice="Clive",                # TTS voice
+        tts_model="inworld-tts-2",    # or "inworld-tts-1.5-mini"
+        stt_model="inworld/inworld-stt-1",
+    )
+)
+```
+
+Inworld-specific extensions (STT tuning, TTS segmentation, memory, back-channel,
+responsiveness fillers, prompt caching, LLM generation params) are passed through
+`provider_data`. It is a typed `ProviderData` (a `TypedDict`), so you get autocompletion and
+type checking while still writing a plain dict:
+
+Automatic responses after a tool result are disabled by default
+(`auto_tool_response=False`) because LiveKit explicitly requests the continuation. Set the
+option to `True` in `provider_data` to opt into server-driven tool continuation.
+
+```python
+llm = inworld.realtime.RealtimeModel(
+    provider_data={
+        "stt": {"voice_profile": True, "language_hints": ["en-US"]},
+        "tts": {"segmenter_strategy": "sentence", "delivery_mode": "CREATIVE"},
+        "memory": {"enabled": True, "turn_interval": 5},
+        "text_generation_config": {"reasoning": {"effort": "LOW"}},
+        "user_id": "user_abc123",
+    },
+)
+```
+
+See the [Inworld Realtime API Extensions](https://docs.inworld.ai/realtime/provider-data)
+reference for every field.
+
+#### Protocol debugging
+
+Inworld Realtime sessions expose the raw OpenAI-compatible protocol events
+`openai_client_event_queued` and `openai_server_event_received`. A model subclass can attach
+temporary logging to every session created by `AgentSession`:
+
+```python
+import logging
+
+from livekit.plugins import inworld
+
+logger = logging.getLogger("inworld-realtime")
+
+
+class DebugRealtimeModel(inworld.realtime.RealtimeModel):
+    def session(self):
+        session = super().session()
+        session.on(
+            "openai_client_event_queued",
+            lambda event: logger.debug("client -> server: %s", event),
+        )
+        session.on(
+            "openai_server_event_received",
+            lambda event: logger.debug("server -> client: %s", event),
+        )
+        return session
+```
+
+For voice sessions, filter or summarize `response.output_audio.delta` events to avoid logging
+large base64 audio payloads.

livekit-plugins/livekit-plugins-inworld/livekit/plugins/inworld/__init__.py

@@ -18,6 +18,7 @@
 https://docs.livekit.io/agents/models/stt/inworld/ for more information.
 """
 
+from . import realtime
 from .stt import STT, SpeechStream
 from .tts import (
     TTS,
@@ -42,6 +43,7 @@
     "TTSModels",
     "TextNormalization",
     "TimestampType",
+    "realtime",
     "__version__",
 ]
 

livekit-plugins/livekit-plugins-inworld/livekit/plugins/inworld/realtime/__init__.py

@@ -0,0 +1,49 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from .provider_data import (
+    BackchannelProviderData,
+    CachingProviderData,
+    MemoryProviderData,
+    ProviderData,
+    ReasoningConfig,
+    ResponsivenessProviderData,
+    STTProviderData,
+    TextGenerationConfig,
+    TTSProviderData,
+)
+from .realtime_model import RealtimeModel, RealtimeSession
+
+__all__ = [
+    "RealtimeModel",
+    "RealtimeSession",
+    "ProviderData",
+    "STTProviderData",
+    "TTSProviderData",
+    "MemoryProviderData",
+    "BackchannelProviderData",
+    "ResponsivenessProviderData",
+    "CachingProviderData",
+    "TextGenerationConfig",
+    "ReasoningConfig",
+]
+
+# Cleanup docs of unexported modules
+_module = dir()
+NOT_IN_ALL = [m for m in _module if m not in __all__]
+
+__pdoc__ = {}
+
+for n in NOT_IN_ALL:
+    __pdoc__[n] = False

livekit-plugins/livekit-plugins-inworld/livekit/plugins/inworld/realtime/provider_data.py

@@ -0,0 +1,145 @@
+# Copyright 2025 LiveKit, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Typed schema for Inworld's ``providerData`` Realtime API extensions.
+
+These are ``TypedDict``s (plain dicts at runtime) so they serialize verbatim into the
+``session.update`` payload. All keys are optional; omit a key to inherit the server default.
+See https://docs.inworld.ai/realtime/provider-data for the field-by-field reference.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal, TypedDict
+
+
+class STTProviderData(TypedDict, total=False):
+    """``providerData.stt`` — STT tuning. Hot-swappable."""
+
+    prompt: str
+    voice_profile: bool
+    language_hints: list[str]
+    end_of_turn_confidence_threshold: float
+    vad_threshold: float
+    min_end_of_turn_silence: int
+    max_turn_silence: int
+
+
+class TTSProviderData(TypedDict, total=False):
+    """``providerData.tts`` — segmentation, language, delivery, alignment."""
+
+    segmenter_strategy: Literal[
+        "auto", "balanced", "sentence", "full_turn", "fast_start", "per_segment_context"
+    ]
+    steering_handling: Literal["repeat_each_chunk", "emit_once"]
+    language: str
+    delivery_mode: Literal["STABLE", "BALANCED", "CREATIVE"]
+    conversational: bool  # locked at session open
+    user_turn_mode: Literal["both", "audio_only", "text_only", "none"]  # locked at session open
+    timestamp_type: Literal["WORD", "CHARACTER"]
+    timestamp_transport_strategy: Literal["SYNC", "ASYNC"]
+
+
+class MemoryProviderData(TypedDict, total=False):
+    """``providerData.memory`` — automatic conversation memory and summarization."""
+
+    enabled: bool
+    turn_interval: int
+    max_memory_length: int
+    max_transcript_items: int
+    max_facts: int
+    trim_after_summarize: bool
+
+
+class BackchannelProviderData(TypedDict, total=False):
+    """``providerData.backchannel`` — short interjections while the user speaks."""
+
+    enabled: bool
+    small_model: str
+    eval_interval_ms: int
+    min_speech_ms: int
+    min_gap_ms: int
+    max_per_turn: int
+    hard_deadline_ms: int
+    history_tail_items: int
+    temperature: float
+    max_tokens: int
+    volume_gain: float
+    require_pause: bool
+    allowed_phrases: list[str]
+    prompt_template: str
+    decider_kind: str
+
+
+class ResponsivenessProviderData(TypedDict, total=False):
+    """``providerData.responsiveness`` — filler audio while the LLM warms up."""
+
+    enabled: bool
+    small_model: str
+    initial_wait_timeout_ms: int
+    hard_deadline_ms: int
+    history_tail_items: int
+    temperature: float
+    max_tokens: int
+    min_filler_gap_ms: int
+    max_initial_per_turn: int
+    max_buffer_deltas: int
+    enable_filler_on_first_assistant_reply: bool
+    prompt_template: str
+    pause_text: str
+
+
+class CachingProviderData(TypedDict, total=False):
+    """``providerData.caching`` — explicit prompt caching for instructions/tools."""
+
+    enabled: bool
+    ttl: str
+    cache_instructions: bool
+    cache_tools: bool
+
+
+class ReasoningConfig(TypedDict, total=False):
+    effort: Literal["NONE", "MINIMAL", "LOW", "MEDIUM", "HIGH", "XHIGH"]
+    maxTokens: int
+    exclude: bool
+
+
+class TextGenerationConfig(TypedDict, total=False):
+    """LLM generation parameters (camelCase on the wire, per Inworld)."""
+
+    reasoning: ReasoningConfig
+    maxNewTokens: int
+    temperature: float
+    topP: float
+    frequencyPenalty: float
+    presencePenalty: float
+    repetitionPenalty: float
+    stopSequences: list[str]
+    seed: int
+    logitBias: list[dict[str, Any]]
+
+
+class ProviderData(TypedDict, total=False):
+    """Root ``providerData`` object merged into the Inworld session config."""
+
+    auto_tool_response: bool
+    stt: STTProviderData
+    tts: TTSProviderData
+    memory: MemoryProviderData
+    backchannel: BackchannelProviderData
+    responsiveness: ResponsivenessProviderData
+    caching: CachingProviderData
+    text_generation_config: TextGenerationConfig
+    user_id: str
+    metadata: dict[str, str]