(expressive mode): register inference replaces presets

livekit-agents/livekit/agents/tts/_provider_format.py

@@ -11,6 +11,7 @@
 - Inworld: https://docs.inworld.ai/tts/best-practices/prompting-for-tts-2
 - xAI: https://docs.x.ai/developers/model-capabilities/audio/text-to-speech
 - xAI: https://docs.x.ai/developers/model-capabilities/audio/voice
+- Fish Audio: https://docs.fish.audio/developer-guide/core-features/emotions
 """
 
 from __future__ import annotations
@@ -118,6 +119,55 @@ def _xai_break_to_bracket(match: re.Match[str]) -> str:
     return "[long-pause]" if secs >= 1.0 else "[pause]"
 
 
+# Fish Audio (s2 family) speech markers, from the Fish docs
+# (https://docs.fish.audio/developer-guide/core-features/emotions).
+#
+# The LLM is instructed in the expr dialect (below); expr lowering produces the
+# framework-standard intermediates (<expression value="..."/>, <sound value="..."/>,
+# <break time="..."/>, <emphasis>word</emphasis>) and convert_markup rewrites them to
+# Fish's native square brackets: [very EMOTION], [SOUND], [break]/[long-break], and
+# [emphasis] word (a prefix marker stressing the word that follows). The tag names stay
+# in _FISHAUDIO_TAGS so hallucinated native markup is still stripped from transcripts.
+_FISHAUDIO_EMOTIONS = [
+    "regretful",
+    "hopeful",
+    "happy",
+    "excited",
+    "curious",
+    "surprised",
+    "sad",
+    "empathetic",
+    "sarcastic",
+]
+_FISHAUDIO_SOUNDS = ["laughing", "chuckling", "clear throat"]
+_FISHAUDIO_TAGS = ["expression", "sound", "break", "emphasis"]
+
+_FISHAUDIO_EXPRESSION_RE = re.compile(
+    r'<expression\s+value="([^"]*)"(?:\s*/>|>(?:.*?)</expression>)'
+)
+_FISHAUDIO_BREAK_RE = re.compile(r'<break\s+time="([^"]*)"\s*/?>')
+_FISHAUDIO_EMPHASIS_RE = re.compile(r"<emphasis(?:\s[^>]*)?>([^<]*)</emphasis\s*>", re.IGNORECASE)
+
+
+def _fishaudio_expression_to_bracket(match: re.Match[str]) -> str:
+    # intensify with a leading "very" so the emotion lands harder in Fish's audio
+    # ([very regretful] steers more strongly than [regretful]); never doubled
+    value = match.group(1).strip()
+    if value and not value.lower().startswith("very "):
+        value = f"very {value}"
+    return f"[{value}]"
+
+
+def _fishaudio_break_to_bracket(match: re.Match[str]) -> str:
+    # Fish has two pause levels ([break], [long-break]); use the longer past ~1s
+    raw = match.group(1).strip().lower()
+    try:
+        secs = float(raw[:-2]) / 1000 if raw.endswith("ms") else float(raw.rstrip("s"))
+    except ValueError:
+        secs = 0.0
+    return "[long-break]" if secs >= 1.0 else "[break]"
+
+
 # --- LiveKit expression markers (expr) ---
 # The LLM emits a single marker tag,
 # <expr type="..." label="..."/>, instead of provider-native tags. The *syntax* is shared,
@@ -156,7 +206,13 @@ def _xai_break_to_bracket(match: re.Match[str]) -> str:
 Just as important is knowing when NOT to reach for a marker. Reserve surprise openers \
 like "oh" or "ah" for genuine surprise — an ordinary request isn't one. Don't stack markers \
 on short replies or decorate every sentence. If a reaction wouldn't happen in a real \
-conversation, skip it — there's always another genuine beat to lean into."""
+conversation, skip it — there's always another genuine beat to lean into.
+
+Match your delivery to the REGISTER of the moment, and reassess every turn. When the \
+moment is professional, high-stakes, or emotionally heavy — bad news, an emergency, \
+real distress — keep delivery composed and restrained. When the moment is casual, \
+playful, or celebratory, let it loosen and brighten. A serious turn in an otherwise \
+casual conversation still gets a composed reply."""
 
 _CARTESIA_EXPR_LLM_INSTRUCTIONS = (
     _EXPR_PREAMBLE
@@ -344,32 +400,131 @@ def _xai_expr_llm_instructions(sounds: list[str], prosody: list[str]) -> str:
     return "\n\n".join(parts)
 
 
+# Examples carried over from the original Fish expressive block (PR #6232), rewritten
+# in the expr dialect. Breaks appear only mid-sentence, never beside a period/?/! —
+# an example pairing a break with sentence punctuation few-shots the LLM into
+# double-pausing every boundary.
+_FISHAUDIO_EXAMPLES = [
+    '<expr type="expression" label="excited"/> That\'s hilarious! <expr type="sound" label="laughing"/> <expr type="expression" label="happy"/> You always lighten the mood.',  # noqa: E501
+    '<expr type="expression" label="empathetic"/> <expr type="sound" label="clear throat"/> That sounds like a <expr type="prosody" label="emphasis">really</expr> difficult experience.',  # noqa: E501
+    '<expr type="expression" label="sad"/> Oh, my goodness <expr type="sound" label="clear throat"/> <expr type="break" label="2s"/> that\'s a real shame.',  # noqa: E501
+    # sound-free, so at least one example survives any steering filter
+    '<expr type="expression" label="happy"/> You\'re all set for <expr type="break" label="500ms"/> Thursday the <expr type="prosody" label="emphasis">ninth</expr>. <expr type="expression" label="curious"/> Is there anything else I can help you with?',  # noqa: E501
+]
+
+# The original block baked light disfluencies into the few-shots — that's what made
+# fillers actually show up in generations. Appended only while steering has
+# disfluencies enabled, so the examples never contradict the "no fillers" guideline.
+_FISHAUDIO_DISFLUENT_EXAMPLES = [
+    '<expr type="expression" label="curious"/> Um, uh... really? <expr type="expression" label="sad"/> Well, I\'m really sorry to hear that.',  # noqa: E501
+    '<expr type="expression" label="regretful"/> I really wish I\'d, um, called sooner. <expr type="expression" label="hopeful"/> But I\'m here now if, if you want to talk.',  # noqa: E501
+    '<expr type="expression" label="surprised"/> What?! No way! I, I\'m flabbergasted! <expr type="expression" label="sarcastic"/> Fair play, I guess.',  # noqa: E501
+]
+
+
+def _fishaudio_expr_llm_instructions(sounds: list[str], disfluencies: bool = True) -> str:
+    sections = [
+        f"""Emotion - sets how a sentence sounds. Self-closing; place at the START of a sentence.
+   <expr type="expression" label="EMOTION"/>
+   Labels are a fixed vocabulary, NOT free-form descriptions: {", ".join(_FISHAUDIO_EMOTIONS)}.
+   Give every sentence its own emotion marker — repeat the same label to carry a \
+feeling across sentences, or switch labels when the feeling shifts."""
+    ]
+    if sounds:
+        sections.append(
+            f"""Sounds - a non-verbal sound between sentences. Self-closing.
+   <expr type="sound" label="{sounds[0]}"/>
+   Labels are a fixed vocabulary: {", ".join(sounds)}.
+   Use non-verbal sounds sparingly, and never the same one twice in a row — reach for \
+one only where it genuinely fits. An enabled sound gets over-used otherwise."""
+        )
+    sections.append(
+        """Pauses - insert silence when appropriate. Self-closing.
+   <expr type="break" label="500ms"/> or <expr type="break" label="2s"/>.
+   NEVER place a break next to a period, question mark, exclamation point, or ellipsis \
+— sentence punctuation already pauses, and a break beside it double-pauses. Most \
+replies need no break markers at all; reserve them for a deliberate mid-sentence beat \
+before a key detail (a date, a name, a number)."""
+    )
+    sections.append(
+        """Emphasis - stresses exactly the ONE word it wraps.
+   Are you <expr type="prosody" label="emphasis">sure</expr> you want to do this?
+   Wrap a single word, never a phrase. "emphasis" is the only prosody label for this \
+voice — there are no other wrapping style markers. Never nest it, and always close it \
+with </expr>."""
+    )
+
+    parts = [
+        _EXPR_PREAMBLE,
+        _numbered_sections(sections),
+        """Write for the EAR, not the page: no em or en dashes anywhere in spoken text — \
+use a comma or a period for a short beat, or a break marker for a real pause. Avoid \
+semicolons, mid-sentence colons, and parenthetical asides; rewrite them as separate \
+sentences or commas.""",
+        """When the conversation is in another language, still write every marker label in \
+English — labels are a fixed vocabulary, never translated.""",
+    ]
+
+    # Vocabulary-specific register guidance on top of the preamble's neutral rule.
+    # Each clause mentions only concepts this steering actually enables, so an
+    # opted-out option is never referenced (not even prohibitively).
+    register = [
+        "At heavy moments reach for empathetic, sad, regretful, or hopeful — never a "
+        'bright label like "happy" or "excited" against hard news; bright labels belong '
+        "to bright moments."
+    ]
+    if any(s in sounds for s in ("laughing", "chuckling")):
+        register.append(
+            "Laughter belongs only in genuinely playful or celebratory beats, never at "
+            "a serious moment."
+        )
+    if disfluencies:
+        register.append(
+            "Save fillers for relaxed moments — never in an emergency or against grave news."
+        )
+    parts.append(" ".join(register))
+
+    pool = _FISHAUDIO_EXAMPLES + (_FISHAUDIO_DISFLUENT_EXAMPLES if disfluencies else [])
+    if examples := _sound_examples(pool, sounds, _FISHAUDIO_SOUNDS):
+        parts.append("Examples:\n" + "\n".join(f"  {ex}" for ex in examples))
+    return "\n\n".join(parts)
+
+
 # Every provider's full expr sound vocabulary (the advertised labels before any
 # speech_steering filtering). Providers absent here have no non-verbal sounds.
 _PROVIDER_SOUNDS: dict[str, list[str]] = {
     "inworld": _INWORLD_SOUNDS,
     "xai": _XAI_INLINE,
+    "fishaudio": _FISHAUDIO_SOUNDS,
 }
 
 
 def _steering_removed(
     table: dict[str, dict[str, list[str]]], provider: str, steering: SpeechSteeringOptions | None
 ) -> set[str]:
-    """Labels from a per-provider governance table that *steering* disables."""
+    """Labels from a per-provider governance table that *steering* disables.
+
+    ``nonverbal_sounds`` accepts a bool or a sparse per-category dict:
+    ``True`` (like omitting the key) keeps the full vocabulary, ``False``
+    disables every sound, and in a dict an omitted category stays ENABLED —
+    ``{"laughing": False}`` removes laughter and nothing else.
+    """
     nonverbals = steering.get("nonverbal_sounds") if steering else None
     labels = table.get(provider)
-    if nonverbals is None or labels is None:
+    if nonverbals is None or nonverbals is True or labels is None:
         return set()
+    if nonverbals is False:
+        return {lb for lbs in labels.values() for lb in lbs}
     flags = dict(nonverbals)
-    return {lb for f, lbs in labels.items() if not flags.get(f, False) for lb in lbs}
+    return {lb for f, lbs in labels.items() if not flags.get(f, True) for lb in lbs}
 
 
 def _allowed_sounds(provider: str, steering: SpeechSteeringOptions | None) -> list[str]:
     """The provider's sound vocabulary minus labels steering disables.
 
     Every label is governed by a ``NonverbalOptions`` field, so passing
-    ``nonverbal_sounds`` with everything off returns an empty list — the
-    instruction builders then omit the Sounds section entirely.
+    ``nonverbal_sounds=False`` returns an empty list — the instruction
+    builders then omit the Sounds section entirely.
     """
     removed = _steering_removed(_NONVERBAL_SOUND_LABELS, provider, steering)
     return [s for s in _PROVIDER_SOUNDS.get(provider, []) if s not in removed]
@@ -391,8 +546,8 @@ def _allowed_prosody(provider: str, steering: SpeechSteeringOptions | None) -> l
 # has no sound for that field (nothing to filter). _allowed_sounds uses this to
 # remove disabled labels from the advertised vocabulary, so a sound steering turns
 # off is never exposed to the LLM in the first place. Every label in
-# _PROVIDER_SOUNDS must be governed by exactly one field, so a preset controls
-# the full vocabulary.
+# _PROVIDER_SOUNDS must be governed by exactly one field, so a steering config
+# controls the full vocabulary.
 _NONVERBAL_SOUND_LABELS: dict[str, dict[str, list[str]]] = {
     "inworld": {
         "laughing": ["laugh"],
@@ -412,6 +567,15 @@ def _allowed_prosody(provider: str, steering: SpeechSteeringOptions | None) -> l
         "mouth_sounds": ["tsk", "tongue-click", "lip-smack"],
         "reflex_sounds": [],  # xAI has no cough/yawn sounds
     },
+    "fishaudio": {
+        "laughing": ["laughing", "chuckling"],
+        "breathing": [],
+        "sighing": [],
+        "crying": [],
+        "vocalizing": [],
+        "mouth_sounds": [],
+        "reflex_sounds": ["clear throat"],
+    },
 }
 
 # NonverbalOptions field -> the provider's wrapping-prosody labels it governs.
@@ -447,13 +611,16 @@ def supported_nonverbals(provider: str) -> dict[str, list[str]]:
 # sentence. Keyed by label, not NonverbalOptions field, so it's provider-agnostic.
 _SOUND_USAGE_HINTS: dict[str, str] = {
     "laugh": "a laugh at something obviously funny",
+    "laughing": "a laugh at something obviously funny",
     "chuckle": "a chuckle at something subtly humorous",
+    "chuckling": "a chuckle at something subtly humorous",
     "giggle": "a chuckle at something subtly humorous",
     "sigh": "a sigh when commiserating",
     "inhale": "a sharp inhale before a big reveal",
     "lip-smack": "a lip-smack or tongue-click as a tiny beat of thought",
     "tongue-click": "a lip-smack or tongue-click as a tiny beat of thought",
     "tsk": "a tsk for mock-disapproval",
+    "clear throat": "a clear-throat when shifting to a new step or topic",
 }
 
 
@@ -511,6 +678,9 @@ def steering_instructions(provider: str, steering: SpeechSteeringOptions) -> str
     # well under xAI's 15,000-char request limit; sized as an expressive batch
     # target (https://docs.x.ai/developers/model-capabilities/audio/text-to-speech)
     "xai": 1000,
+    # fishaudio is deliberately absent: its markers are sentence-scoped (every
+    # sentence carries its own [very EMOTION]), so per-sentence emission loses no
+    # steering and keeps time-to-first-audio low
 }
 
 
@@ -558,6 +728,11 @@ def sentence_tokenizer(provider: str, *, expressive: bool) -> tokenize.SentenceT
 # expr sound labels that differ from xAI's native cue names
 _XAI_SOUND_ALIASES = {"breathe": "breath"}
 
+# expr sound labels that differ from Fish's native marker names (other providers
+# advertise "laugh"/"chuckle", so a hallucinated one still lowers to a sound Fish
+# renders)
+_FISHAUDIO_SOUND_ALIASES = {"laugh": "laughing", "chuckle": "chuckling"}
+
 # Cartesia prosody labels -> native point controls (coarse steps of the numeric ratios)
 _CARTESIA_PROSODY = {
     "slow": '<speed ratio="0.85"/>',
@@ -626,6 +801,9 @@ def _wrap(m: re.Match[str]) -> str:
         if provider == "cartesia":
             # wrapping form of the point controls: apply before the span
             return _CARTESIA_PROSODY.get(label, "") + inner
+        if provider == "fishaudio":
+            # emphasis is Fish's only wrapping control; other labels are dropped
+            return f"<emphasis>{inner}</emphasis>" if label == "emphasis" else inner
         return inner
 
     text = _EXPR_WRAP_RE.sub(_wrap, text)
@@ -638,14 +816,16 @@ def _self(m: re.Match[str]) -> str:
             if provider == "cartesia":
                 # Cartesia's discrete emotion vocabulary (instructions list it)
                 return f'<emotion value="{label}"/>'
-            if provider == "inworld":
+            if provider in ("inworld", "fishaudio"):
                 return f'<expression value="{label}"/>'
             return ""  # xAI has no free-form delivery descriptions
         if marker_type == "sound":
             if provider == "cartesia":
                 return ""  # no non-verbal sound support
             if provider == "xai":
                 label = _XAI_SOUND_ALIASES.get(label.lower(), label)
+            if provider == "fishaudio":
+                label = _FISHAUDIO_SOUND_ALIASES.get(label.lower(), label)
             return f'<sound value="{label}"/>'
         if marker_type == "break":
             return f'<break time="{label}"/>'
@@ -680,6 +860,11 @@ def llm_instructions(provider: str, steering: SpeechSteeringOptions | None = Non
         return _xai_expr_llm_instructions(
             _allowed_sounds(provider, steering), _allowed_prosody(provider, steering)
         )
+    if provider == "fishaudio":
+        return _fishaudio_expr_llm_instructions(
+            _allowed_sounds(provider, steering),
+            disfluencies=steering.get("disfluencies", True) if steering else True,
+        )
     return None
 
 
@@ -690,6 +875,9 @@ def llm_instructions(provider: str, steering: SpeechSteeringOptions | None = Non
     # every tag the LLM is taught is XML (expr markers; native sounds/pauses become
     # [..] only for the TTS in convert_markup), so the transcript has no brackets to strip
     "xai": (_XAI_TAGS, False),
+    # brackets are stripped too: Fish's native dialect IS square brackets, so a
+    # hallucinated [very happy] must not leak into the user-visible transcript
+    "fishaudio": (_FISHAUDIO_TAGS, True),
 }
 
 
@@ -829,6 +1017,7 @@ def expression_attribute(self) -> dict[str, str] | None:
 _SELF_CLOSING_TAGS: dict[str, list[str]] = {
     "cartesia": ["emotion", "speed", "volume", "break"],
     "inworld": ["expression", "sound", "break"],
+    "fishaudio": ["expression", "sound", "break"],
 }
 
 
@@ -861,5 +1050,13 @@ def convert_markup(provider: str, text: str) -> str:
     if provider == "xai":
         # xAI has no <break>; map it to its native [pause]/[long-pause]
         text = _XAI_BREAK_RE.sub(_xai_break_to_bracket, text)
+    if provider == "fishaudio":
+        # <expression value="X"/> -> [very X] first (the intensified form steers
+        # harder), then the generic pass lowers the remaining <sound value="X"/> -> [X]
+        text = _FISHAUDIO_EXPRESSION_RE.sub(_fishaudio_expression_to_bracket, text)
+        text = convert_expression_tags(text)
+        text = _FISHAUDIO_BREAK_RE.sub(_fishaudio_break_to_bracket, text)
+        # Fish's per-word stress marker: <emphasis>word</emphasis> -> [emphasis] word
+        text = _FISHAUDIO_EMPHASIS_RE.sub(lambda m: f"[emphasis] {m.group(1).strip()}", text)
     # <break> is otherwise passed through unchanged: Inworld accepts it as native SSML.
     return text

livekit-agents/livekit/agents/voice/__init__.py

@@ -1,11 +1,10 @@
-from . import io, presets, run_result
+from . import io, run_result
 from .agent import Agent, AgentTask, ModelSettings
 from .agent_session import (
     AgentSession,
     NonverbalOptions,
     RecordingOptions,
     SpeechSteeringOptions,
-    SpeechSteeringPreset,
     VoiceActivityVideoSampler,
 )
 from .audio_recognition import AudioRecognition
@@ -47,7 +46,6 @@
     "NonverbalOptions",
     "RecordingOptions",
     "SpeechSteeringOptions",
-    "SpeechSteeringPreset",
     "RunOutputOptions",
     "VoiceActivityVideoSampler",
     "Agent",
@@ -79,7 +77,6 @@
     "KeytermDetectionOptions",
     "TranscriptSynchronizer",
     "io",
-    "presets",
     "room_io",
     "run_result",
     "_ParticipantAudioOutput",