livekit · chenghao-mou · Sep 14, 2026 · Sep 14, 2026
@@ -85,6 +85,19 @@ def recording_enabled(options: Mapping[str, object]) -> bool:
 The key for the timed transcripts in the audio frame userdata.
 """
 
+USERDATA_AUDIO_RAW = "lk.audio.raw"
+"""Optional ``rtc.AudioFrame`` containing the audio immediately before processing.
+
+The raw frame owns its sample buffer and covers the same source interval as the
+processed frame. Consumers select their target frame before resampling.
+"""
+
+USERDATA_AUDIO_PROCESSING = "lk.audio.processing"
+"""Optional processing label: ``"denoised"`` for NC or ``"isolated"`` for VF.
+
+The processed PCM remains in the containing audio frame's data.
+"""
+
 USERDATA_TTS_STARTED_TIME = "lk.tts_started_time"
 """
 The key for the time (``time.perf_counter()``) at which the synthesized text was first

@@ -417,7 +417,7 @@ async def _forward_task(
                     logging_extra["track_id"]
                 )
                 self._pre_connect_audio_publications.add(pre_connect_key)
-                for frame in self._resample_frames(self._apply_audio_processor(frames)):
+                for frame in self._apply_audio_processor(self._resample_frames(frames)):
                     if self._attached:
                         await self._data_ch.send(frame)
                         duration += frame.duration

@@ -7,6 +7,25 @@ Real-time noise reduction for LiveKit voice agents using [Krisp's VIVA SDK](http
 - **`voice_isolation()`**: Real-time voice isolation and noise reduction `FrameProcessor`
 - **`voice_isolation_telephony()`**: Voice isolation tuned for telephony audio (for example, SIP participants)
 
+## Input audio metadata
+
+Processed frames expose an aligned copy of the audio immediately before filtering
+in `frame.userdata["lk.audio.raw"]`. This value is an `rtc.AudioFrame` with its own
+sample buffer. The filtered PCM stays in `frame.data`.
+
+`frame.userdata["lk.audio.processing"]` is `"isolated"` for the LiveKit Cloud voice
+isolation backend, or `"denoised"` for the license backend's NC session. Both fields
+are optional; frames passed through without processing do not add them.
+
+Select the desired frame at agent input before downstream resampling or batching:
+
+```python
+raw_frame = frame.userdata.get("lk.audio.raw", frame)
+```
+
+Other `FrameProcessor` plugins can expose the same userdata keys. Legacy native
+`NoiseCancellationOptions` filters do not expose raw audio through this contract.
+
 ## Installation
 
 ```bash

@@ -28,6 +28,7 @@
 import numpy as np
 
 from livekit import rtc
+from livekit.agents.types import USERDATA_AUDIO_PROCESSING, USERDATA_AUDIO_RAW
 
 from .log import logger
 
@@ -180,6 +181,7 @@ def __init__(
         # interrupted by injected silence.
         self._in_buf: np.ndarray = np.empty(0, dtype=np.int16)
         self._out_buf: np.ndarray = np.empty(0, dtype=np.int16)
+        self._raw_out_buf: np.ndarray = np.empty(0, dtype=np.int16)
 
         try:
             self._module = _KrispLicenseSDKManager.acquire(license_key)
@@ -241,6 +243,7 @@ def _create_session(self, sample_rate: int) -> None:
             # The pending/processed buffers belong to the old rate; start fresh.
             self._in_buf = np.empty(0, dtype=np.int16)
             self._out_buf = np.empty(0, dtype=np.int16)
+            self._raw_out_buf = np.empty(0, dtype=np.int16)
             logger.info("Krisp session created successfully")
         except Exception as e:
             logger.error(f"Failed to create Krisp session: {e}")
@@ -278,6 +281,7 @@ def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
             consumed = n_chunks * chunk
             pending = self._in_buf[:consumed]
             self._in_buf = self._in_buf[consumed:].copy()
+            self._raw_out_buf = np.concatenate((self._raw_out_buf, pending))
 
             processed: list[np.ndarray] = []
             for i in range(n_chunks):
@@ -306,12 +310,25 @@ def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
         k = min(n, len(self._out_buf))
         out = self._out_buf[:k]
         self._out_buf = self._out_buf[k:].copy()
+        raw = self._raw_out_buf[:k]
+        self._raw_out_buf = self._raw_out_buf[k:].copy()
+
+        userdata = frame.userdata.copy()
+        if k:
+            userdata[USERDATA_AUDIO_RAW] = rtc.AudioFrame(
+                data=raw.tobytes(),
+                sample_rate=frame.sample_rate,
+                num_channels=frame.num_channels,
+                samples_per_channel=k,
+            )
+            userdata[USERDATA_AUDIO_PROCESSING] = "denoised"
 
         return rtc.AudioFrame(
             data=out.tobytes(),
             sample_rate=frame.sample_rate,
             num_channels=frame.num_channels,
             samples_per_channel=len(out),
+            userdata=userdata,
         )
 
     @property
@@ -340,6 +357,7 @@ def _close(self) -> None:
             self._session = None
         self._in_buf = np.empty(0, dtype=np.int16)
         self._out_buf = np.empty(0, dtype=np.int16)
+        self._raw_out_buf = np.empty(0, dtype=np.int16)
         logger.debug("Krisp frame processor session closed")
 
     def __del__(self) -> None:

@@ -34,6 +34,7 @@
 from typing import Any, Literal, Protocol
 
 from livekit import rtc
+from livekit.agents.types import USERDATA_AUDIO_PROCESSING, USERDATA_AUDIO_RAW
 from livekit.plugins.krisp_internal import VivaMode
 
 from .auth import KrispLicenseAuthProvider, LiveKitCloudAuthProvider
@@ -249,6 +250,7 @@ def __init__(
                 _FRAME_PARAMS_DEPRECATION_SHOWN = True
 
         provider = _resolve_auth_provider(auth_provider, model_path)
+        self._is_cloud = isinstance(provider, LiveKitCloudAuthProvider)
         self._inner = _build_inner(
             mode,
             provider,
@@ -260,7 +262,31 @@ def __init__(
     # ----- FrameProcessor hooks: forward to the inner processor -------------
 
     def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
-        return self._inner._process(frame)
+        # The license backend pairs its raw samples with its buffered output.
+        if not self._is_cloud or not self.enabled or frame.num_channels != 1:
+            return self._inner._process(frame)
+
+        raw = rtc.AudioFrame(
+            data=bytearray(frame.data),
+            sample_rate=frame.sample_rate,
+            num_channels=frame.num_channels,
+            samples_per_channel=frame.samples_per_channel,
+        )
+        processed = self._inner._process(frame)
+        if processed is frame:
+            return frame
+
+        return rtc.AudioFrame(
+            data=processed.data,
+            sample_rate=processed.sample_rate,
+            num_channels=processed.num_channels,
+            samples_per_channel=processed.samples_per_channel,
+            userdata={
+                **processed.userdata,
+                USERDATA_AUDIO_RAW: raw,
+                USERDATA_AUDIO_PROCESSING: "isolated",
+            },
+        )
 
     def _on_credentials_updated(self, *, token: str, url: str) -> None:
         self._inner._on_credentials_updated(token=token, url=url)

@@ -21,7 +21,10 @@
 import pytest
 
 from livekit import rtc
+from livekit.agents.types import USERDATA_AUDIO_PROCESSING, USERDATA_AUDIO_RAW
+from livekit.plugins import krisp
 from livekit.plugins.krisp._krisp import _KrispLicenseFrameProcessor
+from livekit.plugins.krisp.viva_filter import VivaMode
 
 pytestmark = pytest.mark.unit
 
@@ -50,6 +53,7 @@ def _make_processor(sample_rate: int, chunk_samples: int) -> _KrispLicenseFrameP
     proc._frame_duration_ms = int(chunk_samples * 1000 / sample_rate)
     proc._in_buf = np.empty(0, dtype=np.int16)
     proc._out_buf = np.empty(0, dtype=np.int16)
+    proc._raw_out_buf = np.empty(0, dtype=np.int16)
     return proc
 
 
@@ -119,3 +123,109 @@ def test_frame_equals_chunk_is_exact_passthrough() -> None:
     proc = _make_processor(sample_rate, chunk)
     in_stream, out_stream = _feed(proc, sample_rate, [chunk] * 20)
     assert np.array_equal(out_stream, in_stream)
+
+
+@pytest.mark.parametrize("sizes", [[100] * 40, [137, 53, 200, 80, 160, 45, 300, 10, 90] * 4])
+def test_raw_audio_matches_buffered_nc_output(sizes: list[int]) -> None:
+    class InPlaceSession:
+        def process(self, chunk_in: np.ndarray, level: int) -> np.ndarray:
+            chunk_in *= -1
+            return chunk_in
+
+    proc = _make_processor(16000, 160)
+    proc._session = InPlaceSession()
+    inputs: list[np.ndarray] = []
+    raw_outputs: list[np.ndarray] = []
+    count = 1
+    for size in sizes:
+        samples = np.arange(count, count + size, dtype=np.int16)
+        count += size
+        inputs.append(samples)
+        frame = rtc.AudioFrame(bytearray(samples), 16000, 1, size, userdata={"custom": "value"})
+        output = proc._process(frame)
+        assert output.userdata["custom"] == "value"
+        assert frame.userdata == {"custom": "value"}
+        if not output.samples_per_channel:
+            assert USERDATA_AUDIO_RAW not in output.userdata
+            assert USERDATA_AUDIO_PROCESSING not in output.userdata
+            continue
+
+        raw = output.userdata[USERDATA_AUDIO_RAW]
+        assert isinstance(raw, rtc.AudioFrame)
+        assert raw.sample_rate == output.sample_rate
+        assert raw.num_channels == output.num_channels
+        assert raw.samples_per_channel == output.samples_per_channel
+        assert output.userdata[USERDATA_AUDIO_PROCESSING] == "denoised"
+        raw_samples = np.frombuffer(raw.data, dtype=np.int16).copy()
+        np.testing.assert_array_equal(raw_samples, -np.frombuffer(output.data, dtype=np.int16))
+        raw_outputs.append(raw_samples)
+        frame.data[0] = 0
+        np.testing.assert_array_equal(np.frombuffer(raw.data, dtype=np.int16), raw_samples)
+
+    raw_stream = np.concatenate(raw_outputs)
+    np.testing.assert_array_equal(raw_stream, np.concatenate(inputs)[: len(raw_stream)])
+    proc._close()
+    assert len(proc._raw_out_buf) == 0
+
+
+@pytest.mark.parametrize("enabled,channels", [(False, 1), (True, 2)])
+def test_nc_passthrough_does_not_add_metadata(enabled: bool, channels: int) -> None:
+    proc = _make_processor(16000, 160)
+    proc.enabled = enabled
+    frame = rtc.AudioFrame.create(16000, channels, 160)
+    assert proc._process(frame) is frame
+    assert frame.userdata == {}
+
+
+@pytest.mark.parametrize("mode", list(VivaMode))
+def test_vf_preserves_raw_before_in_place_processing(
+    monkeypatch: pytest.MonkeyPatch, mode: VivaMode
+) -> None:
+    class Backend:
+        enabled = True
+
+        def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
+            frame.data[0] = 99
+            return rtc.AudioFrame(
+                frame.data,
+                frame.sample_rate,
+                frame.num_channels,
+                frame.samples_per_channel,
+                userdata=frame.userdata,
+            )
+
+    monkeypatch.setattr(krisp.viva_filter, "_build_inner", lambda *args, **kwargs: Backend())
+    proc = krisp.KrispVivaFilterFrameProcessor(mode=mode, auth_provider=krisp.auth.livekit_cloud())
+    frame = rtc.AudioFrame(
+        bytearray(b"\x01\x00" * 160), 16000, 1, 160, userdata={"custom": "value"}
+    )
+    output = proc._process(frame)
+
+    assert output.data[0] == 99
+    assert output.userdata[USERDATA_AUDIO_PROCESSING] == "isolated"
+    assert frame.userdata == {"custom": "value"}
+    assert output.userdata["custom"] == "value"
+    raw = output.userdata[USERDATA_AUDIO_RAW]
+    assert raw.data[0] == 1
+    assert raw.samples_per_channel == output.samples_per_channel
+    assert raw.userdata == {}
+    output.data[0] = 0
+    assert raw.data[0] == 1
+
+
+@pytest.mark.parametrize("enabled", [False, True])
+def test_vf_passthrough_does_not_add_metadata(
+    monkeypatch: pytest.MonkeyPatch, enabled: bool
+) -> None:
+    class Backend:
+        def __init__(self) -> None:
+            self.enabled = enabled
+
+        def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
+            return frame
+
+    monkeypatch.setattr(krisp.viva_filter, "_build_inner", lambda *args, **kwargs: Backend())
+    proc = krisp.KrispVivaFilterFrameProcessor(auth_provider=krisp.auth.livekit_cloud())
+    frame = rtc.AudioFrame.create(16000, 1, 160)
+    assert proc._process(frame) is frame
+    assert frame.userdata == {}
@@ -9,6 +9,7 @@
 
 from livekit import rtc
 from livekit.agents import NOT_GIVEN, utils
+from livekit.agents.types import USERDATA_AUDIO_PROCESSING, USERDATA_AUDIO_RAW
 from livekit.agents.voice.io import PlaybackFinishedEvent
 from livekit.agents.voice.room_io._input import (
     _ParticipantAudioInputStream,
@@ -507,6 +508,50 @@ async def test_audio_input_closes_active_track_on_unsubscribe() -> None:
     await audio_input.aclose()
 
 
+@pytest.mark.asyncio
+@pytest.mark.parametrize("source_rate", [16000, 24000])
+async def test_pre_connect_audio_preserves_processing_metadata_at_input(source_rate: int) -> None:
+    class Processor(_MockFrameProcessor):
+        def _process(self, frame: rtc.AudioFrame) -> rtc.AudioFrame:
+            assert frame.sample_rate == 24000
+            return rtc.AudioFrame(
+                frame.data,
+                frame.sample_rate,
+                frame.num_channels,
+                frame.samples_per_channel,
+                userdata={USERDATA_AUDIO_RAW: frame, USERDATA_AUDIO_PROCESSING: "isolated"},
+            )
+
+    frames = [rtc.AudioFrame.create(source_rate, 1, source_rate // 10)]
+    audio_input = _ParticipantAudioInputStream(
+        _FakeRoom(),
+        sample_rate=24000,
+        num_channels=1,
+        noise_cancellation=Processor(),
+        auto_gain_control=False,
+        pre_connect_audio_handler=SimpleNamespace(wait_for_data=AsyncMock(return_value=frames)),
+    )
+    audio_input.set_participant("test-user")
+    track, publication, participant = _make_track_available_args()
+    publication.audio_features = [AudioTrackFeature.TF_PRECONNECT_BUFFER]
+    stream = _MockAudioStream()
+    try:
+        with patch("livekit.rtc.AudioStream.from_track", return_value=stream):
+            assert audio_input._on_track_available(track, publication, participant)
+            await asyncio.wait_for(stream.started.wait(), timeout=1)
+
+        output_frames = [await audio_input.__anext__() for _ in range(audio_input._data_ch.qsize())]
+        assert output_frames
+        assert sum(frame.duration for frame in output_frames) == pytest.approx(0.1, abs=0.001)
+        for frame in output_frames:
+            raw = frame.userdata[USERDATA_AUDIO_RAW]
+            assert frame.userdata[USERDATA_AUDIO_PROCESSING] == "isolated"
+            assert frame.sample_rate == raw.sample_rate == 24000
+            assert frame.samples_per_channel == raw.samples_per_channel
+    finally:
+        await audio_input.aclose()
+
+
 @pytest.mark.asyncio
 async def test_pre_connect_audio_runs_once_across_concrete_track_replacement() -> None:
     room = _FakeRoom()