Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/test-python-websocket.yml
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@ on:
- "tests/test_websocket_server_hotwords.py"
- "tests/test_websocket_file_finalization.py"
- "tests/test_websocket_client_hotwords.py"
- "tests/test_websocket_server_audio_fs.py"
- ".github/workflows/test-python-websocket.yml"
push:
branches:
Expand All @@ -20,6 +21,7 @@ on:
- "tests/test_websocket_server_hotwords.py"
- "tests/test_websocket_file_finalization.py"
- "tests/test_websocket_client_hotwords.py"
- "tests/test_websocket_server_audio_fs.py"
- ".github/workflows/test-python-websocket.yml"

permissions:
Expand All @@ -42,13 +44,14 @@ jobs:
run: |
python -m pip install pytest numpy scipy websockets
python -m pip install torch --index-url https://download.pytorch.org/whl/cpu
- name: Run constructor, hotword, client hotword and file-finalization regressions
- name: Run constructor, hotword, client hotword, file-finalization and sample-rate regressions
run: >-
python -m pytest -q
tests/test_websocket_server_asr_model.py
tests/test_websocket_server_hotwords.py
tests/test_websocket_file_finalization.py
tests/test_websocket_client_hotwords.py
tests/test_websocket_server_audio_fs.py
--junitxml=websocket-test-results.xml
- uses: actions/upload-artifact@v4
if: always()
Expand Down
37 changes: 35 additions & 2 deletions runtime/python/websocket/funasr_wss_server.py
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,23 @@ def _safe_int(v, default):
return default


# Sample rates the models are told about. AutoModel resamples PCM input from fs to
# the model's rate; from these rates that is cheap, while an arbitrary value (0, a
# negative or a rate such as 44101) would fail or build a very large resampling
# kernel on every call. Any other audio_fs keeps the models at 16000 Hz, as before.
MODEL_AUDIO_FS = (8000, 11025, 12000, 16000, 22050, 24000, 32000, 44100, 48000)


def _model_audio_fs(audio_fs):
if audio_fs in MODEL_AUDIO_FS:
return audio_fs
print(
f"audio_fs {audio_fs} is not a supported sample rate; the models read the audio as 16000 Hz",
flush=True,
)
return 16000


def _model_hotwords(hotwords):
"""Turn protocol hotwords into the space-separated string AutoModel reads.

Expand Down Expand Up @@ -372,6 +389,15 @@ async def ws_serve(websocket, path=None):
websocket.status_dict_asr = {} # hotword 等
websocket.status_dict_asr_online = {"cache": {}, "is_final": False}
websocket.status_dict_vad = {"cache": {}, "is_final": False}
# Every call names its sample rate, so one connection never reads its audio at
# another connection's rate through the shared models' settings.
websocket.model_audio_fs = 16000
for status in (
websocket.status_dict_asr,
websocket.status_dict_asr_online,
websocket.status_dict_vad,
):
status["fs"] = websocket.model_audio_fs
websocket.status_dict_punc = {"cache": {}}

websocket.chunk_interval = 10
Expand Down Expand Up @@ -536,6 +562,12 @@ async def finish_input(send_end_ack):

if "audio_fs" in messagejson:
websocket.audio_fs = _safe_int(messagejson["audio_fs"], 16000)
# AutoModel reads PCM input at fs (16000 by default) and resamples it
# to the model's rate, as the C++ servers do with audio_fs.
websocket.model_audio_fs = _model_audio_fs(websocket.audio_fs)
websocket.status_dict_vad["fs"] = websocket.model_audio_fs
websocket.status_dict_asr_online["fs"] = websocket.model_audio_fs
websocket.status_dict_asr["fs"] = websocket.model_audio_fs

if end_of_input:
await finish_input(send_end_ack=bool(messagejson.get("is_end")))
Expand Down Expand Up @@ -692,15 +724,15 @@ async def async_vad(websocket, audio_in: bytes):
return speech_start, speech_end


def _sv_and_match_sync(audio_in: bytes, reload_sec: int):
def _sv_and_match_sync(audio_in: bytes, reload_sec: int, fs: int = 16000):
"""
同步执行:SV embedding + speaker_db 匹配
返回 (spk_name, best_score)
"""
spk_name = "unknown"
best_score = 0.0

sv_out = model_sv.generate(input=audio_in, embedding=True)[0]
sv_out = model_sv.generate(input=audio_in, embedding=True, fs=fs)[0]
embedding = sv_out["spk_embedding"][0].cpu().numpy()

now_ts = time.time()
Expand Down Expand Up @@ -757,6 +789,7 @@ async def async_asr(websocket, audio_in: bytes):
_sv_and_match_sync,
audio_in,
int(args.speaker_db_reload_sec),
websocket.model_audio_fs,
sem=SEM_SV,
)
except Exception as e:
Expand Down
1 change: 1 addition & 0 deletions tests/test_websocket_file_finalization.py
Original file line number Diff line number Diff line change
Expand Up @@ -139,6 +139,7 @@ async def ws_reset(websocket):
"_safe_int": lambda value, default: (
int(value) if value is not None else default
),
"_model_audio_fs": lambda audio_fs: audio_fs,
"_pcm_duration_ms": lambda pcm, fs, ch, sampwidth: 10,
"async_vad": async_vad,
"async_asr": async_asr,
Expand Down
176 changes: 176 additions & 0 deletions tests/test_websocket_server_audio_fs.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
"""Client sample rate in runtime/python/websocket/funasr_wss_server.py.

The websocket protocol (runtime/docs/websocket_protocol.md) sends ``audio_fs``
with PCM audio, and its real-time recognition takes 8000 and 16000 Hz.
funasr_wss_client.py sends the sample rate of its WAV file or its --audio_fs,
and the HTML5 client the sample rate of its WAV file. The C++ servers resample
that audio to the model's rate. The Python server used ``audio_fs`` only to time
the frames, and gave AutoModel the PCM frames without it, so its models read
8 kHz audio as 16 kHz audio. AutoModel reads PCM input at the ``fs`` it is given
(16000 by default) and resamples it to the model's rate.
"""

import asyncio
import importlib.util
import json
import sys
import types
from pathlib import Path
from types import SimpleNamespace

import pytest

ROOT = Path(__file__).resolve().parents[1]
SERVER_PATH = ROOT / "runtime/python/websocket/funasr_wss_server.py"


class FakeWebSocket:
def __init__(self, incoming):
self.incoming = iter(incoming)
self.sent = []
self.path = None

def __aiter__(self):
return self

async def __anext__(self):
try:
return next(self.incoming)
except StopIteration:
raise StopAsyncIteration

async def send(self, message):
self.sent.append(message)

async def close(self):
pass


@pytest.fixture
def server(monkeypatch):
for name in ("websockets", "scipy", "torch"):
pytest.importorskip(name)
# The server builds its models with AutoModel when it is imported.
fake_funasr = types.ModuleType("funasr")
fake_funasr.AutoModel = lambda **kwargs: SimpleNamespace(**kwargs)
monkeypatch.setitem(sys.modules, "funasr", fake_funasr)
monkeypatch.setattr(sys, "argv", [str(SERVER_PATH)])
spec = importlib.util.spec_from_file_location("funasr_wss_server_audio_fs", SERVER_PATH)
module = importlib.util.module_from_spec(spec)
spec.loader.exec_module(module)
yield module
module.EXECUTOR.shutdown(wait=True)


def _model_rates(server, audio_fs):
"""Stream 1.2 s of audio in 2pass mode and return the rate each model reads it at."""
rates = {"vad": [], "online": [], "offline": [], "speaker": []}

def model(name, result):
def generate(input, **kwargs):
# AutoModel reads PCM input at 16000 Hz unless it is given fs.
rates[name].append(kwargs.get("fs", 16000))
return [result]

return SimpleNamespace(generate=generate)

embedding = SimpleNamespace(cpu=lambda: SimpleNamespace(numpy=lambda: None))
server.model_vad = model("vad", {"value": []})
server.model_asr_streaming = model("online", {"text": "online"})
server.model_asr = model("offline", {"text": "offline"})
server.model_sv = model("speaker", {"spk_embedding": [embedding]})
server.model_punc = None
config = {
"mode": "2pass",
"chunk_size": [5, 10, 5],
"chunk_interval": 10,
"wav_name": "demo",
"is_speaking": True,
}
if audio_fs is not None:
config["audio_fs"] = audio_fs
# 60 ms of PCM16 per frame, as funasr_wss_client.py sends with these options.
frame_rate = audio_fs if isinstance(audio_fs, int) and audio_fs > 0 else 16000
frame = b"\x01\x00" * (frame_rate * 60 // 1000)
websocket = FakeWebSocket(
[
json.dumps(config),
*[frame] * 20,
json.dumps({"is_speaking": False, "is_end": True}),
]
)

asyncio.run(server.ws_serve(websocket))

assert json.loads(websocket.sent[-1])["is_final"] is True
return rates


@pytest.mark.parametrize(
"audio_fs, expected",
[
# funasr_wss_client.py --audio_in with an 8 kHz WAV file, or --audio_fs 8000
(8000, 8000),
(16000, 16000),
# The microphone mode of funasr_wss_client.py sends 16 kHz audio without audio_fs.
(None, 16000),
],
)
def test_models_read_pcm_at_the_client_sample_rate(server, audio_fs, expected):
rates = _model_rates(server, audio_fs)
# VAD and online ASR read every frame; offline ASR and the speaker model the whole input.
assert all(rates.values())
assert {name: set(calls) for name, calls in rates.items()} == {
name: {expected} for name in rates
}


@pytest.mark.parametrize("audio_fs", [44100, 48000, 22050])
def test_wav_sample_rates_reach_the_models(server, audio_fs):
# The bundled clients send the rate of the WAV file they stream.
rates = _model_rates(server, audio_fs)
assert {name: set(calls) for name, calls in rates.items()} == {
name: {audio_fs} for name in rates
}


@pytest.mark.parametrize("audio_fs", [0, -8000, 44101, "8k"])
def test_unsupported_audio_fs_keeps_the_models_at_16000(server, audio_fs):
# 0 or a negative rate would make every model call fail, and a rate such as
# 44101 would make the resampler build a very large kernel on every call.
rates = _model_rates(server, audio_fs)
assert all(rates.values())
assert {name: set(calls) for name, calls in rates.items()} == {name: {16000} for name in rates}


def test_every_model_call_names_its_sample_rate(server):
# A connection without audio_fs still passes fs, so it never reads its audio at
# another connection's rate through the shared models' settings.
seen = []

def model(result):
def generate(input, **kwargs):
seen.append("fs" in kwargs)
return [result]

return SimpleNamespace(generate=generate)

embedding = SimpleNamespace(cpu=lambda: SimpleNamespace(numpy=lambda: None))
server.model_vad = model({"value": []})
server.model_asr_streaming = model({"text": "online"})
server.model_asr = model({"text": "offline"})
server.model_sv = model({"spk_embedding": [embedding]})
server.model_punc = None
config = {
"mode": "2pass",
"chunk_size": [5, 10, 5],
"chunk_interval": 10,
"wav_name": "demo",
"is_speaking": True,
}
frame = b"\x01\x00" * 960
websocket = FakeWebSocket(
[json.dumps(config), *[frame] * 20, json.dumps({"is_speaking": False, "is_end": True})]
)
asyncio.run(server.ws_serve(websocket))
assert seen and all(seen)
Loading