From f1ef7ceca9f2c7e4308b5e55d4f15807cfc43127 Mon Sep 17 00:00:00 2001 From: Tanishq Date: Sun, 8 Nov 2026 20:55:55 +0530 Subject: [PATCH 1/2] fix(runtime): honor use_itn in funasr_onnx and funasr_torch SenseVoiceSmall SenseVoiceSmall.__call__ in funasr_onnx and funasr_torch read only the textnorm keyword and defaulted it to "woitn", so use_itn was ignored. model(wavs, language="auto", use_itn=True), the call in the ONNX and Libtorch demos and in the SenseVoice READMEs, sent the woitn query (15) like use_itn=False, so the model was asked for text without punctuation and inverse text normalization. Both wrappers read use_itn until 925910c59 replaced it with the textnorm list/file reader. Fall back to use_itn when textnorm is not given, as SenseVoiceSmall.inference does for text_norm. An explicit textnorm still wins, and the default stays woitn. Add tests/test_runtime_sensevoice_use_itn.py, which runs both wrappers on a tiny graph that returns its language and textnorm queries as CTC tokens, exported to ONNX and traced to TorchScript, and run it in the runtime evaluation workflow. --- .github/workflows/test-runtime-eval.yml | 4 +- .../libtorch/funasr_torch/sensevoice_bin.py | 4 +- .../onnxruntime/funasr_onnx/sensevoice_bin.py | 4 +- tests/test_runtime_sensevoice_use_itn.py | 128 ++++++++++++++++++ 4 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 tests/test_runtime_sensevoice_use_itn.py diff --git a/.github/workflows/test-runtime-eval.yml b/.github/workflows/test-runtime-eval.yml index 2e3a1901ea..f4ce8ee399 100644 --- a/.github/workflows/test-runtime-eval.yml +++ b/.github/workflows/test-runtime-eval.yml @@ -7,6 +7,7 @@ on: - "runtime/python/libtorch/funasr_torch/**" - "runtime/python/onnxruntime/funasr_onnx/**" - "tests/test_runtime_eval_*.py" + - "tests/test_runtime_sensevoice_use_itn.py" - ".github/workflows/test-runtime-eval.yml" push: branches: [main] @@ -15,6 +16,7 @@ on: - "runtime/python/libtorch/funasr_torch/**" - "runtime/python/onnxruntime/funasr_onnx/**" - "tests/test_runtime_eval_*.py" + - "tests/test_runtime_sensevoice_use_itn.py" - ".github/workflows/test-runtime-eval.yml" permissions: @@ -47,7 +49,7 @@ jobs: run: | git rev-parse HEAD > source-commit.txt python -m pip freeze > resolved-dependencies.txt - python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml + python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py tests/test_runtime_sensevoice_use_itn.py --junitxml=runtime-eval-results.xml - uses: actions/upload-artifact@v4 if: always() with: diff --git a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py index c27bfdd839..0dd1e9903a 100644 --- a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py +++ b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py @@ -147,7 +147,9 @@ def read_tags(self, language_input, textnorm_input): def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): language_input = kwargs.get("language", "auto") - textnorm_input = kwargs.get("textnorm", "woitn") + textnorm_input = kwargs.get("textnorm") + if textnorm_input is None: + textnorm_input = "withitn" if kwargs.get("use_itn", False) else "woitn" language_list, textnorm_list = self.read_tags(language_input, textnorm_input) waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) diff --git a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py index 0bc265ebb5..1036e53844 100644 --- a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py @@ -152,7 +152,9 @@ def read_tags(self, language_input, textnorm_input): def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): language_input = kwargs.get("language", "auto") - textnorm_input = kwargs.get("textnorm", "woitn") + textnorm_input = kwargs.get("textnorm") + if textnorm_input is None: + textnorm_input = "withitn" if kwargs.get("use_itn", False) else "woitn" language_list, textnorm_list = self.read_tags(language_input, textnorm_input) waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) diff --git a/tests/test_runtime_sensevoice_use_itn.py b/tests/test_runtime_sensevoice_use_itn.py new file mode 100644 index 0000000000..5a9bb17288 --- /dev/null +++ b/tests/test_runtime_sensevoice_use_itn.py @@ -0,0 +1,128 @@ +"""SenseVoiceSmall in funasr_onnx and funasr_torch sends the textnorm query that use_itn selects. + +The synthetic graph returns its language and textnorm queries as CTC tokens, and the fixture +SentencePiece model names those ids, so each result shows the queries the wrapper sent. +The graph and the tokenizer are test fixtures, not a pretrained SenseVoice model. +""" + +import importlib +import io +import json +from pathlib import Path +import wave + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +# SentencePiece keeps ids 0-2 for , and , so the user-defined symbols take ids +# 3-15, the query ids of the wrappers' lid_dict and textnorm_dict. +QUERY_TOKENS = {4: "<|en|>", 14: "<|withitn|>", 15: "<|woitn|>"} + + +@pytest.fixture(scope="module") +def sensevoice_fixture(tmp_path_factory): + torch = pytest.importorskip("torch") + np = pytest.importorskip("numpy") + onnx = pytest.importorskip("onnx") + spm = pytest.importorskip("sentencepiece") + pytest.importorskip("onnxruntime") + pytest.importorskip("kaldi_native_fbank") + directory = tmp_path_factory.mktemp("runtime-sensevoice") + + class QueryEcho(torch.nn.Module): + def forward(self, speech, speech_lengths, language, textnorm): + # Three frames: the language query, blank and the textnorm query. + queries = torch.stack([language, torch.zeros_like(language), textnorm], dim=1) + logits = torch.nn.functional.one_hot(queries.long(), 16).float() + lengths = torch.full_like(speech_lengths, 3, dtype=torch.int32) + # 0 * speech keeps speech as an input of the ONNX graph. + return logits + 0.0 * speech.mean(), lengths + + model = QueryEcho().eval() + inputs = ( + torch.zeros(2, 10, 4), + torch.tensor([10, 8], dtype=torch.int32), + torch.tensor([0, 4], dtype=torch.int32), + torch.tensor([14, 15], dtype=torch.int32), + ) + torch.jit.trace(model, inputs).save(str(directory / "model.torchscript")) + torch.onnx.export( + model, + inputs, + str(directory / "model.onnx"), + dynamo=False, + input_names=["speech", "speech_lengths", "language", "textnorm"], + output_names=["ctc_logits", "encoder_out_lens"], + dynamic_axes={ + "speech": {0: "B", 1: "T"}, + "speech_lengths": {0: "B"}, + "language": {0: "B"}, + "textnorm": {0: "B"}, + "ctc_logits": {0: "B"}, + "encoder_out_lens": {0: "B"}, + }, + opset_version=17, + ) + onnx.checker.check_model(str(directory / "model.onnx")) + bpemodel = io.BytesIO() + spm.SentencePieceTrainer.train( + sentence_iterator=iter(["abc"]), + model_writer=bpemodel, + model_type="char", + vocab_size=20, + user_defined_symbols=[QUERY_TOKENS.get(i, f"<|query{i}|>") for i in range(3, 16)], + minloglevel=2, + ) + (directory / "chn_jpn_yue_eng_ko_spectok.bpe.model").write_bytes(bpemodel.getvalue()) + (directory / "config.yaml").write_text( + json.dumps({"frontend_conf": {"n_mels": 4, "lfr_m": 1, "lfr_n": 1, "dither": 0.0}}) + ) + (directory / "am.mvn").write_text( + " 4 4\n 1 [ 0 0 0 0 ]\n" + " 4 4\n 1 [ 1 1 1 1 ]\n" + ) + paths = [] + for index in range(2): + path = directory / f"audio {index}.wav" + signal = (np.sin(np.arange(8000) * 2 * np.pi * 440 / 16000) * 3000).astype(""), + ({"use_itn": False}, "<|woitn|>"), + ({"use_itn": True}, "<|withitn|>"), + ({"textnorm": "withitn"}, "<|withitn|>"), + ({"textnorm": "woitn", "use_itn": True}, "<|woitn|>"), + ({"language": "en", "use_itn": True}, "<|en|><|withitn|>"), + ], + ids=[ + "default", + "use_itn_false", + "use_itn_true", + "textnorm_withitn", + "textnorm_wins", + "language_en", + ], +) +def test_sensevoice_use_itn_selects_textnorm( + sensevoice_fixture, monkeypatch, backend, batch_size, kwargs, expected +): + directory, paths = sensevoice_fixture + location = "libtorch" if backend == "torch" else "onnxruntime" + monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location)) + module = importlib.import_module("funasr_" + backend) + model = module.SenseVoiceSmall(str(directory), batch_size=batch_size, intra_op_num_threads=1) + # Two files, so with batch_size=1 the second file is decoded in a batch of its own. + assert model(paths, **kwargs) == [expected, expected] From e764f4912c6c8b199c24d80502a20147292a4044 Mon Sep 17 00:00:00 2001 From: zhifu gao Date: Mon, 9 Nov 2026 12:49:20 +0800 Subject: [PATCH 2/2] fix(runtime): keep SenseVoice batch query tags independent Preserve the original PR #3860 commit and use_itn fallback. Retain singleton query tags independently while slicing per-utterance language and textnorm inputs. Add real ONNX/TorchScript regressions for lists, files and partial batches. Signed-off-by: LauraGPT --- .../libtorch/funasr_torch/sensevoice_bin.py | 11 ++--- .../onnxruntime/funasr_onnx/sensevoice_bin.py | 11 ++--- tests/test_runtime_sensevoice_use_itn.py | 42 +++++++++++++++++++ 3 files changed, 54 insertions(+), 10 deletions(-) diff --git a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py index 0dd1e9903a..9781efe4aa 100644 --- a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py +++ b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py @@ -164,11 +164,12 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): for beg_idx in range(0, waveform_nums, self.batch_size): end_idx = min(waveform_nums, beg_idx + self.batch_size) feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx]) - _language_list = language_list[beg_idx:end_idx] - _textnorm_list = textnorm_list[beg_idx:end_idx] - if not len(_language_list): - _language_list = [language_list[0]] - _textnorm_list = [textnorm_list[0]] + _language_list = ( + language_list if len(language_list) == 1 else language_list[beg_idx:end_idx] + ) + _textnorm_list = ( + textnorm_list if len(textnorm_list) == 1 else textnorm_list[beg_idx:end_idx] + ) B = feats.shape[0] if len(_language_list) == 1 and B != 1: _language_list = _language_list * B diff --git a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py index 1036e53844..00a3176d65 100644 --- a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py @@ -169,11 +169,12 @@ def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): for beg_idx in range(0, waveform_nums, self.batch_size): end_idx = min(waveform_nums, beg_idx + self.batch_size) feats, feats_len = self.extract_feat(waveform_list[beg_idx:end_idx]) - _language_list = language_list[beg_idx:end_idx] - _textnorm_list = textnorm_list[beg_idx:end_idx] - if not len(_language_list): - _language_list = [language_list[0]] - _textnorm_list = [textnorm_list[0]] + _language_list = ( + language_list if len(language_list) == 1 else language_list[beg_idx:end_idx] + ) + _textnorm_list = ( + textnorm_list if len(textnorm_list) == 1 else textnorm_list[beg_idx:end_idx] + ) B = feats.shape[0] if len(_language_list) == 1 and B != 1: _language_list = _language_list * B diff --git a/tests/test_runtime_sensevoice_use_itn.py b/tests/test_runtime_sensevoice_use_itn.py index 5a9bb17288..8abc92e27d 100644 --- a/tests/test_runtime_sensevoice_use_itn.py +++ b/tests/test_runtime_sensevoice_use_itn.py @@ -126,3 +126,45 @@ def test_sensevoice_use_itn_selects_textnorm( model = module.SenseVoiceSmall(str(directory), batch_size=batch_size, intra_op_num_threads=1) # Two files, so with batch_size=1 the second file is decoded in a batch of its own. assert model(paths, **kwargs) == [expected, expected] + + +@pytest.mark.parametrize("backend", ["onnx", "torch"]) +@pytest.mark.parametrize("batch_size", [1, 2, 3]) +@pytest.mark.parametrize( + "language_kind, textnorm_kind", + [ + ("scalar", "list"), + ("scalar", "file"), + ("list", "scalar"), + ("file", "scalar"), + ("list", "list"), + ("file", "file"), + ], +) +def test_sensevoice_mixed_scalar_and_per_utterance_tags( + sensevoice_fixture, monkeypatch, tmp_path, backend, batch_size, language_kind, textnorm_kind +): + directory, paths = sensevoice_fixture + location = "libtorch" if backend == "torch" else "onnxruntime" + monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location)) + module = importlib.import_module("funasr_" + backend) + model = module.SenseVoiceSmall(str(directory), batch_size=batch_size, intra_op_num_threads=1) + kwargs = {"use_itn": True} + for name, kind, values in [ + ("language", language_kind, ["en", "zh", "ko"]), + ("textnorm", textnorm_kind, ["withitn", "woitn", "woitn"]), + ]: + if kind == "list": + kwargs[name] = values + elif kind == "file": + tag_file = tmp_path / (name + ".txt") + tag_file.write_text("\n".join(values) + "\n") + kwargs[name] = str(tag_file) + if language_kind == "scalar": + expected = ["<|withitn|>", "<|woitn|>", "<|woitn|>"] + elif textnorm_kind == "scalar": + expected = ["<|en|><|withitn|>", "<|query3|><|withitn|>", "<|query12|><|withitn|>"] + else: + expected = ["<|en|><|withitn|>", "<|query3|><|woitn|>", "<|query12|><|woitn|>"] + # Three routing entries also exercise the final partial batch at batch_size=2. + assert model(paths + [paths[0]], **kwargs) == expected