From f1ef7ceca9f2c7e4308b5e55d4f15807cfc43127 Mon Sep 17 00:00:00 2001 From: Tanishq Date: Sun, 8 Nov 2026 20:55:55 +0530 Subject: [PATCH] fix(runtime): honor use_itn in funasr_onnx and funasr_torch SenseVoiceSmall SenseVoiceSmall.__call__ in funasr_onnx and funasr_torch read only the textnorm keyword and defaulted it to "woitn", so use_itn was ignored. model(wavs, language="auto", use_itn=True), the call in the ONNX and Libtorch demos and in the SenseVoice READMEs, sent the woitn query (15) like use_itn=False, so the model was asked for text without punctuation and inverse text normalization. Both wrappers read use_itn until 925910c59 replaced it with the textnorm list/file reader. Fall back to use_itn when textnorm is not given, as SenseVoiceSmall.inference does for text_norm. An explicit textnorm still wins, and the default stays woitn. Add tests/test_runtime_sensevoice_use_itn.py, which runs both wrappers on a tiny graph that returns its language and textnorm queries as CTC tokens, exported to ONNX and traced to TorchScript, and run it in the runtime evaluation workflow. --- .github/workflows/test-runtime-eval.yml | 4 +- .../libtorch/funasr_torch/sensevoice_bin.py | 4 +- .../onnxruntime/funasr_onnx/sensevoice_bin.py | 4 +- tests/test_runtime_sensevoice_use_itn.py | 128 ++++++++++++++++++ 4 files changed, 137 insertions(+), 3 deletions(-) create mode 100644 tests/test_runtime_sensevoice_use_itn.py diff --git a/.github/workflows/test-runtime-eval.yml b/.github/workflows/test-runtime-eval.yml index 2e3a1901ea..f4ce8ee399 100644 --- a/.github/workflows/test-runtime-eval.yml +++ b/.github/workflows/test-runtime-eval.yml @@ -7,6 +7,7 @@ on: - "runtime/python/libtorch/funasr_torch/**" - "runtime/python/onnxruntime/funasr_onnx/**" - "tests/test_runtime_eval_*.py" + - "tests/test_runtime_sensevoice_use_itn.py" - ".github/workflows/test-runtime-eval.yml" push: branches: [main] @@ -15,6 +16,7 @@ on: - "runtime/python/libtorch/funasr_torch/**" - "runtime/python/onnxruntime/funasr_onnx/**" - "tests/test_runtime_eval_*.py" + - "tests/test_runtime_sensevoice_use_itn.py" - ".github/workflows/test-runtime-eval.yml" permissions: @@ -47,7 +49,7 @@ jobs: run: | git rev-parse HEAD > source-commit.txt python -m pip freeze > resolved-dependencies.txt - python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py --junitxml=runtime-eval-results.xml + python -m pytest -q tests/test_runtime_eval_cli.py tests/test_runtime_eval_native.py tests/test_runtime_sensevoice_use_itn.py --junitxml=runtime-eval-results.xml - uses: actions/upload-artifact@v4 if: always() with: diff --git a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py index c27bfdd839..0dd1e9903a 100644 --- a/runtime/python/libtorch/funasr_torch/sensevoice_bin.py +++ b/runtime/python/libtorch/funasr_torch/sensevoice_bin.py @@ -147,7 +147,9 @@ def read_tags(self, language_input, textnorm_input): def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): language_input = kwargs.get("language", "auto") - textnorm_input = kwargs.get("textnorm", "woitn") + textnorm_input = kwargs.get("textnorm") + if textnorm_input is None: + textnorm_input = "withitn" if kwargs.get("use_itn", False) else "woitn" language_list, textnorm_list = self.read_tags(language_input, textnorm_input) waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) diff --git a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py index 0bc265ebb5..1036e53844 100644 --- a/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py +++ b/runtime/python/onnxruntime/funasr_onnx/sensevoice_bin.py @@ -152,7 +152,9 @@ def read_tags(self, language_input, textnorm_input): def __call__(self, wav_content: Union[str, np.ndarray, List[str]], **kwargs): language_input = kwargs.get("language", "auto") - textnorm_input = kwargs.get("textnorm", "woitn") + textnorm_input = kwargs.get("textnorm") + if textnorm_input is None: + textnorm_input = "withitn" if kwargs.get("use_itn", False) else "woitn" language_list, textnorm_list = self.read_tags(language_input, textnorm_input) waveform_list = self.load_data(wav_content, self.frontend.opts.frame_opts.samp_freq) diff --git a/tests/test_runtime_sensevoice_use_itn.py b/tests/test_runtime_sensevoice_use_itn.py new file mode 100644 index 0000000000..5a9bb17288 --- /dev/null +++ b/tests/test_runtime_sensevoice_use_itn.py @@ -0,0 +1,128 @@ +"""SenseVoiceSmall in funasr_onnx and funasr_torch sends the textnorm query that use_itn selects. + +The synthetic graph returns its language and textnorm queries as CTC tokens, and the fixture +SentencePiece model names those ids, so each result shows the queries the wrapper sent. +The graph and the tokenizer are test fixtures, not a pretrained SenseVoice model. +""" + +import importlib +import io +import json +from pathlib import Path +import wave + +import pytest + + +ROOT = Path(__file__).resolve().parents[1] +# SentencePiece keeps ids 0-2 for , and , so the user-defined symbols take ids +# 3-15, the query ids of the wrappers' lid_dict and textnorm_dict. +QUERY_TOKENS = {4: "<|en|>", 14: "<|withitn|>", 15: "<|woitn|>"} + + +@pytest.fixture(scope="module") +def sensevoice_fixture(tmp_path_factory): + torch = pytest.importorskip("torch") + np = pytest.importorskip("numpy") + onnx = pytest.importorskip("onnx") + spm = pytest.importorskip("sentencepiece") + pytest.importorskip("onnxruntime") + pytest.importorskip("kaldi_native_fbank") + directory = tmp_path_factory.mktemp("runtime-sensevoice") + + class QueryEcho(torch.nn.Module): + def forward(self, speech, speech_lengths, language, textnorm): + # Three frames: the language query, blank and the textnorm query. + queries = torch.stack([language, torch.zeros_like(language), textnorm], dim=1) + logits = torch.nn.functional.one_hot(queries.long(), 16).float() + lengths = torch.full_like(speech_lengths, 3, dtype=torch.int32) + # 0 * speech keeps speech as an input of the ONNX graph. + return logits + 0.0 * speech.mean(), lengths + + model = QueryEcho().eval() + inputs = ( + torch.zeros(2, 10, 4), + torch.tensor([10, 8], dtype=torch.int32), + torch.tensor([0, 4], dtype=torch.int32), + torch.tensor([14, 15], dtype=torch.int32), + ) + torch.jit.trace(model, inputs).save(str(directory / "model.torchscript")) + torch.onnx.export( + model, + inputs, + str(directory / "model.onnx"), + dynamo=False, + input_names=["speech", "speech_lengths", "language", "textnorm"], + output_names=["ctc_logits", "encoder_out_lens"], + dynamic_axes={ + "speech": {0: "B", 1: "T"}, + "speech_lengths": {0: "B"}, + "language": {0: "B"}, + "textnorm": {0: "B"}, + "ctc_logits": {0: "B"}, + "encoder_out_lens": {0: "B"}, + }, + opset_version=17, + ) + onnx.checker.check_model(str(directory / "model.onnx")) + bpemodel = io.BytesIO() + spm.SentencePieceTrainer.train( + sentence_iterator=iter(["abc"]), + model_writer=bpemodel, + model_type="char", + vocab_size=20, + user_defined_symbols=[QUERY_TOKENS.get(i, f"<|query{i}|>") for i in range(3, 16)], + minloglevel=2, + ) + (directory / "chn_jpn_yue_eng_ko_spectok.bpe.model").write_bytes(bpemodel.getvalue()) + (directory / "config.yaml").write_text( + json.dumps({"frontend_conf": {"n_mels": 4, "lfr_m": 1, "lfr_n": 1, "dither": 0.0}}) + ) + (directory / "am.mvn").write_text( + " 4 4\n 1 [ 0 0 0 0 ]\n" + " 4 4\n 1 [ 1 1 1 1 ]\n" + ) + paths = [] + for index in range(2): + path = directory / f"audio {index}.wav" + signal = (np.sin(np.arange(8000) * 2 * np.pi * 440 / 16000) * 3000).astype(""), + ({"use_itn": False}, "<|woitn|>"), + ({"use_itn": True}, "<|withitn|>"), + ({"textnorm": "withitn"}, "<|withitn|>"), + ({"textnorm": "woitn", "use_itn": True}, "<|woitn|>"), + ({"language": "en", "use_itn": True}, "<|en|><|withitn|>"), + ], + ids=[ + "default", + "use_itn_false", + "use_itn_true", + "textnorm_withitn", + "textnorm_wins", + "language_en", + ], +) +def test_sensevoice_use_itn_selects_textnorm( + sensevoice_fixture, monkeypatch, backend, batch_size, kwargs, expected +): + directory, paths = sensevoice_fixture + location = "libtorch" if backend == "torch" else "onnxruntime" + monkeypatch.syspath_prepend(str(ROOT / "runtime/python" / location)) + module = importlib.import_module("funasr_" + backend) + model = module.SenseVoiceSmall(str(directory), batch_size=batch_size, intra_op_num_threads=1) + # Two files, so with batch_size=1 the second file is decoded in a batch of its own. + assert model(paths, **kwargs) == [expected, expected]