diff --git a/README.md b/README.md index db74816d7..91c3682cf 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,12 @@ from funasr import AutoModel model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda") result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20") +# Optional Silero VAD (install first: python -m pip install "funasr[silero]") +model = AutoModel( + model="paraformer-zh", vad_model="silero-vad", device="cuda", + vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100}, +) +result = model.generate(input="audio.wav") # Streaming real-time (feed audio chunk by chunk) import soundfile as sf diff --git a/README_zh.md b/README_zh.md index f2916d422..de2b3d9b4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -239,6 +239,13 @@ from funasr import AutoModel model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda") result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20") +# 使用 Silero VAD（先安装：python -m pip install "funasr[silero]"） +model = AutoModel( + model="paraformer-zh", vad_model="silero-vad", device="cpu", + vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100}, +) +result = model.generate(input="audio.wav") + # 中/英/日 + 中文方言 model = AutoModel(model="FunAudioLLM/Fun-ASR-Nano-2512", hub="hf", trust_remote_code=True, vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000}, device="cuda") diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index dd94a3fd2..3dbb7a4f2 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -536,6 +536,12 @@ def build_model(**kwargs): kwargs contains the resolved configuration. """ assert "model" in kwargs + # Silero VAD is loaded by its optional Python package rather than a + # FunASR model repository. Supplying model_conf keeps it on the normal + # AutoModel construction path while bypassing hub config resolution. + if kwargs["model"] in {"silero-vad", "silero_vad"}: + kwargs.setdefault("model_conf", {}) + kwargs["model"] = "SileroVad" if "model_conf" not in kwargs: logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms"))) kwargs = download_model(**kwargs) diff --git a/funasr/models/silero_vad/__init__.py b/funasr/models/silero_vad/__init__.py new file mode 100644 index 000000000..d62060435 --- /dev/null +++ b/funasr/models/silero_vad/__init__.py @@ -0,0 +1 @@ +"""Silero VAD adapter for the FunASR AutoModel pipeline.""" diff --git a/funasr/models/silero_vad/model.py b/funasr/models/silero_vad/model.py new file mode 100644 index 000000000..7f90ceeda --- /dev/null +++ b/funasr/models/silero_vad/model.py @@ -0,0 +1,97 @@ +"""Adapter that makes Silero VAD return FunASR-compatible millisecond segments.""" + +import time + +import torch + +from funasr.register import tables +from funasr.utils.load_utils import load_audio_text_image_video + + +@tables.register("model_classes", "SileroVad") +class SileroVad(torch.nn.Module): + """Offline Silero VAD adapter used by ``AutoModel(vad_model='silero-vad')``. + + Requires the official ``silero-vad`` Python package. + """ + + def __init__(self, **kwargs): + super().__init__() + self.anchor = torch.nn.Parameter(torch.empty(0), requires_grad=False) + try: + from silero_vad import get_speech_timestamps, load_silero_vad + except ImportError as error: + raise ImportError( + "Silero VAD requires the optional dependency. Install it with " + '`python -m pip install "funasr[silero]"` or ' + "`python -m pip install silero-vad`." + ) from error + self.onnx = bool(kwargs.get("silero_onnx", False)) + self.model = load_silero_vad(onnx=self.onnx) + self.get_speech_timestamps = get_speech_timestamps + + @staticmethod + def _split_long_segments(segments, max_single_segment_time): + if not max_single_segment_time: + return segments + limit_ms = int(max_single_segment_time) + if limit_ms <= 0: + raise ValueError( + "max_single_segment_time must resolve to a positive millisecond value" + ) + split = [] + for start, end in segments: + while end - start > limit_ms: + split.append([start, start + limit_ms]) + start += limit_ms + split.append([start, end]) + return split + + def inference(self, data_in, key=None, **kwargs): + sample_rate = int(kwargs.get("silero_sampling_rate", 16000)) + if sample_rate not in (8000, 16000): + raise ValueError("Silero VAD supports silero_sampling_rate=8000 or 16000") + audio_list = load_audio_text_image_video( + data_in, + fs=sample_rate, + audio_fs=kwargs.get("fs", sample_rate), + data_type=kwargs.get("data_type", "sound"), + ) + if not isinstance(audio_list, list): + audio_list = [audio_list] + + started = time.perf_counter() + results = [] + for index, audio in enumerate(audio_list): + device = torch.device("cpu") if self.onnx else self.anchor.device + waveform = torch.as_tensor(audio, dtype=torch.float32).flatten().to(device) + timestamps = self.get_speech_timestamps( + waveform, + self.model, + sampling_rate=sample_rate, + threshold=kwargs.get("silero_threshold", 0.5), + min_speech_duration_ms=kwargs.get("silero_min_speech_duration_ms", 250), + min_silence_duration_ms=kwargs.get( + "silero_min_silence_duration_ms", 100 + ), + speech_pad_ms=kwargs.get("silero_speech_pad_ms", 30), + ) + segments = [ + [ + int(item["start"] * 1000 / sample_rate), + int(item["end"] * 1000 / sample_rate), + ] + for item in timestamps + ] + segments = self._split_long_segments( + segments, kwargs.get("max_single_segment_time") + ) + results.append( + {"key": key[index] if key else str(index), "value": segments} + ) + elapsed = time.perf_counter() - started + total_samples = sum(len(torch.as_tensor(audio)) for audio in audio_list) + return results, { + "batch_data_time": total_samples / sample_rate, + "forward": elapsed, + } diff --git a/setup.py b/setup.py index 7875d77e3..d96614136 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,9 @@ "train": [ "rapidfuzz>=3.0.0", ], + "silero": [ + "silero-vad>=6.0.0", + ], # all: The modules should be optionally installled due to some reason. # Please consider moving them to "install" occasionally "all": [ diff --git a/tests/test_silero_vad_adapter.py b/tests/test_silero_vad_adapter.py new file mode 100644 index 000000000..3399e1381 --- /dev/null +++ b/tests/test_silero_vad_adapter.py @@ -0,0 +1,100 @@ +import unittest +from importlib.util import find_spec +from unittest.mock import patch + +import torch + +from funasr.auto.auto_model import AutoModel +from funasr.models.silero_vad.model import SileroVad + + +@unittest.skipUnless(find_spec("silero_vad"), "silero-vad is not installed") +class TestSileroVadAdapter(unittest.TestCase): + def _timestamps_stub(self, waveform, model, sampling_rate, **options): + self.assertEqual(sampling_rate, 16000) + self.assertEqual(options["threshold"], 0.6) + return [{"start": 1600, "end": 17600}] + + def _load_stub(self, *args, **kwargs): + self.assertEqual(kwargs, {"onnx": False}) + return torch.nn.Identity() + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_returns_funasr_millisecond_segments_and_honors_max_length( + self, load_model, timestamps + ): + load_model.side_effect = self._load_stub + timestamps.side_effect = self._timestamps_stub + model = SileroVad() + results, metadata = model.inference( + data_in=[torch.zeros(32000)], + key=["sample"], + silero_threshold=0.6, + max_single_segment_time=500, + ) + + self.assertEqual( + results, [{"key": "sample", "value": [[100, 600], [600, 1100]]}] + ) + self.assertEqual(metadata["batch_data_time"], 2.0) + load_model.assert_called_once_with(onnx=False) + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_rejects_unsupported_sampling_rate(self, load_model, timestamps): + load_model.side_effect = self._load_stub + model = SileroVad() + with self.assertRaisesRegex(ValueError, "8000 or 16000"): + model.inference(data_in=[torch.zeros(16000)], silero_sampling_rate=44100) + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_auto_model_alias_uses_the_existing_vad_build_path( + self, load_model, timestamps + ): + load_model.side_effect = self._load_stub + model, resolved = AutoModel.build_model(model="silero-vad", device="cpu") + self.assertIsInstance(model, SileroVad) + self.assertEqual(resolved["model"], "SileroVad") + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_waveform_follows_the_adapter_device(self, load_model, timestamps): + load_model.side_effect = self._load_stub + + def timestamps_stub(waveform, model, sampling_rate, **options): + self.assertEqual(waveform.device.type, "meta") + return [] + + timestamps.side_effect = timestamps_stub + model = SileroVad().to("meta") + results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"]) + self.assertEqual(results, [{"key": "sample", "value": []}]) + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_onnx_waveform_stays_on_cpu(self, load_model, timestamps): + load_model.return_value = object() + + def timestamps_stub(waveform, model, sampling_rate, **options): + self.assertEqual(waveform.device.type, "cpu") + return [] + + timestamps.side_effect = timestamps_stub + model = SileroVad(silero_onnx=True).to("meta") + results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"]) + self.assertEqual(results, [{"key": "sample", "value": []}]) + load_model.assert_called_once_with(onnx=True) + + def test_rejects_negative_max_segment_length(self): + with self.assertRaisesRegex(ValueError, "positive"): + SileroVad._split_long_segments([[100, 1100]], -500) + + def test_rejects_sub_millisecond_max_segment_length(self): + with self.assertRaisesRegex(ValueError, "positive"): + SileroVad._split_long_segments([[100, 1100]], 0.5) + + +if __name__ == "__main__": + unittest.main()