From 22920822a2ed197aa2dc38278cf38e97e07ec509 Mon Sep 17 00:00:00 2001 From: wuhongsheng <664116298@qq.com> Date: Fri, 14 Aug 2026 10:56:47 +0800 Subject: [PATCH 1/3] feat(vad): add Silero VAD adapter --- README_zh.md | 7 +++ funasr/auto/auto_model.py | 6 +++ funasr/models/silero_vad/__init__.py | 1 + funasr/models/silero_vad/model.py | 81 ++++++++++++++++++++++++++++ setup.py | 3 ++ tests/test_silero_vad_adapter.py | 57 ++++++++++++++++++++ 6 files changed, 155 insertions(+) create mode 100644 funasr/models/silero_vad/__init__.py create mode 100644 funasr/models/silero_vad/model.py create mode 100644 tests/test_silero_vad_adapter.py diff --git a/README_zh.md b/README_zh.md index f2916d422..de2b3d9b4 100644 --- a/README_zh.md +++ b/README_zh.md @@ -239,6 +239,13 @@ from funasr import AutoModel model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda") result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20") +# 使用 Silero VAD（先安装：python -m pip install "funasr[silero]"） +model = AutoModel( + model="paraformer-zh", vad_model="silero-vad", device="cpu", + vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100}, +) +result = model.generate(input="audio.wav") + # 中/英/日 + 中文方言 model = AutoModel(model="FunAudioLLM/Fun-ASR-Nano-2512", hub="hf", trust_remote_code=True, vad_model="fsmn-vad", vad_kwargs={"max_single_segment_time": 30000}, device="cuda") diff --git a/funasr/auto/auto_model.py b/funasr/auto/auto_model.py index dd94a3fd2..3dbb7a4f2 100644 --- a/funasr/auto/auto_model.py +++ b/funasr/auto/auto_model.py @@ -536,6 +536,12 @@ def build_model(**kwargs): kwargs contains the resolved configuration. """ assert "model" in kwargs + # Silero VAD is loaded by its optional Python package rather than a + # FunASR model repository. Supplying model_conf keeps it on the normal + # AutoModel construction path while bypassing hub config resolution. + if kwargs["model"] in {"silero-vad", "silero_vad"}: + kwargs.setdefault("model_conf", {}) + kwargs["model"] = "SileroVad" if "model_conf" not in kwargs: logging.info("download models from model hub: {}".format(kwargs.get("hub", "ms"))) kwargs = download_model(**kwargs) diff --git a/funasr/models/silero_vad/__init__.py b/funasr/models/silero_vad/__init__.py new file mode 100644 index 000000000..d62060435 --- /dev/null +++ b/funasr/models/silero_vad/__init__.py @@ -0,0 +1 @@ +"""Silero VAD adapter for the FunASR AutoModel pipeline.""" diff --git a/funasr/models/silero_vad/model.py b/funasr/models/silero_vad/model.py new file mode 100644 index 000000000..1ba989f2a --- /dev/null +++ b/funasr/models/silero_vad/model.py @@ -0,0 +1,81 @@ +"""Adapter that makes Silero VAD return FunASR-compatible millisecond segments.""" + +import time + +import torch + +from funasr.register import tables +from funasr.utils.load_utils import load_audio_text_image_video + + +@tables.register("model_classes", "SileroVad") +class SileroVad(torch.nn.Module): + """Offline Silero VAD adapter used by ``AutoModel(vad_model='silero-vad')``. + + Requires the official ``silero-vad`` Python package. + """ + + def __init__(self, **kwargs): + super().__init__() + self.anchor = torch.nn.Parameter(torch.empty(0), requires_grad=False) + try: + from silero_vad import get_speech_timestamps, load_silero_vad + except ImportError as error: + raise ImportError( + "Silero VAD requires the optional dependency. Install it with " + '`python -m pip install "funasr[silero]"` or ' + "`python -m pip install silero-vad`." + ) from error + self.model = load_silero_vad(onnx=kwargs.get("silero_onnx", False)) + self.get_speech_timestamps = get_speech_timestamps + + @staticmethod + def _split_long_segments(segments, max_single_segment_time): + if not max_single_segment_time: + return segments + limit_ms = int(max_single_segment_time) + split = [] + for start, end in segments: + while end - start > limit_ms: + split.append([start, start + limit_ms]) + start += limit_ms + split.append([start, end]) + return split + + def inference(self, data_in, key=None, **kwargs): + sample_rate = int(kwargs.get("silero_sampling_rate", 16000)) + if sample_rate not in (8000, 16000): + raise ValueError("Silero VAD supports silero_sampling_rate=8000 or 16000") + audio_list = load_audio_text_image_video( + data_in, + fs=sample_rate, + audio_fs=kwargs.get("fs", sample_rate), + data_type=kwargs.get("data_type", "sound"), + ) + if not isinstance(audio_list, list): + audio_list = [audio_list] + + started = time.perf_counter() + results = [] + for index, audio in enumerate(audio_list): + waveform = torch.as_tensor(audio, dtype=torch.float32).flatten().cpu() + timestamps = self.get_speech_timestamps( + waveform, + self.model, + sampling_rate=sample_rate, + threshold=kwargs.get("silero_threshold", 0.5), + min_speech_duration_ms=kwargs.get("silero_min_speech_duration_ms", 250), + min_silence_duration_ms=kwargs.get("silero_min_silence_duration_ms", 100), + speech_pad_ms=kwargs.get("silero_speech_pad_ms", 30), + ) + segments = [ + [int(item["start"] * 1000 / sample_rate), int(item["end"] * 1000 / sample_rate)] + for item in timestamps + ] + segments = self._split_long_segments( + segments, kwargs.get("max_single_segment_time") + ) + results.append({"key": key[index] if key else str(index), "value": segments}) + elapsed = time.perf_counter() - started + total_samples = sum(len(torch.as_tensor(audio)) for audio in audio_list) + return results, {"batch_data_time": total_samples / sample_rate, "forward": elapsed} diff --git a/setup.py b/setup.py index 7875d77e3..d96614136 100644 --- a/setup.py +++ b/setup.py @@ -48,6 +48,9 @@ "train": [ "rapidfuzz>=3.0.0", ], + "silero": [ + "silero-vad>=6.0.0", + ], # all: The modules should be optionally installled due to some reason. # Please consider moving them to "install" occasionally "all": [ diff --git a/tests/test_silero_vad_adapter.py b/tests/test_silero_vad_adapter.py new file mode 100644 index 000000000..0e5819382 --- /dev/null +++ b/tests/test_silero_vad_adapter.py @@ -0,0 +1,57 @@ +import unittest +from importlib.util import find_spec +from unittest.mock import patch + +import torch + +from funasr.auto.auto_model import AutoModel +from funasr.models.silero_vad.model import SileroVad + + +@unittest.skipUnless(find_spec("silero_vad"), "silero-vad is not installed") +class TestSileroVadAdapter(unittest.TestCase): + def _timestamps_stub(self, waveform, model, sampling_rate, **options): + self.assertEqual(sampling_rate, 16000) + self.assertEqual(options["threshold"], 0.6) + return [{"start": 1600, "end": 17600}] + + def _load_stub(self, *args, **kwargs): + self.assertEqual(kwargs, {"onnx": False}) + return torch.nn.Identity() + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_returns_funasr_millisecond_segments_and_honors_max_length(self, load_model, timestamps): + load_model.side_effect = self._load_stub + timestamps.side_effect = self._timestamps_stub + model = SileroVad() + results, metadata = model.inference( + data_in=[torch.zeros(32000)], + key=["sample"], + silero_threshold=0.6, + max_single_segment_time=500, + ) + + self.assertEqual(results, [{"key": "sample", "value": [[100, 600], [600, 1100]]}]) + self.assertEqual(metadata["batch_data_time"], 2.0) + load_model.assert_called_once_with(onnx=False) + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_rejects_unsupported_sampling_rate(self, load_model, timestamps): + load_model.side_effect = self._load_stub + model = SileroVad() + with self.assertRaisesRegex(ValueError, "8000 or 16000"): + model.inference(data_in=[torch.zeros(16000)], silero_sampling_rate=44100) + + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_auto_model_alias_uses_the_existing_vad_build_path(self, load_model, timestamps): + load_model.side_effect = self._load_stub + model, resolved = AutoModel.build_model(model="silero-vad", device="cpu") + self.assertIsInstance(model, SileroVad) + self.assertEqual(resolved["model"], "SileroVad") + + +if __name__ == "__main__": + unittest.main() From 419674e544062230d712d29ec810a866bbfa596c Mon Sep 17 00:00:00 2001 From: LauraGPT Date: Fri, 14 Aug 2026 07:32:23 +0000 Subject: [PATCH 2/3] fix(vad): keep Silero inputs on the model device --- README.md | 6 ++++++ funasr/models/silero_vad/model.py | 26 +++++++++++++++++++++----- tests/test_silero_vad_adapter.py | 30 +++++++++++++++++++++++++++--- 3 files changed, 54 insertions(+), 8 deletions(-) diff --git a/README.md b/README.md index db74816d7..91c3682cf 100644 --- a/README.md +++ b/README.md @@ -189,6 +189,12 @@ from funasr import AutoModel model = AutoModel(model="paraformer-zh", vad_model="fsmn-vad", punc_model="ct-punc", spk_model="cam++", device="cuda") result = model.generate(input="https://isv-data.oss-cn-hangzhou.aliyuncs.com/ics/MaaS/ASR/test_audio/asr_example_zh.wav", hotword="关键词 20") +# Optional Silero VAD (install first: python -m pip install "funasr[silero]") +model = AutoModel( + model="paraformer-zh", vad_model="silero-vad", device="cuda", + vad_kwargs={"silero_threshold": 0.5, "silero_min_silence_duration_ms": 100}, +) +result = model.generate(input="audio.wav") # Streaming real-time (feed audio chunk by chunk) import soundfile as sf diff --git a/funasr/models/silero_vad/model.py b/funasr/models/silero_vad/model.py index 1ba989f2a..c5f120d46 100644 --- a/funasr/models/silero_vad/model.py +++ b/funasr/models/silero_vad/model.py @@ -34,6 +34,8 @@ def _split_long_segments(segments, max_single_segment_time): if not max_single_segment_time: return segments limit_ms = int(max_single_segment_time) + if limit_ms < 0: + raise ValueError("max_single_segment_time must be non-negative") split = [] for start, end in segments: while end - start > limit_ms: @@ -58,24 +60,38 @@ def inference(self, data_in, key=None, **kwargs): started = time.perf_counter() results = [] for index, audio in enumerate(audio_list): - waveform = torch.as_tensor(audio, dtype=torch.float32).flatten().cpu() + waveform = ( + torch.as_tensor(audio, dtype=torch.float32) + .flatten() + .to(self.anchor.device) + ) timestamps = self.get_speech_timestamps( waveform, self.model, sampling_rate=sample_rate, threshold=kwargs.get("silero_threshold", 0.5), min_speech_duration_ms=kwargs.get("silero_min_speech_duration_ms", 250), - min_silence_duration_ms=kwargs.get("silero_min_silence_duration_ms", 100), + min_silence_duration_ms=kwargs.get( + "silero_min_silence_duration_ms", 100 + ), speech_pad_ms=kwargs.get("silero_speech_pad_ms", 30), ) segments = [ - [int(item["start"] * 1000 / sample_rate), int(item["end"] * 1000 / sample_rate)] + [ + int(item["start"] * 1000 / sample_rate), + int(item["end"] * 1000 / sample_rate), + ] for item in timestamps ] segments = self._split_long_segments( segments, kwargs.get("max_single_segment_time") ) - results.append({"key": key[index] if key else str(index), "value": segments}) + results.append( + {"key": key[index] if key else str(index), "value": segments} + ) elapsed = time.perf_counter() - started total_samples = sum(len(torch.as_tensor(audio)) for audio in audio_list) - return results, {"batch_data_time": total_samples / sample_rate, "forward": elapsed} + return results, { + "batch_data_time": total_samples / sample_rate, + "forward": elapsed, + } diff --git a/tests/test_silero_vad_adapter.py b/tests/test_silero_vad_adapter.py index 0e5819382..2effe6f49 100644 --- a/tests/test_silero_vad_adapter.py +++ b/tests/test_silero_vad_adapter.py @@ -21,7 +21,9 @@ def _load_stub(self, *args, **kwargs): @patch("silero_vad.get_speech_timestamps") @patch("silero_vad.load_silero_vad") - def test_returns_funasr_millisecond_segments_and_honors_max_length(self, load_model, timestamps): + def test_returns_funasr_millisecond_segments_and_honors_max_length( + self, load_model, timestamps + ): load_model.side_effect = self._load_stub timestamps.side_effect = self._timestamps_stub model = SileroVad() @@ -32,7 +34,9 @@ def test_returns_funasr_millisecond_segments_and_honors_max_length(self, load_mo max_single_segment_time=500, ) - self.assertEqual(results, [{"key": "sample", "value": [[100, 600], [600, 1100]]}]) + self.assertEqual( + results, [{"key": "sample", "value": [[100, 600], [600, 1100]]}] + ) self.assertEqual(metadata["batch_data_time"], 2.0) load_model.assert_called_once_with(onnx=False) @@ -46,12 +50,32 @@ def test_rejects_unsupported_sampling_rate(self, load_model, timestamps): @patch("silero_vad.get_speech_timestamps") @patch("silero_vad.load_silero_vad") - def test_auto_model_alias_uses_the_existing_vad_build_path(self, load_model, timestamps): + def test_auto_model_alias_uses_the_existing_vad_build_path( + self, load_model, timestamps + ): load_model.side_effect = self._load_stub model, resolved = AutoModel.build_model(model="silero-vad", device="cpu") self.assertIsInstance(model, SileroVad) self.assertEqual(resolved["model"], "SileroVad") + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_waveform_follows_the_adapter_device(self, load_model, timestamps): + load_model.side_effect = self._load_stub + + def timestamps_stub(waveform, model, sampling_rate, **options): + self.assertEqual(waveform.device.type, "meta") + return [] + + timestamps.side_effect = timestamps_stub + model = SileroVad().to("meta") + results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"]) + self.assertEqual(results, [{"key": "sample", "value": []}]) + + def test_rejects_negative_max_segment_length(self): + with self.assertRaisesRegex(ValueError, "non-negative"): + SileroVad._split_long_segments([[100, 1100]], -500) + if __name__ == "__main__": unittest.main() From e8ea59a02c9362ae78e6730ca574862f5aedd6c7 Mon Sep 17 00:00:00 2001 From: LauraGPT Date: Fri, 14 Aug 2026 07:35:21 +0000 Subject: [PATCH 3/3] fix(vad): preserve Silero ONNX CPU inputs --- funasr/models/silero_vad/model.py | 16 ++++++++-------- tests/test_silero_vad_adapter.py | 21 ++++++++++++++++++++- 2 files changed, 28 insertions(+), 9 deletions(-) diff --git a/funasr/models/silero_vad/model.py b/funasr/models/silero_vad/model.py index c5f120d46..7f90ceeda 100644 --- a/funasr/models/silero_vad/model.py +++ b/funasr/models/silero_vad/model.py @@ -26,7 +26,8 @@ def __init__(self, **kwargs): '`python -m pip install "funasr[silero]"` or ' "`python -m pip install silero-vad`." ) from error - self.model = load_silero_vad(onnx=kwargs.get("silero_onnx", False)) + self.onnx = bool(kwargs.get("silero_onnx", False)) + self.model = load_silero_vad(onnx=self.onnx) self.get_speech_timestamps = get_speech_timestamps @staticmethod @@ -34,8 +35,10 @@ def _split_long_segments(segments, max_single_segment_time): if not max_single_segment_time: return segments limit_ms = int(max_single_segment_time) - if limit_ms < 0: - raise ValueError("max_single_segment_time must be non-negative") + if limit_ms <= 0: + raise ValueError( + "max_single_segment_time must resolve to a positive millisecond value" + ) split = [] for start, end in segments: while end - start > limit_ms: @@ -60,11 +63,8 @@ def inference(self, data_in, key=None, **kwargs): started = time.perf_counter() results = [] for index, audio in enumerate(audio_list): - waveform = ( - torch.as_tensor(audio, dtype=torch.float32) - .flatten() - .to(self.anchor.device) - ) + device = torch.device("cpu") if self.onnx else self.anchor.device + waveform = torch.as_tensor(audio, dtype=torch.float32).flatten().to(device) timestamps = self.get_speech_timestamps( waveform, self.model, diff --git a/tests/test_silero_vad_adapter.py b/tests/test_silero_vad_adapter.py index 2effe6f49..3399e1381 100644 --- a/tests/test_silero_vad_adapter.py +++ b/tests/test_silero_vad_adapter.py @@ -72,10 +72,29 @@ def timestamps_stub(waveform, model, sampling_rate, **options): results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"]) self.assertEqual(results, [{"key": "sample", "value": []}]) + @patch("silero_vad.get_speech_timestamps") + @patch("silero_vad.load_silero_vad") + def test_onnx_waveform_stays_on_cpu(self, load_model, timestamps): + load_model.return_value = object() + + def timestamps_stub(waveform, model, sampling_rate, **options): + self.assertEqual(waveform.device.type, "cpu") + return [] + + timestamps.side_effect = timestamps_stub + model = SileroVad(silero_onnx=True).to("meta") + results, _ = model.inference(data_in=[torch.zeros(16000)], key=["sample"]) + self.assertEqual(results, [{"key": "sample", "value": []}]) + load_model.assert_called_once_with(onnx=True) + def test_rejects_negative_max_segment_length(self): - with self.assertRaisesRegex(ValueError, "non-negative"): + with self.assertRaisesRegex(ValueError, "positive"): SileroVad._split_long_segments([[100, 1100]], -500) + def test_rejects_sub_millisecond_max_segment_length(self): + with self.assertRaisesRegex(ValueError, "positive"): + SileroVad._split_long_segments([[100, 1100]], 0.5) + if __name__ == "__main__": unittest.main()