From 8060eeb840d63a65065d9bebc7dfbeeb26202318 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:32:33 +0530 Subject: [PATCH 1/2] fix(audio): average the channels of 2-D floating-point waveforms load_audio_text_image_video averages the channels of audio files, but in-memory numpy arrays and tensors passed to generate() were handed on unchanged. A (1, N) array from torchaudio.load or a stereo array made the FSMN-VAD frontend assert on the window size and, with (N, 1), raise a ZeroDivisionError. Reduce 2-D floating-point waveforms to mono (the shorter axis is the channel axis) before resampling, honoring reduce_channels like the file path. Empty 2-D input becomes empty 1-D; integer input is left as before. --- funasr/utils/load_utils.py | 17 ++++++ tests/test_load_audio_multichannel_arrays.py | 60 ++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tests/test_load_audio_multichannel_arrays.py diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec6..5e0bd20195 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -176,6 +176,23 @@ def load_audio_text_image_video( pass # print(f"unsupport data type: {data_or_path_or_list}, return raw data") + if ( + isinstance(data_or_path_or_list, torch.Tensor) + and data_or_path_or_list.ndim == 2 + and data_or_path_or_list.is_floating_point() + and data_type in (None, "sound") + and kwargs.get("reduce_channels", True) + ): + # In-memory waveforms follow either torchaudio's (channels, samples) or + # soundfile's (samples, channels) layout; the channel axis is the shorter one. + # Files are already reduced to mono above. + if data_or_path_or_list.numel() == 0: + data_or_path_or_list = data_or_path_or_list.reshape(-1) + else: + data_or_path_or_list = data_or_path_or_list.mean( + 0 if data_or_path_or_list.shape[0] <= data_or_path_or_list.shape[1] else 1 + ) + if audio_fs != fs and data_type != "text": if torchaudio is not None: resampler = torchaudio.transforms.Resample(audio_fs, fs) diff --git a/tests/test_load_audio_multichannel_arrays.py b/tests/test_load_audio_multichannel_arrays.py new file mode 100644 index 0000000000..16a9685c00 --- /dev/null +++ b/tests/test_load_audio_multichannel_arrays.py @@ -0,0 +1,60 @@ +import unittest + +import numpy as np +import torch + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestMultichannelWaveformInput(unittest.TestCase): + def setUp(self): + t = np.arange(1600, dtype=np.float32) + self.left = np.sin(t / 20.0).astype(np.float32) + self.right = 0.5 * np.sin(t / 7.0).astype(np.float32) + self.mono = (self.left + self.right) / 2 + + def _check(self, waveform, **kwargs): + out = load_audio_text_image_video(waveform, **kwargs) + self.assertEqual(tuple(out.shape), (1600,)) + np.testing.assert_allclose(out.numpy(), self.mono, atol=1e-6) + + def test_channels_first_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right])) + + def test_channels_last_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right], axis=1)) + + def test_channels_first_tensor_is_averaged(self): + self._check(torch.from_numpy(np.stack([self.left, self.right]))) + + def test_single_channel_2d_input_becomes_1d(self): + for waveform in (self.left[None, :], self.left[:, None]): + out = load_audio_text_image_video(waveform) + np.testing.assert_allclose(out.numpy(), self.left, atol=1e-6) + + def test_stereo_is_reduced_before_resampling(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), fs=16000, audio_fs=8000 + ) + self.assertEqual(out.ndim, 1) + self.assertEqual(out.shape[0], 3200) + + def test_reduce_channels_false_keeps_channels(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), reduce_channels=False + ) + self.assertEqual(tuple(out.shape), (2, 1600)) + + def test_fbank_features_are_not_averaged(self): + feats = np.random.default_rng(0).standard_normal((50, 80)).astype(np.float32) + out = load_audio_text_image_video(feats, data_type="fbank") + self.assertEqual(tuple(out.shape), (50, 80)) + + def test_empty_2d_input_becomes_empty_1d(self): + for shape in ((0, 2), (2, 0)): + out = load_audio_text_image_video(np.zeros(shape, dtype=np.float32)) + self.assertEqual(tuple(out.shape), (0,)) + + +if __name__ == "__main__": + unittest.main() From d1887df1e6a9cf3553a2f7693df01a0614d78475 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:33:40 +0530 Subject: [PATCH 2/2] fix(audio): scale int16 numpy waveforms to [-1, 1) in load_audio_text_image_video A np.int16 array (for example from PyAudio or sounddevice) was converted to an int16 tensor without scaling. The frontend multiplies float waveforms by 32768, so the signal was overdriven and VAD segments changed, and with a sample rate other than 16 kHz the resampler raised a TypeError. Scale int16 arrays by 1/32768 like load_bytes does for raw PCM. --- funasr/utils/load_utils.py | 3 ++ tests/test_load_audio_int16_array.py | 43 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/test_load_audio_int16_array.py diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec6..72d66f8d23 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -159,6 +159,9 @@ def load_audio_text_image_video( elif isinstance(data_or_path_or_list, str) and data_type == "text" and tokenizer is not None: data_or_path_or_list = tokenizer.encode(data_or_path_or_list) elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point + if data_or_path_or_list.dtype == np.int16 and data_type in (None, "sound"): + # PCM16 samples, as load_bytes and the kaldi_ark branch below read them. + data_or_path_or_list = data_or_path_or_list.astype(np.float32) / 32768.0 data_or_path_or_list = torch.from_numpy(data_or_path_or_list) # .squeeze() # [n_samples,] elif isinstance(data_or_path_or_list, str) and data_type == "kaldi_ark": data_mat = kaldiio.load_mat(data_or_path_or_list) diff --git a/tests/test_load_audio_int16_array.py b/tests/test_load_audio_int16_array.py new file mode 100644 index 0000000000..17aea2c310 --- /dev/null +++ b/tests/test_load_audio_int16_array.py @@ -0,0 +1,43 @@ +import unittest + +import numpy as np + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestInt16WaveformInput(unittest.TestCase): + def test_int16_array_is_scaled_to_float_range(self): + pcm = np.array([0, 16384, -16384, 32767, -32768], dtype=np.int16) + out = load_audio_text_image_video(pcm) + self.assertTrue(out.is_floating_point()) + np.testing.assert_allclose( + out.numpy(), [0.0, 0.5, -0.5, 32767 / 32768, -1.0], atol=1e-7 + ) + + def test_int16_array_matches_equivalent_float_array(self): + t = np.arange(16000, dtype=np.float32) + wave = (0.3 * np.sin(t / 15.0)).astype(np.float32) + pcm = np.round(wave * 32768.0).astype(np.int16) + from_pcm = load_audio_text_image_video(pcm).numpy() + from_float = load_audio_text_image_video(wave).numpy() + np.testing.assert_allclose(from_pcm, from_float, atol=1 / 32768) + + def test_float_array_is_unchanged(self): + wave = np.array([0.25, -0.5], dtype=np.float32) + out = load_audio_text_image_video(wave) + np.testing.assert_array_equal(out.numpy(), wave) + + def test_int16_array_with_resampling_does_not_fail(self): + pcm = (np.sin(np.arange(1600) / 10.0) * 10000).astype(np.int16) + out = load_audio_text_image_video(pcm, fs=16000, audio_fs=8000) + self.assertEqual(out.shape[0], 3200) + self.assertLessEqual(float(out.abs().max()), 1.0) + + def test_int16_array_that_is_not_sound_is_untouched(self): + feats = np.array([3, 5, 7], dtype=np.int16) + out = load_audio_text_image_video(feats, data_type="fbank") + np.testing.assert_array_equal(out.numpy(), feats) + + +if __name__ == "__main__": + unittest.main()