From 8060eeb840d63a65065d9bebc7dfbeeb26202318 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:32:33 +0530 Subject: [PATCH] fix(audio): average the channels of 2-D floating-point waveforms load_audio_text_image_video averages the channels of audio files, but in-memory numpy arrays and tensors passed to generate() were handed on unchanged. A (1, N) array from torchaudio.load or a stereo array made the FSMN-VAD frontend assert on the window size and, with (N, 1), raise a ZeroDivisionError. Reduce 2-D floating-point waveforms to mono (the shorter axis is the channel axis) before resampling, honoring reduce_channels like the file path. Empty 2-D input becomes empty 1-D; integer input is left as before. --- funasr/utils/load_utils.py | 17 ++++++ tests/test_load_audio_multichannel_arrays.py | 60 ++++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 tests/test_load_audio_multichannel_arrays.py diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec6..5e0bd20195 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -176,6 +176,23 @@ def load_audio_text_image_video( pass # print(f"unsupport data type: {data_or_path_or_list}, return raw data") + if ( + isinstance(data_or_path_or_list, torch.Tensor) + and data_or_path_or_list.ndim == 2 + and data_or_path_or_list.is_floating_point() + and data_type in (None, "sound") + and kwargs.get("reduce_channels", True) + ): + # In-memory waveforms follow either torchaudio's (channels, samples) or + # soundfile's (samples, channels) layout; the channel axis is the shorter one. + # Files are already reduced to mono above. + if data_or_path_or_list.numel() == 0: + data_or_path_or_list = data_or_path_or_list.reshape(-1) + else: + data_or_path_or_list = data_or_path_or_list.mean( + 0 if data_or_path_or_list.shape[0] <= data_or_path_or_list.shape[1] else 1 + ) + if audio_fs != fs and data_type != "text": if torchaudio is not None: resampler = torchaudio.transforms.Resample(audio_fs, fs) diff --git a/tests/test_load_audio_multichannel_arrays.py b/tests/test_load_audio_multichannel_arrays.py new file mode 100644 index 0000000000..16a9685c00 --- /dev/null +++ b/tests/test_load_audio_multichannel_arrays.py @@ -0,0 +1,60 @@ +import unittest + +import numpy as np +import torch + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestMultichannelWaveformInput(unittest.TestCase): + def setUp(self): + t = np.arange(1600, dtype=np.float32) + self.left = np.sin(t / 20.0).astype(np.float32) + self.right = 0.5 * np.sin(t / 7.0).astype(np.float32) + self.mono = (self.left + self.right) / 2 + + def _check(self, waveform, **kwargs): + out = load_audio_text_image_video(waveform, **kwargs) + self.assertEqual(tuple(out.shape), (1600,)) + np.testing.assert_allclose(out.numpy(), self.mono, atol=1e-6) + + def test_channels_first_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right])) + + def test_channels_last_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right], axis=1)) + + def test_channels_first_tensor_is_averaged(self): + self._check(torch.from_numpy(np.stack([self.left, self.right]))) + + def test_single_channel_2d_input_becomes_1d(self): + for waveform in (self.left[None, :], self.left[:, None]): + out = load_audio_text_image_video(waveform) + np.testing.assert_allclose(out.numpy(), self.left, atol=1e-6) + + def test_stereo_is_reduced_before_resampling(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), fs=16000, audio_fs=8000 + ) + self.assertEqual(out.ndim, 1) + self.assertEqual(out.shape[0], 3200) + + def test_reduce_channels_false_keeps_channels(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), reduce_channels=False + ) + self.assertEqual(tuple(out.shape), (2, 1600)) + + def test_fbank_features_are_not_averaged(self): + feats = np.random.default_rng(0).standard_normal((50, 80)).astype(np.float32) + out = load_audio_text_image_video(feats, data_type="fbank") + self.assertEqual(tuple(out.shape), (50, 80)) + + def test_empty_2d_input_becomes_empty_1d(self): + for shape in ((0, 2), (2, 0)): + out = load_audio_text_image_video(np.zeros(shape, dtype=np.float32)) + self.assertEqual(tuple(out.shape), (0,)) + + +if __name__ == "__main__": + unittest.main()