diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index 370553668..bfeeff440 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -30,6 +30,8 @@ on: - "tests/test_ct_transformer_streaming_empty_input.py" - "tests/test_pcm_input_format.py" - "tests/test_load_audio_bytes.py" + - "tests/test_load_audio_int16_array.py" + - "tests/test_load_audio_multichannel_arrays.py" - "funasr/frontends/**" - "funasr/models/normalize/utterance_mvn.py" - "tests/test_utterance_mvn.py" @@ -74,6 +76,8 @@ on: - "tests/test_ct_transformer_streaming_empty_input.py" - "tests/test_pcm_input_format.py" - "tests/test_load_audio_bytes.py" + - "tests/test_load_audio_int16_array.py" + - "tests/test_load_audio_multichannel_arrays.py" - "funasr/frontends/**" - "funasr/models/normalize/utterance_mvn.py" - "tests/test_utterance_mvn.py" @@ -146,6 +150,8 @@ jobs: tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ tests/test_load_audio_bytes.py \ + tests/test_load_audio_int16_array.py \ + tests/test_load_audio_multichannel_arrays.py \ tests/test_timestamp_tools.py \ tests/test_punc_model_none.py \ tests/test_paraformer_empty_token_batch.py \ diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec..8dfcdb952 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -159,6 +159,9 @@ def load_audio_text_image_video( elif isinstance(data_or_path_or_list, str) and data_type == "text" and tokenizer is not None: data_or_path_or_list = tokenizer.encode(data_or_path_or_list) elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point + if data_or_path_or_list.dtype == np.int16 and data_type in (None, "sound"): + # PCM16 samples, as load_bytes and the kaldi_ark branch below read them. + data_or_path_or_list = data_or_path_or_list.astype(np.float32) / 32768.0 data_or_path_or_list = torch.from_numpy(data_or_path_or_list) # .squeeze() # [n_samples,] elif isinstance(data_or_path_or_list, str) and data_type == "kaldi_ark": data_mat = kaldiio.load_mat(data_or_path_or_list) @@ -176,6 +179,23 @@ def load_audio_text_image_video( pass # print(f"unsupport data type: {data_or_path_or_list}, return raw data") + if ( + isinstance(data_or_path_or_list, torch.Tensor) + and data_or_path_or_list.ndim == 2 + and data_or_path_or_list.is_floating_point() + and data_type in (None, "sound") + and kwargs.get("reduce_channels", True) + ): + # In-memory waveforms follow either torchaudio's (channels, samples) or + # soundfile's (samples, channels) layout; the channel axis is the shorter one. + # Files are already reduced to mono above. + if data_or_path_or_list.numel() == 0: + data_or_path_or_list = data_or_path_or_list.reshape(-1) + else: + data_or_path_or_list = data_or_path_or_list.mean( + 0 if data_or_path_or_list.shape[0] <= data_or_path_or_list.shape[1] else 1 + ) + if audio_fs != fs and data_type != "text": if torchaudio is not None: resampler = torchaudio.transforms.Resample(audio_fs, fs) diff --git a/tests/test_load_audio_int16_array.py b/tests/test_load_audio_int16_array.py new file mode 100644 index 000000000..17aea2c31 --- /dev/null +++ b/tests/test_load_audio_int16_array.py @@ -0,0 +1,43 @@ +import unittest + +import numpy as np + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestInt16WaveformInput(unittest.TestCase): + def test_int16_array_is_scaled_to_float_range(self): + pcm = np.array([0, 16384, -16384, 32767, -32768], dtype=np.int16) + out = load_audio_text_image_video(pcm) + self.assertTrue(out.is_floating_point()) + np.testing.assert_allclose( + out.numpy(), [0.0, 0.5, -0.5, 32767 / 32768, -1.0], atol=1e-7 + ) + + def test_int16_array_matches_equivalent_float_array(self): + t = np.arange(16000, dtype=np.float32) + wave = (0.3 * np.sin(t / 15.0)).astype(np.float32) + pcm = np.round(wave * 32768.0).astype(np.int16) + from_pcm = load_audio_text_image_video(pcm).numpy() + from_float = load_audio_text_image_video(wave).numpy() + np.testing.assert_allclose(from_pcm, from_float, atol=1 / 32768) + + def test_float_array_is_unchanged(self): + wave = np.array([0.25, -0.5], dtype=np.float32) + out = load_audio_text_image_video(wave) + np.testing.assert_array_equal(out.numpy(), wave) + + def test_int16_array_with_resampling_does_not_fail(self): + pcm = (np.sin(np.arange(1600) / 10.0) * 10000).astype(np.int16) + out = load_audio_text_image_video(pcm, fs=16000, audio_fs=8000) + self.assertEqual(out.shape[0], 3200) + self.assertLessEqual(float(out.abs().max()), 1.0) + + def test_int16_array_that_is_not_sound_is_untouched(self): + feats = np.array([3, 5, 7], dtype=np.int16) + out = load_audio_text_image_video(feats, data_type="fbank") + np.testing.assert_array_equal(out.numpy(), feats) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_load_audio_multichannel_arrays.py b/tests/test_load_audio_multichannel_arrays.py new file mode 100644 index 000000000..16a9685c0 --- /dev/null +++ b/tests/test_load_audio_multichannel_arrays.py @@ -0,0 +1,60 @@ +import unittest + +import numpy as np +import torch + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestMultichannelWaveformInput(unittest.TestCase): + def setUp(self): + t = np.arange(1600, dtype=np.float32) + self.left = np.sin(t / 20.0).astype(np.float32) + self.right = 0.5 * np.sin(t / 7.0).astype(np.float32) + self.mono = (self.left + self.right) / 2 + + def _check(self, waveform, **kwargs): + out = load_audio_text_image_video(waveform, **kwargs) + self.assertEqual(tuple(out.shape), (1600,)) + np.testing.assert_allclose(out.numpy(), self.mono, atol=1e-6) + + def test_channels_first_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right])) + + def test_channels_last_ndarray_is_averaged(self): + self._check(np.stack([self.left, self.right], axis=1)) + + def test_channels_first_tensor_is_averaged(self): + self._check(torch.from_numpy(np.stack([self.left, self.right]))) + + def test_single_channel_2d_input_becomes_1d(self): + for waveform in (self.left[None, :], self.left[:, None]): + out = load_audio_text_image_video(waveform) + np.testing.assert_allclose(out.numpy(), self.left, atol=1e-6) + + def test_stereo_is_reduced_before_resampling(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), fs=16000, audio_fs=8000 + ) + self.assertEqual(out.ndim, 1) + self.assertEqual(out.shape[0], 3200) + + def test_reduce_channels_false_keeps_channels(self): + out = load_audio_text_image_video( + np.stack([self.left, self.right]), reduce_channels=False + ) + self.assertEqual(tuple(out.shape), (2, 1600)) + + def test_fbank_features_are_not_averaged(self): + feats = np.random.default_rng(0).standard_normal((50, 80)).astype(np.float32) + out = load_audio_text_image_video(feats, data_type="fbank") + self.assertEqual(tuple(out.shape), (50, 80)) + + def test_empty_2d_input_becomes_empty_1d(self): + for shape in ((0, 2), (2, 0)): + out = load_audio_text_image_video(np.zeros(shape, dtype=np.float32)) + self.assertEqual(tuple(out.shape), (0,)) + + +if __name__ == "__main__": + unittest.main()