diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec..72d66f8d2 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -159,6 +159,9 @@ def load_audio_text_image_video( elif isinstance(data_or_path_or_list, str) and data_type == "text" and tokenizer is not None: data_or_path_or_list = tokenizer.encode(data_or_path_or_list) elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point + if data_or_path_or_list.dtype == np.int16 and data_type in (None, "sound"): + # PCM16 samples, as load_bytes and the kaldi_ark branch below read them. + data_or_path_or_list = data_or_path_or_list.astype(np.float32) / 32768.0 data_or_path_or_list = torch.from_numpy(data_or_path_or_list) # .squeeze() # [n_samples,] elif isinstance(data_or_path_or_list, str) and data_type == "kaldi_ark": data_mat = kaldiio.load_mat(data_or_path_or_list) diff --git a/tests/test_load_audio_int16_array.py b/tests/test_load_audio_int16_array.py new file mode 100644 index 000000000..17aea2c31 --- /dev/null +++ b/tests/test_load_audio_int16_array.py @@ -0,0 +1,43 @@ +import unittest + +import numpy as np + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestInt16WaveformInput(unittest.TestCase): + def test_int16_array_is_scaled_to_float_range(self): + pcm = np.array([0, 16384, -16384, 32767, -32768], dtype=np.int16) + out = load_audio_text_image_video(pcm) + self.assertTrue(out.is_floating_point()) + np.testing.assert_allclose( + out.numpy(), [0.0, 0.5, -0.5, 32767 / 32768, -1.0], atol=1e-7 + ) + + def test_int16_array_matches_equivalent_float_array(self): + t = np.arange(16000, dtype=np.float32) + wave = (0.3 * np.sin(t / 15.0)).astype(np.float32) + pcm = np.round(wave * 32768.0).astype(np.int16) + from_pcm = load_audio_text_image_video(pcm).numpy() + from_float = load_audio_text_image_video(wave).numpy() + np.testing.assert_allclose(from_pcm, from_float, atol=1 / 32768) + + def test_float_array_is_unchanged(self): + wave = np.array([0.25, -0.5], dtype=np.float32) + out = load_audio_text_image_video(wave) + np.testing.assert_array_equal(out.numpy(), wave) + + def test_int16_array_with_resampling_does_not_fail(self): + pcm = (np.sin(np.arange(1600) / 10.0) * 10000).astype(np.int16) + out = load_audio_text_image_video(pcm, fs=16000, audio_fs=8000) + self.assertEqual(out.shape[0], 3200) + self.assertLessEqual(float(out.abs().max()), 1.0) + + def test_int16_array_that_is_not_sound_is_untouched(self): + feats = np.array([3, 5, 7], dtype=np.int16) + out = load_audio_text_image_video(feats, data_type="fbank") + np.testing.assert_array_equal(out.numpy(), feats) + + +if __name__ == "__main__": + unittest.main()