From d1887df1e6a9cf3553a2f7693df01a0614d78475 Mon Sep 17 00:00:00 2001 From: sohailshk <122166523+sohailshk@users.noreply.github.com> Date: Fri, 13 Nov 2026 09:33:40 +0530 Subject: [PATCH] fix(audio): scale int16 numpy waveforms to [-1, 1) in load_audio_text_image_video A np.int16 array (for example from PyAudio or sounddevice) was converted to an int16 tensor without scaling. The frontend multiplies float waveforms by 32768, so the signal was overdriven and VAD segments changed, and with a sample rate other than 16 kHz the resampler raised a TypeError. Scale int16 arrays by 1/32768 like load_bytes does for raw PCM. --- funasr/utils/load_utils.py | 3 ++ tests/test_load_audio_int16_array.py | 43 ++++++++++++++++++++++++++++ 2 files changed, 46 insertions(+) create mode 100644 tests/test_load_audio_int16_array.py diff --git a/funasr/utils/load_utils.py b/funasr/utils/load_utils.py index b689d65ec6..72d66f8d23 100644 --- a/funasr/utils/load_utils.py +++ b/funasr/utils/load_utils.py @@ -159,6 +159,9 @@ def load_audio_text_image_video( elif isinstance(data_or_path_or_list, str) and data_type == "text" and tokenizer is not None: data_or_path_or_list = tokenizer.encode(data_or_path_or_list) elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point + if data_or_path_or_list.dtype == np.int16 and data_type in (None, "sound"): + # PCM16 samples, as load_bytes and the kaldi_ark branch below read them. + data_or_path_or_list = data_or_path_or_list.astype(np.float32) / 32768.0 data_or_path_or_list = torch.from_numpy(data_or_path_or_list) # .squeeze() # [n_samples,] elif isinstance(data_or_path_or_list, str) and data_type == "kaldi_ark": data_mat = kaldiio.load_mat(data_or_path_or_list) diff --git a/tests/test_load_audio_int16_array.py b/tests/test_load_audio_int16_array.py new file mode 100644 index 0000000000..17aea2c310 --- /dev/null +++ b/tests/test_load_audio_int16_array.py @@ -0,0 +1,43 @@ +import unittest + +import numpy as np + +from funasr.utils.load_utils import load_audio_text_image_video + + +class TestInt16WaveformInput(unittest.TestCase): + def test_int16_array_is_scaled_to_float_range(self): + pcm = np.array([0, 16384, -16384, 32767, -32768], dtype=np.int16) + out = load_audio_text_image_video(pcm) + self.assertTrue(out.is_floating_point()) + np.testing.assert_allclose( + out.numpy(), [0.0, 0.5, -0.5, 32767 / 32768, -1.0], atol=1e-7 + ) + + def test_int16_array_matches_equivalent_float_array(self): + t = np.arange(16000, dtype=np.float32) + wave = (0.3 * np.sin(t / 15.0)).astype(np.float32) + pcm = np.round(wave * 32768.0).astype(np.int16) + from_pcm = load_audio_text_image_video(pcm).numpy() + from_float = load_audio_text_image_video(wave).numpy() + np.testing.assert_allclose(from_pcm, from_float, atol=1 / 32768) + + def test_float_array_is_unchanged(self): + wave = np.array([0.25, -0.5], dtype=np.float32) + out = load_audio_text_image_video(wave) + np.testing.assert_array_equal(out.numpy(), wave) + + def test_int16_array_with_resampling_does_not_fail(self): + pcm = (np.sin(np.arange(1600) / 10.0) * 10000).astype(np.int16) + out = load_audio_text_image_video(pcm, fs=16000, audio_fs=8000) + self.assertEqual(out.shape[0], 3200) + self.assertLessEqual(float(out.abs().max()), 1.0) + + def test_int16_array_that_is_not_sound_is_untouched(self): + feats = np.array([3, 5, 7], dtype=np.int16) + out = load_audio_text_image_video(feats, data_type="fbank") + np.testing.assert_array_equal(out.numpy(), feats) + + +if __name__ == "__main__": + unittest.main()