Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions funasr/utils/load_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -159,6 +159,9 @@ def load_audio_text_image_video(
elif isinstance(data_or_path_or_list, str) and data_type == "text" and tokenizer is not None:
data_or_path_or_list = tokenizer.encode(data_or_path_or_list)
elif isinstance(data_or_path_or_list, np.ndarray): # audio sample point
if data_or_path_or_list.dtype == np.int16 and data_type in (None, "sound"):
# PCM16 samples, as load_bytes and the kaldi_ark branch below read them.
data_or_path_or_list = data_or_path_or_list.astype(np.float32) / 32768.0
data_or_path_or_list = torch.from_numpy(data_or_path_or_list) # .squeeze() # [n_samples,]
elif isinstance(data_or_path_or_list, str) and data_type == "kaldi_ark":
data_mat = kaldiio.load_mat(data_or_path_or_list)
Expand Down
43 changes: 43 additions & 0 deletions tests/test_load_audio_int16_array.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
import unittest

import numpy as np

from funasr.utils.load_utils import load_audio_text_image_video


class TestInt16WaveformInput(unittest.TestCase):
def test_int16_array_is_scaled_to_float_range(self):
pcm = np.array([0, 16384, -16384, 32767, -32768], dtype=np.int16)
out = load_audio_text_image_video(pcm)
self.assertTrue(out.is_floating_point())
np.testing.assert_allclose(
out.numpy(), [0.0, 0.5, -0.5, 32767 / 32768, -1.0], atol=1e-7
)

def test_int16_array_matches_equivalent_float_array(self):
t = np.arange(16000, dtype=np.float32)
wave = (0.3 * np.sin(t / 15.0)).astype(np.float32)
pcm = np.round(wave * 32768.0).astype(np.int16)
from_pcm = load_audio_text_image_video(pcm).numpy()
from_float = load_audio_text_image_video(wave).numpy()
np.testing.assert_allclose(from_pcm, from_float, atol=1 / 32768)

def test_float_array_is_unchanged(self):
wave = np.array([0.25, -0.5], dtype=np.float32)
out = load_audio_text_image_video(wave)
np.testing.assert_array_equal(out.numpy(), wave)

def test_int16_array_with_resampling_does_not_fail(self):
pcm = (np.sin(np.arange(1600) / 10.0) * 10000).astype(np.int16)
out = load_audio_text_image_video(pcm, fs=16000, audio_fs=8000)
self.assertEqual(out.shape[0], 3200)
self.assertLessEqual(float(out.abs().max()), 1.0)

def test_int16_array_that_is_not_sound_is_untouched(self):
feats = np.array([3, 5, 7], dtype=np.int16)
out = load_audio_text_image_video(feats, data_type="fbank")
np.testing.assert_array_equal(out.numpy(), feats)


if __name__ == "__main__":
unittest.main()