diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index be3009f46..352e5d6a8 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -14,6 +14,8 @@ on: - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_char_tokenizer_seg_dict.py" + - "funasr/datasets/audio_datasets/samplers.py" + - "tests/test_batch_sampler_drop_last.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -62,6 +64,8 @@ on: - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_char_tokenizer_seg_dict.py" + - "funasr/datasets/audio_datasets/samplers.py" + - "tests/test_batch_sampler_drop_last.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -151,6 +155,7 @@ jobs: tests/test_sentencepiece_tokenizer.py \ tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_char_tokenizer_seg_dict.py \ + tests/test_batch_sampler_drop_last.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/funasr/datasets/audio_datasets/samplers.py b/funasr/datasets/audio_datasets/samplers.py index 2cd41c15d..e0b2da47c 100644 --- a/funasr/datasets/audio_datasets/samplers.py +++ b/funasr/datasets/audio_datasets/samplers.py @@ -111,7 +111,10 @@ def __iter__(self): # Add extra samples to make it evenly divisible padding_size = self.total_size - len(indices) - if padding_size <= len(indices): + if padding_size < 0: + # drop_last: discard the tail that does not fill a batch on every replica. + indices = indices[: self.total_size] + elif padding_size <= len(indices): indices += indices[:padding_size] else: indices += ( @@ -139,7 +142,7 @@ def __iter__(self): ] # Drop the last batch if it's not full and drop_last is True - if self.drop_last and len(batches[-1]) != self.batch_size: + if self.drop_last and batches and len(batches[-1]) != self.batch_size: batches = batches[:-1] return iter(batches) @@ -225,7 +228,10 @@ def __iter__(self): # Add extra samples to make it evenly divisible padding_size = self.total_size - len(indices) - if padding_size <= len(indices): + if padding_size < 0: + # drop_last: discard the tail that does not fill a batch on every replica. + indices = indices[: self.total_size] + elif padding_size <= len(indices): indices += indices[:padding_size] else: indices += ( diff --git a/tests/test_batch_sampler_drop_last.py b/tests/test_batch_sampler_drop_last.py new file mode 100644 index 000000000..d61364caa --- /dev/null +++ b/tests/test_batch_sampler_drop_last.py @@ -0,0 +1,63 @@ +import unittest + +from funasr.datasets.audio_datasets.samplers import ( + CustomDistributedBatchSampler, + CustomDistributedBufferBatchSampler, +) + + +class _Dataset: + def __init__(self, n): + self.n = n + + def __len__(self): + return self.n + + def get_source_len(self, index): + return 1 + + +class TestBatchSamplerDropLast(unittest.TestCase): + def test_drop_last_keeps_only_full_batches(self): + sampler = CustomDistributedBatchSampler( + _Dataset(10), batch_size=4, drop_last=True, shuffle=False + ) + batches = list(sampler) + self.assertEqual(batches, [[0, 1, 2, 3], [4, 5, 6, 7]]) + self.assertEqual(len(sampler), len(batches)) + + def test_drop_last_with_shuffle_never_repeats_an_example(self): + sampler = CustomDistributedBatchSampler( + _Dataset(10), batch_size=4, drop_last=True, shuffle=True + ) + seen = [i for batch in sampler for i in batch] + self.assertEqual(len(seen), 8) + self.assertEqual(len(set(seen)), 8) + + def test_drop_last_with_fewer_examples_than_one_batch_yields_nothing(self): + sampler = CustomDistributedBatchSampler( + _Dataset(3), batch_size=4, drop_last=True, shuffle=False + ) + self.assertEqual(list(sampler), []) + + def test_without_drop_last_the_last_batch_is_padded(self): + sampler = CustomDistributedBatchSampler( + _Dataset(10), batch_size=4, drop_last=False, shuffle=False + ) + self.assertEqual(list(sampler), [[0, 1, 2, 3], [4, 5, 6, 7], [8, 9, 0, 1]]) + + def test_buffer_sampler_drop_last_keeps_only_full_batches(self): + sampler = CustomDistributedBufferBatchSampler( + _Dataset(10), batch_size=4, drop_last=True, shuffle=False, sort_size=1024 + ) + self.assertEqual(list(sampler), [[0, 1, 2, 3], [4, 5, 6, 7]]) + + def test_buffer_sampler_drop_last_with_fewer_examples_than_one_batch(self): + sampler = CustomDistributedBufferBatchSampler( + _Dataset(3), batch_size=4, drop_last=True, shuffle=False, sort_size=1024 + ) + self.assertEqual(list(sampler), []) + + +if __name__ == "__main__": + unittest.main()