diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index d62d0f4cd..be3009f46 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -13,6 +13,7 @@ on: - "funasr/tokenizer/char_tokenizer.py" - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" + - "tests/test_char_tokenizer_seg_dict.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -60,6 +61,7 @@ on: - "funasr/tokenizer/char_tokenizer.py" - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" + - "tests/test_char_tokenizer_seg_dict.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -148,6 +150,7 @@ jobs: tests/test_sensevoice_word_timestamps.py \ tests/test_sentencepiece_tokenizer.py \ tests/test_char_tokenizer_non_linguistic_symbols.py \ + tests/test_char_tokenizer_seg_dict.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index 69e8cd3a4..1165ba533 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -114,6 +114,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/tests/test_char_tokenizer_seg_dict.py b/tests/test_char_tokenizer_seg_dict.py new file mode 100644 index 000000000..72f93752b --- /dev/null +++ b/tests/test_char_tokenizer_seg_dict.py @@ -0,0 +1,22 @@ +"""CharTokenizer must skip blank lines in a seg_dict file instead of raising IndexError.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer, load_seg_dict + + +class LoadSegDictBlankLineTest(unittest.TestCase): + def test_blank_lines_are_skipped(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "seg_dict") + with open(path, "w", encoding="utf-8") as f: + f.write("你 你\n\n好 好\n \nhello he@@ llo\n\n") + self.assertEqual(load_seg_dict(path), {"你": "你", "好": "好", "hello": "he@@ llo"}) + tokenizer = CharTokenizer(seg_dict=path) + self.assertEqual(tokenizer.text2tokens("你好 hello"), ["你", "好", "he@@", "llo"]) + + +if __name__ == "__main__": + unittest.main()