From bf61bd7d86655a4afbbe21875a3954ab0cd6b6c0 Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi Date: Sat, 14 Nov 2026 19:09:57 +0000 Subject: [PATCH] fix(tokenizer): skip blank lines when loading a seg_dict file load_seg_dict indexed the first field of every line, so a blank or whitespace-only line (for example a trailing empty line) raised IndexError and CharTokenizer(seg_dict=...) could not be built. --- funasr/tokenizer/char_tokenizer.py | 2 ++ tests/test_char_tokenizer_seg_dict.py | 22 ++++++++++++++++++++++ 2 files changed, 24 insertions(+) create mode 100644 tests/test_char_tokenizer_seg_dict.py diff --git a/funasr/tokenizer/char_tokenizer.py b/funasr/tokenizer/char_tokenizer.py index 69e8cd3a4..1165ba533 100644 --- a/funasr/tokenizer/char_tokenizer.py +++ b/funasr/tokenizer/char_tokenizer.py @@ -114,6 +114,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/tests/test_char_tokenizer_seg_dict.py b/tests/test_char_tokenizer_seg_dict.py new file mode 100644 index 000000000..72f93752b --- /dev/null +++ b/tests/test_char_tokenizer_seg_dict.py @@ -0,0 +1,22 @@ +"""CharTokenizer must skip blank lines in a seg_dict file instead of raising IndexError.""" + +import os +import tempfile +import unittest + +from funasr.tokenizer.char_tokenizer import CharTokenizer, load_seg_dict + + +class LoadSegDictBlankLineTest(unittest.TestCase): + def test_blank_lines_are_skipped(self): + with tempfile.TemporaryDirectory() as tmpdir: + path = os.path.join(tmpdir, "seg_dict") + with open(path, "w", encoding="utf-8") as f: + f.write("你 你\n\n好 好\n \nhello he@@ llo\n\n") + self.assertEqual(load_seg_dict(path), {"你": "你", "好": "好", "hello": "he@@ llo"}) + tokenizer = CharTokenizer(seg_dict=path) + self.assertEqual(tokenizer.text2tokens("你好 hello"), ["你", "好", "he@@", "llo"]) + + +if __name__ == "__main__": + unittest.main()