diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py index 9531afcd3..4b889984c 100644 --- a/funasr/models/contextual_paraformer/model.py +++ b/funasr/models/contextual_paraformer/model.py @@ -554,6 +554,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py index c1c3ba55a..7e295fc18 100644 --- a/funasr/models/seaco_paraformer/model.py +++ b/funasr/models/seaco_paraformer/model.py @@ -728,6 +728,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.py index c12ff4557..34cd6a098 100644 --- a/tests/test_paraformer_hotword_file.py +++ b/tests/test_paraformer_hotword_file.py @@ -82,3 +82,20 @@ def cp1252_default_open(file, mode="r", *args, **kwargs): hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n") assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]] + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_blank_lines_in_seg_dict_are_skipped(model_cls, tmp_path): + # A seg_dict with blank or whitespace-only lines (e.g. a trailing newline + # added while editing it) used to raise IndexError while loading. + (tmp_path / "seg_dict").write_text("阿 阿\n\n里 里\n \n巴 巴\n云 云\n\n", encoding="utf-8") + frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn")) + + hotword_list = model_cls.generate_hotwords_list( + types.SimpleNamespace(sos=SOS), + "阿里巴巴 阿里云", + tokenizer=_Tokenizer(), + frontend=frontend, + ) + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]