From 1142e8283c24d9fd42c8531e3774f3207078a3ca Mon Sep 17 00:00:00 2001 From: Mohammad Hijjawi Date: Mon, 16 Nov 2026 00:18:42 +0000 Subject: [PATCH] fix(paraformer): skip blank lines when loading the hotword seg_dict SeACo-Paraformer and Contextual Paraformer load the model directory's seg_dict with their own copy of load_seg_dict, which read the first field of every line without checking it. A blank or whitespace-only line made generate_hotwords_list raise IndexError whenever hotwords were passed. Skip empty lines, as funasr.tokenizer.char_tokenizer.load_seg_dict does. --- funasr/models/contextual_paraformer/model.py | 2 ++ funasr/models/seaco_paraformer/model.py | 2 ++ tests/test_paraformer_hotword_file.py | 17 +++++++++++++++++ 3 files changed, 21 insertions(+) diff --git a/funasr/models/contextual_paraformer/model.py b/funasr/models/contextual_paraformer/model.py index 9531afcd30..4b889984c2 100644 --- a/funasr/models/contextual_paraformer/model.py +++ b/funasr/models/contextual_paraformer/model.py @@ -554,6 +554,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/funasr/models/seaco_paraformer/model.py b/funasr/models/seaco_paraformer/model.py index c1c3ba55a2..7e295fc188 100644 --- a/funasr/models/seaco_paraformer/model.py +++ b/funasr/models/seaco_paraformer/model.py @@ -728,6 +728,8 @@ def load_seg_dict(seg_dict_file): lines = f.readlines() for line in lines: s = line.strip().split() + if not s: + continue key = s[0] value = s[1:] seg_dict[key] = " ".join(value) diff --git a/tests/test_paraformer_hotword_file.py b/tests/test_paraformer_hotword_file.py index c12ff45575..34cd6a0984 100644 --- a/tests/test_paraformer_hotword_file.py +++ b/tests/test_paraformer_hotword_file.py @@ -82,3 +82,20 @@ def cp1252_default_open(file, mode="r", *args, **kwargs): hotword_list = _generate(model_cls, tmp_path, "阿里巴巴\n阿里云\n") assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]] + + +@pytest.mark.parametrize("model_cls", [SeacoParaformer, ContextualParaformer]) +def test_blank_lines_in_seg_dict_are_skipped(model_cls, tmp_path): + # A seg_dict with blank or whitespace-only lines (e.g. a trailing newline + # added while editing it) used to raise IndexError while loading. + (tmp_path / "seg_dict").write_text("阿 阿\n\n里 里\n \n巴 巴\n云 云\n\n", encoding="utf-8") + frontend = types.SimpleNamespace(cmvn_file=os.path.join(str(tmp_path), "am.mvn")) + + hotword_list = model_cls.generate_hotwords_list( + types.SimpleNamespace(sos=SOS), + "阿里巴巴 阿里云", + tokenizer=_Tokenizer(), + frontend=frontend, + ) + + assert hotword_list == [[2, 3, 4, 4], [2, 3, 5], [SOS]]