diff --git a/.github/workflows/test-numpy-compatibility.yml b/.github/workflows/test-numpy-compatibility.yml index be3009f46..5a9d02c38 100644 --- a/.github/workflows/test-numpy-compatibility.yml +++ b/.github/workflows/test-numpy-compatibility.yml @@ -14,6 +14,8 @@ on: - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_char_tokenizer_seg_dict.py" + - "funasr/tokenizer/cleaner.py" + - "tests/test_text_cleaner_korean.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -62,6 +64,8 @@ on: - "funasr/tokenizer/phoneme_tokenizer.py" - "tests/test_char_tokenizer_non_linguistic_symbols.py" - "tests/test_char_tokenizer_seg_dict.py" + - "funasr/tokenizer/cleaner.py" + - "tests/test_text_cleaner_korean.py" - "tests/test_sentencepiece_tokenizer.py" - "tests/test_speaker_optional_dependency.py" - "funasr/utils/load_utils.py" @@ -151,6 +155,7 @@ jobs: tests/test_sentencepiece_tokenizer.py \ tests/test_char_tokenizer_non_linguistic_symbols.py \ tests/test_char_tokenizer_seg_dict.py \ + tests/test_text_cleaner_korean.py \ tests/test_pcm_input_format.py \ tests/test_realtime_ws_service.py \ tests/test_realtime_ws_hf_overrides.py \ diff --git a/funasr/tokenizer/cleaner.py b/funasr/tokenizer/cleaner.py index 1c609caf0..962554af2 100644 --- a/funasr/tokenizer/cleaner.py +++ b/funasr/tokenizer/cleaner.py @@ -2,6 +2,8 @@ from jaconv import jaconv +from funasr.tokenizer.korean_cleaner import KoreanCleaner + # import tacotron_cleaner.cleaners try: diff --git a/tests/test_text_cleaner_korean.py b/tests/test_text_cleaner_korean.py new file mode 100644 index 000000000..8b578b3ec --- /dev/null +++ b/tests/test_text_cleaner_korean.py @@ -0,0 +1,21 @@ +import unittest + +from funasr.tokenizer.cleaner import TextCleaner + + +class TestKoreanTextCleaner(unittest.TestCase): + def test_korean_cleaner_spells_out_digits_and_latin_letters(self): + cleaner = TextCleaner("korean_cleaner") + self.assertEqual(cleaner(" abc 12 "), "에이비씨 일이") + + def test_korean_cleaner_leaves_hangul_unchanged(self): + cleaner = TextCleaner(["korean_cleaner"]) + self.assertEqual(cleaner("안녕하세요"), "안녕하세요") + + def test_unsupported_cleaner_type_is_still_rejected(self): + with self.assertRaises(RuntimeError): + TextCleaner("no_such_cleaner")("text") + + +if __name__ == "__main__": + unittest.main()