fixed jieba bug (#219)

Author: mortii

ankimorphs/jieba_wrapper.py

@@ -5,30 +5,30 @@
 import sys
 from types import ModuleType
 
-posseg: ModuleType | None = None
+from .morpheme import Morpheme
 
+posseg: ModuleType | None = None
 successful_startup: bool = False
 
 ################################################################################
-# This section about cjk_ideographs is from zhon/hanzi.py
-# zhon: https://github.com/tsroten/zhon
+# This section about cjk_ideographs is based on zhon/hanzi.py in:
+# https://github.com/tsroten/zhon
 ################################################################################
 
 #: Character code ranges for pertinent CJK ideograph Unicode blocks.
-# cjk_ideographs = (
-CJK_IDEOGRAPHS: str = (
-    "\u3007"  # Ideographic number zero, see issue #17
-    "\u4E00-\u9FFF"  # CJK Unified Ideographs
-    "\u3400-\u4DBF"  # CJK Unified Ideographs Extension A
-    "\uF900-\uFAFF"  # CJK Compatibility Ideographs
-)
+cjk_ideograph_unicode_ranges = [
+    (0x3007, 0x3007),  # Ideographic number zero
+    (0x4E00, 0x9FFF),  # CJK Unified Ideographs
+    (0x3400, 0x4DBF),  # CJK Unified Ideographs Extension A
+    (0xF900, 0xFAFF),  # CJK Compatibility Ideographs
+]
 if sys.maxunicode > 0xFFFF:
-    CJK_IDEOGRAPHS += (
-        "\U00020000-\U0002A6DF"  # CJK Unified Ideographs Extension B
-        "\U0002A700-\U0002B73F"  # CJK Unified Ideographs Extension C
-        "\U0002B740-\U0002B81F"  # CJK Unified Ideographs Extension D
-        "\U0002F800-\U0002FA1F"  # CJK Compatibility Ideographs Supplement
-    )
+    cjk_ideograph_unicode_ranges += [
+        (0x20000, 0x2A6DF),  # CJK Unified Ideographs Extension B
+        (0x2A700, 0x2B73F),  # CJK Unified Ideographs Extension C
+        (0x2B740, 0x2B81F),  # CJK Unified Ideographs Extension D
+        (0x2F800, 0x2FA1F),  # CJK Compatibility Ideographs Supplement
+    ]
 ################################################################################
 
 
@@ -43,3 +43,34 @@ def import_jieba() -> None:
         return
 
     successful_startup = True
+
+
+def get_morphemes_jieba(expression: str) -> list[Morpheme]:
+    assert posseg is not None
+    _morphs: list[Morpheme] = []
+
+    # The "posseg.cut" function returns "Pair" instances:
+    #   Pair.word
+    #   Pair.flag  # part of speech
+    for posseg_pair in posseg.cut(expression):
+        if text_contains_only_cjk_ranges(_text=posseg_pair.word) is False:
+            continue
+
+        # chinese does not have inflections, so we use the lemma for both
+        _morphs.append(Morpheme(lemma=posseg_pair.word, inflection=posseg_pair.word))
+
+    return _morphs
+
+
+def char_found_in_cjk_ranges(_char: str) -> bool:
+    for start, end in cjk_ideograph_unicode_ranges:
+        if start <= ord(_char) <= end:
+            return True
+    return False
+
+
+def text_contains_only_cjk_ranges(_text: str) -> bool:
+    for char in _text:
+        if not char_found_in_cjk_ranges(char):
+            return False
+    return True

ankimorphs/morphemizer.py

@@ -4,9 +4,11 @@
 import re
 
 from . import jieba_wrapper, mecab_wrapper, spacy_wrapper
-from .mecab_wrapper import get_morphemes_mecab
 from .morpheme import Morpheme
 
+space_char_regex = re.compile(" ")
+
+
 ####################################################################################################
 # Base Class
 ####################################################################################################
@@ -80,8 +82,6 @@ def get_morphemizer_by_description(description: str) -> Morphemizer | None:
 # Mecab Morphemizer
 ####################################################################################################
 
-space_char_regex = re.compile(" ")
-
 
 class MecabMorphemizer(Morphemizer):
 
@@ -93,7 +93,7 @@ def _get_morphemes_from_expr(self, expression: str) -> list[Morpheme]:
         # Remove simple spaces that could be added by other add-ons and break the parsing.
         if space_char_regex.search(expression):
             expression = space_char_regex.sub("", expression)
-        return get_morphemes_mecab(expression)
+        return mecab_wrapper.get_morphemes_mecab(expression)
 
     def get_description(self) -> str:
         return "AnkiMorphs: Japanese"
@@ -166,23 +166,7 @@ def __init__(self) -> None:
         jieba_wrapper.import_jieba()
 
     def _get_morphemes_from_expr(self, expression: str) -> list[Morpheme]:
-        assert jieba_wrapper.posseg is not None
-        expression_morphs: list[Morpheme] = []
-
-        # only retain the cjk ideographs
-        expression = "".join(
-            re.findall(
-                f"[{jieba_wrapper.CJK_IDEOGRAPHS}]",
-                expression,
-            )
-        )
-
-        for jieba_segment in jieba_wrapper.posseg.cut(expression):
-            # chinese does not have inflections, so we use the lemma for both
-            _morph = Morpheme(lemma=jieba_segment.word, inflection=jieba_segment.word)
-            expression_morphs.append(_morph)
-
-        return expression_morphs
+        return jieba_wrapper.get_morphemes_jieba(expression)
 
     def get_description(self) -> str:
         return "AnkiMorphs: Chinese"

tests/mecab_and_jieba_test.py

@@ -55,7 +55,6 @@ def test_mecab_morpheme_generation(fake_environment):  # pylint:disable=unused-a
 def test_jieba_morpheme_generation(fake_environment):  # pylint:disable=unused-argument
     morphemizer = get_morphemizer_by_description("AnkiMorphs: Chinese")
 
-    # sentence = "本当に重要な任務の時しか 動かない"
     sentence = "请您说得慢些好吗？"
     correct_morphs: set[Morpheme] = {
         Morpheme("吗", "吗"),
@@ -72,3 +71,17 @@ def test_jieba_morpheme_generation(fake_environment):  # pylint:disable=unused-a
 
     for morph in extracted_morphs:
         assert morph in correct_morphs
+
+    sentence = "一，二，三，跳！"
+    correct_morphs: set[Morpheme] = {
+        Morpheme("一", "一"),
+        Morpheme("二", "二"),
+        Morpheme("三", "三"),
+        Morpheme("跳", "跳"),
+    }
+
+    extracted_morphs = morphemizer.get_morphemes_from_expr(sentence)
+    assert len(extracted_morphs) == 4
+
+    for morph in extracted_morphs:
+        assert morph in correct_morphs