fixed morphemizer not removing names (#41)

Author: mortii

ankimorphs/__init__.py

@@ -406,7 +406,7 @@ def add_name_action(web_view: AnkiWebView, menu: QMenu) -> None:
 
 
 def create_frequency_file_action() -> QAction:
-    action = QAction("&Generate Frequency File", mw)
+    action = QAction("&Frequency File Generator", mw)
     action.triggered.connect(frequency_file_generator.main)
     return action
 
@@ -423,21 +423,27 @@ def test_function() -> None:
     assert mw is not None
     assert mw.col.db is not None
 
-    am_db = AnkiMorphsDB()
-
-    with am_db.con:
-        result = am_db.con.execute(
-            """
-                SELECT morph_norm, morph_inflected, highest_learning_interval
-                FROM Card_Morph_Map
-                INNER JOIN Morphs ON
-                        Card_Morph_Map.morph_norm = Morphs.norm AND Card_Morph_Map.morph_inflected = Morphs.inflected
-                WHERE card_id = 1691325367067
-                """
-        ).fetchall()
-    print(f"result?: {result}")
+    # am_db = AnkiMorphsDB()
+    #
+    # with am_db.con:
+    #     result = am_db.con.execute(
+    #         """
+    #             SELECT morph_norm, morph_inflected, highest_learning_interval
+    #             FROM Card_Morph_Map
+    #             INNER JOIN Morphs ON
+    #                     Card_Morph_Map.morph_norm = Morphs.norm AND Card_Morph_Map.morph_inflected = Morphs.inflected
+    #             WHERE card_id = 1691325367067
+    #             """
+    #     ).fetchall()
+    # print(f"result?: {result}")
+    #
+    # am_db.con.close()
 
-    am_db.con.close()
+    # morphemizer = MecabMorphemizer()
+    # expression = "うん あ カフェ モカ お願い おかわり"
+    # morphs = get_morphemes(morphemizer, expression, am_config=AnkiMorphsConfig())
+    # morph_list = [morph.inflected for morph in morphs]
+    # print(f"morphs: {morph_list}")
 
 
 main()

ankimorphs/frequency_file_generator.py

@@ -18,7 +18,8 @@
 
 from .exceptions import CancelledOperationException, EmptyFileSelectionException
 from .morph_utils import (
-    _remove_names,
+    remove_names_morphemizer,
+    remove_names_textfile,
     round_brackets_regex,
     slim_round_brackets_regexp,
     square_brackets_regex,
@@ -136,8 +137,10 @@ def _background_generate_frequency_file(self, col: Collection) -> None:
                 for line in file:
                     expression = self._filter_expression(line)
                     morphs = morphemizer.get_morphemes_from_expr(expression)
+                    if self.ui.namesMorphemizerCheckBox.isChecked():
+                        morphs = remove_names_morphemizer(morphs)
                     if self.ui.namesFileCheckBox.isChecked():
-                        morphs = _remove_names(morphs)
+                        morphs = remove_names_textfile(morphs)
                     for morph in morphs:
                         key = morph.norm + morph.inflected
                         if key in morph_frequency_dict:

ankimorphs/morph_utils.py

@@ -15,9 +15,13 @@ def get_morphemes(
 ) -> list[Morpheme]:
     expression = _get_parsed_expression(am_config, expression)
     morphs = morphemizer.get_morphemes_from_expr(expression)
-    if not am_config.parse_ignore_names_textfile:
-        return morphs
-    morphs = _remove_names(morphs)
+
+    if am_config.parse_ignore_names_morphemizer:
+        morphs = remove_names_morphemizer(morphs)
+
+    if am_config.parse_ignore_names_textfile:
+        morphs = remove_names_textfile(morphs)
+
     return morphs
 
 
@@ -37,7 +41,7 @@ def _get_parsed_expression(am_config: AnkiMorphsConfig, expression: str) -> str:
     return expression
 
 
-def _remove_names(morphs: list[Morpheme]) -> list[Morpheme]:
+def remove_names_textfile(morphs: list[Morpheme]) -> list[Morpheme]:
     names = name_file_utils.create_hash_set_out_of_names()
     non_name_morphs: list[Morpheme] = []
 
@@ -46,3 +50,7 @@ def _remove_names(morphs: list[Morpheme]) -> list[Morpheme]:
             non_name_morphs.append(morph)
 
     return non_name_morphs
+
+
+def remove_names_morphemizer(morphs: list[Morpheme]) -> list[Morpheme]:
+    return [morph for morph in morphs if not morph.is_proper_noun()]

ankimorphs/morpheme.py

@@ -53,6 +53,9 @@ def __hash__(self):
             (self.norm, self.base, self.inflected, self.read, self.pos, self.sub_pos)
         )
 
+    def is_proper_noun(self) -> bool:
+        return self.sub_pos == "固有名詞" or self.pos == "PROPN"
+
 
 class SimplifiedMorph:
     __slots__ = (

ankimorphs/settings_dialog.py

@@ -54,7 +54,7 @@ def __init__(self, parent: Optional[QMainWindow] = None) -> None:
         self.ui.tabWidget.currentChanged.connect(self.tab_change)
 
         # Semantic Versioning https://semver.org/
-        self.ui.ankimorphs_version_label.setText("AnkiMorphs version: 0.5.0-alpha")
+        self.ui.ankimorphs_version_label.setText("AnkiMorphs version: 0.5.1-alpha")
 
     def _setup_note_filters_table(
         self, config_filters: list[AnkiMorphsConfigFilter]