fixed readability report accuracy (#134)

Co-authored-by: Vilhelm-Ian <[email protected]>

Author: mortii

ankimorphs/anki_data_utils.py

@@ -91,7 +91,7 @@ def __init__(
         self.note_id = anki_row_data.note_id
 
         # this is set later when spacy is used
-        self.morphs: Optional[set[Morpheme]] = None
+        self.morphs: Optional[list[Morpheme]] = None
 
 
 class AnkiMorphsCardData:

ankimorphs/frequency_file_generator.py

@@ -109,7 +109,7 @@ def _background_generate_frequency_file(  # pylint:disable=too-many-locals
                 # NB! Never use readlines(), it loads the entire file to memory
                 for counter, line in enumerate(file):
                     print(f"line: {counter}")
-                    morphs: set[Morpheme] = self._get_morphs_from_line(
+                    morphs: list[Morpheme] = self._get_morphs_from_line(
                         morphemizer, nlp, line
                     )
                     for morph in morphs:

ankimorphs/generator_dialog.py

@@ -130,16 +130,16 @@ def _filter_expression(self, expression: str) -> str:
 
     def _get_morphs_from_line(  # type: ignore[no-untyped-def]
         self, _morphemizer: Morphemizer, nlp, line: str
-    ) -> set[Morpheme]:
+    ) -> list[Morpheme]:
         # todo: this is horrible, create a callback or something
         if nlp is None:
             return self._get_morphs_from_line_morphemizer(_morphemizer, line)
         return self._get_morphs_from_line_spacy(nlp, line)
 
-    def _get_morphs_from_line_spacy(self, nlp, line: str) -> set[Morpheme]:  # type: ignore[no-untyped-def]
+    def _get_morphs_from_line_spacy(self, nlp, line: str) -> list[Morpheme]:  # type: ignore[no-untyped-def]
         # nlp: spacy.Language
 
-        morphs: set[Morpheme] = set()
+        morphs: list[Morpheme] = []
         expression = self._filter_expression(line)
 
         doc = nlp(expression)
@@ -152,7 +152,7 @@ def _get_morphs_from_line_spacy(self, nlp, line: str) -> set[Morpheme]:  # type:
                 if w.pos == 96:  # PROPN
                     continue
 
-            morphs.add(
+            morphs.append(
                 Morpheme(
                     lemma=w.lemma_,
                     inflection=w.text,
@@ -166,9 +166,9 @@ def _get_morphs_from_line_spacy(self, nlp, line: str) -> set[Morpheme]:  # type:
 
     def _get_morphs_from_line_morphemizer(
         self, _morphemizer: Morphemizer, line: str
-    ) -> set[Morpheme]:
+    ) -> list[Morpheme]:
         expression = self._filter_expression(line)
-        morphs: set[Morpheme] = _morphemizer.get_morphemes_from_expr(expression)
+        morphs: list[Morpheme] = _morphemizer.get_morphemes_from_expr(expression)
         if self.ui.namesMorphemizerCheckBox.isChecked():
             morphs = text_preprocessing.remove_names_morphemizer(morphs)
         if self.ui.namesFileCheckBox.isChecked():

ankimorphs/mecab_wrapper.py

@@ -99,7 +99,7 @@ def get_morpheme(  # pylint:disable=too-many-return-statements
 control_chars_re = re.compile("[\x00-\x1f\x7f-\x9f]")
 
 
-def get_morphemes_mecab(expression) -> set[Morpheme]:
+def get_morphemes_mecab(expression) -> list[Morpheme]:
     # HACK: mecab sometimes does not produce the right morphs if there are no extra characters in the expression,
     # so we just add a whitespace and a japanese punctuation mark "。" the end to prevent the problem.
     expression += " 。"
@@ -109,7 +109,7 @@ def get_morphemes_mecab(expression) -> set[Morpheme]:
 
     _morphs = [get_morpheme(m.split("\t")) for m in interact(expression).split("\r")]
 
-    _morphs = {_morph for _morph in _morphs if _morph is not None}
+    _morphs = [_morph for _morph in _morphs if _morph is not None]
     return _morphs
 
 
@@ -203,68 +203,11 @@ def mecab():  # pylint: disable=too-many-branches,too-many-statements
     else:
         startup_info = None
 
-    # Search for mecab
-    reading = None
-
-    # # 1st priority - MecabUnidic
-    # if importlib.util.find_spec("MecabUnidic"):
-    #     try:
-    #         reading = importlib.import_module("MecabUnidic.reading")
-    #         mecab_source = "MecabUnidic from addon MecabUnidic"
-    #     except ModuleNotFoundError:
-    #         pass
-    #
-    # if importlib.util.find_spec("13462835"):
-    #     try:
-    #         reading = importlib.import_module("13462835.reading")
-    #         mecab_source = "MecabUnidic from addon 13462835"
-    #     except ModuleNotFoundError:
-    #         pass
-    #
-    # # 2nd priority - Japanese Support
-    # if (not reading) and importlib.util.find_spec("3918629684"):
-    #     try:
-    #         reading = importlib.import_module("3918629684.reading")
-    #         mecab_source = "Japanese Support from addon 3918629684"
-    #     except ModuleNotFoundError:
-    #         pass
-    #
-    # # 3nd priority - MIAJapaneseSupport
-    # if (not reading) and importlib.util.find_spec("MIAJapaneseSupport"):
-    #     try:
-    #         reading = importlib.import_module("MIAJapaneseSupport.reading")
-    #         mecab_source = "MIAJapaneseSupport from addon MIAJapaneseSupport"
-    #     except ModuleNotFoundError:
-    #         pass
-    # # 4nd priority - MigakuJapaneseSupport via Anki code (278530045)
-    # if (not reading) and importlib.util.find_spec("278530045"):
-    #     try:
-    #         reading = importlib.import_module("278530045.reading")
-    #         mecab_source = "Migaku Japanese support from addon 278530045"
-    #     except ModuleNotFoundError:
-    #         pass
-
-    # 5th priority - From Morphman
-    if not reading:
-        file_path = os.path.realpath(__file__)
-        am_dir = file_path.split(os.sep)[-2]
-        mecab_dir = am_dir + ".deps.mecab.reading"
-        reading = importlib.import_module(mecab_dir)
-        mecab_source = "AnkiMorphs"
-
-    # 6th priority - system mecab
-    # if not reading:
-    #     try:
-    #         return spawn_mecab(["mecab"], startup_info), "System"
-    #     except Exception as error:
-    #         raise OSError(
-    #             """
-    #         Mecab Japanese analyzer could not be found.
-    #         Please install one of the following Anki add-ons:
-    #              https://ankiweb.net/shared/info/3918629684
-    #              https://ankiweb.net/shared/info/13462835
-    #              https://ankiweb.net/shared/info/278530045"""
-    #         ) from error
+    file_path = os.path.realpath(__file__)
+    am_dir = file_path.split(os.sep)[-2]
+    mecab_dir = am_dir + ".deps.mecab.reading"
+    reading = importlib.import_module(mecab_dir)
+    mecab_source = "AnkiMorphs"
 
     _mecab = reading.MecabController()
     _mecab.setup()

ankimorphs/morphemizer.py

@@ -18,17 +18,17 @@ def __init__(self) -> None:
 
     # the cache needs to have a max size to maintain garbage collection
     @functools.lru_cache(maxsize=131072)
-    def get_morphemes_from_expr(self, expression: str) -> set[Morpheme]:
+    def get_morphemes_from_expr(self, expression: str) -> list[Morpheme]:
         morphs = self._get_morphemes_from_expr(expression)
         return morphs
 
     def _get_morphemes_from_expr(  # pylint:disable=unused-argument
         self, expression: str
-    ) -> set[Morpheme]:
+    ) -> list[Morpheme]:
         """
         The heart of this plugin: convert an expression to a list of its morphemes.
         """
-        return set()
+        return []
 
     def get_description(self) -> str:
         """
@@ -80,7 +80,7 @@ class MecabMorphemizer(Morphemizer):
     a extra tool called 'mecab' has to be used.
     """
 
-    def _get_morphemes_from_expr(self, expression: str) -> set[Morpheme]:
+    def _get_morphemes_from_expr(self, expression: str) -> list[Morpheme]:
         # Remove simple spaces that could be added by other add-ons and break the parsing.
         if space_char_regex.search(expression):
             expression = space_char_regex.sub("", expression)
@@ -105,7 +105,7 @@ class SpaceMorphemizer(Morphemizer):
     a general-use-morphemizer, it can't generate the base form from inflection.
     """
 
-    def _get_morphemes_from_expr(self, expression: str) -> set[Morpheme]:
+    def _get_morphemes_from_expr(self, expression: str) -> list[Morpheme]:
         # We want the expression: "At 3 o'clock that god-forsaken-man shows up..."
         # to produce: ['at', '3', "o'clock", 'that', 'god-forsaken-man', 'shows', 'up']
         #
@@ -126,7 +126,7 @@ def _get_morphemes_from_expr(self, expression: str) -> set[Morpheme]:
             word.lower()
             for word in re.findall(r"\w+(?:[-']\w+)*", expression, re.UNICODE)
         ]
-        return {Morpheme(lemma=word, inflection=word) for word in word_list}
+        return [Morpheme(lemma=word, inflection=word) for word in word_list]
 
     def get_description(self) -> str:
         return "AnkiMoprhs: Language w/ Spaces"

ankimorphs/readability_report_generator.py

@@ -19,6 +19,7 @@
 from .generator_dialog import GeneratorDialog
 from .morpheme import Morpheme, MorphOccurrence
 from .morphemizer import Morphemizer, SpacyMorphemizer
+from .table_utils import QTableWidgetIntegerItem, QTableWidgetPercentItem
 from .ui.readability_report_generator_ui import Ui_ReadabilityReportGeneratorDialog
 
 
@@ -36,7 +37,7 @@ def __init__(self) -> None:
         self._unknowns_column = 4
         self._number_of_columns = 5
         self._setup_absolute_table()
-        self._setup_perecentages_table()
+        self._setup_percentages_table()
         self._setup_buttons()
         self.show()
 
@@ -63,7 +64,7 @@ def _setup_absolute_table(self) -> None:
             QAbstractItemView.EditTrigger.NoEditTriggers
         )
 
-    def _setup_perecentages_table(self) -> None:
+    def _setup_percentages_table(self) -> None:
         assert isinstance(self.ui, Ui_ReadabilityReportGeneratorDialog)
 
         self.ui.percentTableWidget.setAlternatingRowColors(True)
@@ -169,17 +170,15 @@ def _background_generate_report(  # pylint:disable=too-many-locals
                 am_config, am_db, file_morphs
             )
 
-            self._populate_absolute_table(
+            self._populate_numerical_table(
                 _input_file,
-                file_morphs,
                 _row,
                 known_morphs,
                 learning_morphs,
                 unknown_morphs,
             )
             self._populate_percentage_table(
                 _input_file,
-                file_morphs,
                 _row,
                 known_morphs,
                 learning_morphs,
@@ -196,7 +195,7 @@ def _create_file_morphs_dict(self, file: TextIO, morphemizer, nlp) -> dict[str,
 
         file_morphs: dict[str, MorphOccurrence] = {}
         for line in file:
-            morphs: set[Morpheme] = self._get_morphs_from_line(morphemizer, nlp, line)
+            morphs: list[Morpheme] = self._get_morphs_from_line(morphemizer, nlp, line)
             for morph in morphs:
                 key = morph.lemma + morph.inflection
                 if key in file_morphs:
@@ -217,28 +216,28 @@ def _get_morph_statuses(
 
         for morph_occurrence_object in file_morphs.values():
             morph = morph_occurrence_object.morph
+            occurrence = morph_occurrence_object.occurrence
 
             highest_learning_interval: Optional[
                 int
             ] = am_db.get_highest_learning_interval(morph.lemma, morph.inflection)
 
             if highest_learning_interval is None:
-                unknown_morphs += 1
+                unknown_morphs += occurrence
                 continue
 
             if highest_learning_interval == 0:
-                unknown_morphs += 1
+                unknown_morphs += occurrence
             elif highest_learning_interval < am_config.recalc_interval_for_known:
-                learning_morphs += 1
+                learning_morphs += occurrence
             else:
-                known_morphs += 1
+                known_morphs += occurrence
 
         return known_morphs, learning_morphs, unknown_morphs
 
-    def _populate_absolute_table(
+    def _populate_numerical_table(
         self,
         _input_file: Path,
-        file_morphs: dict[str, MorphOccurrence],
         _row: int,
         known_morphs: int,
         learning_morphs: int,
@@ -248,11 +247,13 @@ def _populate_absolute_table(
 
         relative_path = _input_file.relative_to(self._input_dir_root)
 
+        total_morphs: int = known_morphs + learning_morphs + unknown_morphs
+
         file_name_item = QTableWidgetItem(str(relative_path))
-        total_morphs_item = QTableWidgetItem(str(len(file_morphs)))
-        known_item = QTableWidgetItem(str(known_morphs))
-        learning_item = QTableWidgetItem(str(learning_morphs))
-        unknowns_item = QTableWidgetItem(str(unknown_morphs))
+        total_morphs_item = QTableWidgetIntegerItem(total_morphs)
+        known_item = QTableWidgetIntegerItem(known_morphs)
+        learning_item = QTableWidgetIntegerItem(learning_morphs)
+        unknowns_item = QTableWidgetIntegerItem(unknown_morphs)
 
         total_morphs_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter)
         known_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter)
@@ -272,26 +273,30 @@ def _populate_absolute_table(
     def _populate_percentage_table(
         self,
         _input_file: Path,
-        file_morphs: dict[str, MorphOccurrence],
         _row: int,
         known_morphs: int,
         learning_morphs: int,
         unknown_morphs: int,
     ) -> None:
         assert isinstance(self.ui, Ui_ReadabilityReportGeneratorDialog)
 
-        total_morphs = len(file_morphs)
-        known_morphs_percent = (known_morphs / total_morphs) * 100
-        learning_morphs_percent = (learning_morphs / total_morphs) * 100
-        unknown_morphs_percent = (unknown_morphs / total_morphs) * 100
+        total_morphs: int = known_morphs + learning_morphs + unknown_morphs
+        known_morphs_percent: float = 0
+        learning_morphs_percent: float = 0
+        unknown_morphs_percent: float = 0
+
+        if total_morphs != 0:
+            known_morphs_percent = (known_morphs / total_morphs) * 100
+            learning_morphs_percent = (learning_morphs / total_morphs) * 100
+            unknown_morphs_percent = (unknown_morphs / total_morphs) * 100
 
         relative_path = _input_file.relative_to(self._input_dir_root)
 
         file_name_item = QTableWidgetItem(str(relative_path))
-        total_morphs_item = QTableWidgetItem(str(len(file_morphs)))
-        known_item = QTableWidgetItem(f"{round(known_morphs_percent, 1)} %")
-        learning_item = QTableWidgetItem(f"{round(learning_morphs_percent, 1)} %")
-        unknowns_item = QTableWidgetItem(f"{round(unknown_morphs_percent, 1)} %")
+        total_morphs_item = QTableWidgetIntegerItem(total_morphs)
+        known_item = QTableWidgetPercentItem(round(known_morphs_percent, 1))
+        learning_item = QTableWidgetPercentItem(round(learning_morphs_percent, 1))
+        unknowns_item = QTableWidgetPercentItem(round(unknown_morphs_percent, 1))
 
         total_morphs_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter)
         known_item.setTextAlignment(Qt.AlignmentFlag.AlignCenter)

ankimorphs/settings_dialog.py

@@ -109,7 +109,7 @@ def __init__(self) -> None:
         )
 
         # Semantic Versioning https://semver.org/
-        self.ui.ankimorphs_version_label.setText("AnkiMorphs version: 0.16.0-alpha")
+        self.ui.ankimorphs_version_label.setText("AnkiMorphs version: 0.16.1-alpha")
 
         self.show()