added custom char filter to generators (#325)

Author: mortii

ankimorphs/extra_settings/ankimorphs_extra_settings.py

@@ -57,6 +57,8 @@ def save_generators_window_settings(
         self.setValue(PreprocessKeys.IGNORE_NAMES_MORPHEMIZER, ui.namesMorphemizerCheckBox.isChecked())
         self.setValue(PreprocessKeys.IGNORE_NAMES_IN_FILE, ui.namesFileCheckBox.isChecked())
         self.setValue(PreprocessKeys.IGNORE_NUMBERS, ui.numbersCheckBox.isChecked())
+        self.setValue(PreprocessKeys.IGNORE_CUSTOM_CHARS, ui.customCharactersCheckBox.isChecked())
+        self.setValue(PreprocessKeys.CHARS_TO_IGNORE, ui.customCharactersLineEdit.text())
         self.endGroup()  # preprocess group
         self.endGroup()  # generators window group
         # fmt: on

ankimorphs/extra_settings/extra_settings_keys.py

@@ -36,6 +36,8 @@ class PreprocessKeys:
     IGNORE_NAMES_MORPHEMIZER = "ignore_names_morphemizer"
     IGNORE_NAMES_IN_FILE = "ignore_names_in_file"
     IGNORE_NUMBERS = "ignore_numbers"
+    IGNORE_CUSTOM_CHARS = "ignore_custom_chars"
+    CHARS_TO_IGNORE = "chars_to_ignore"
 
 
 class KnownMorphsExporterKeys:

ankimorphs/generators/generators_utils.py

@@ -104,14 +104,16 @@ def __add__(self, other: FileMorphsStats) -> FileMorphsStats:
         return self
 
 
-class PreprocessOptions:
+class PreprocessOptions:  # pylint:disable=too-many-instance-attributes
     def __init__(self, ui: Ui_GeneratorsWindow):
         self.filter_square_brackets: bool = ui.squareBracketsCheckBox.isChecked()
         self.filter_round_brackets: bool = ui.roundBracketsCheckBox.isChecked()
         self.filter_slim_round_brackets: bool = ui.slimRoundBracketsCheckBox.isChecked()
         self.filter_numbers: bool = ui.numbersCheckBox.isChecked()
         self.filter_morphemizer_names: bool = ui.namesMorphemizerCheckBox.isChecked()
         self.filter_names_from_file: bool = ui.namesFileCheckBox.isChecked()
+        self.filter_custom_chars: bool = ui.customCharactersCheckBox.isChecked()
+        self.custom_chars_to_ignore: str = ui.customCharactersLineEdit.text()
 
     def to_mock_am_config(self) -> AnkiMorphsConfig:
         return Mock(
@@ -122,7 +124,8 @@ def to_mock_am_config(self) -> AnkiMorphsConfig:
             preprocess_ignore_numbers=self.filter_numbers,
             preprocess_ignore_names_morphemizer=self.filter_morphemizer_names,
             preprocess_ignore_names_textfile=self.filter_names_from_file,
-            preprocess_ignore_custom_characters="",  # todo: add option in generators window?
+            preprocess_ignore_custom_characters=self.filter_custom_chars,
+            preprocess_custom_characters_to_ignore=self.custom_chars_to_ignore,
         )
 
 
@@ -206,7 +209,6 @@ def generate_morph_occurrences_by_file(
     )
     preprocess_options = PreprocessOptions(ui)
     morph_occurrences_by_file: dict[Path, dict[str, MorphOccurrence]] = {}
-
     sorted_input_files: list[Path]
 
     if sorted_by_table:
@@ -217,6 +219,8 @@ def generate_morph_occurrences_by_file(
     else:
         sorted_input_files = input_files
 
+    translation_table = str.maketrans("", "", preprocess_options.custom_chars_to_ignore)
+
     for input_file in sorted_input_files:
         if mw.progress.want_cancel():  # user clicked 'x' button
             raise CancelledOperationException
@@ -233,6 +237,7 @@ def generate_morph_occurrences_by_file(
                 preprocess_options=preprocess_options,
                 file_path=input_file,
                 morphemizer=_morphemizer,
+                translation_table=translation_table,
             )
         )
         morph_occurrences_by_file[input_file] = file_morph_occurrences
@@ -244,9 +249,9 @@ def create_file_morph_occurrences(
     preprocess_options: PreprocessOptions,
     file_path: Path,
     morphemizer: Morphemizer,
+    translation_table: dict[int, int | None],
 ) -> dict[str, MorphOccurrence]:
 
-    morph_occurrences: dict[str, MorphOccurrence]
     raw_lines: list[str]
     filtered_lines: list[str] = []
     extension = file_path.suffix
@@ -261,22 +266,21 @@ def create_file_morph_occurrences(
         for line in raw_lines:
             # lower-case to avoid proper noun false-positives
             filtered_line = text_preprocessing.get_processed_text(
-                am_config=mock_am_config, text=line.strip().lower()
+                am_config=mock_am_config,
+                text=line.strip().lower(),
+                translation_table=translation_table,
             )
             if filtered_line:
                 filtered_lines.append(filtered_line)
-
     except UnicodeDecodeError as exc:
         raise UnicodeException(path=file_path) from exc
 
-    morph_occurrences = get_morph_occurrences(
+    return get_morph_occurrences(
         mock_am_config=mock_am_config,
         morphemizer=morphemizer,
         all_lines=filtered_lines,
     )
 
-    return morph_occurrences
-
 
 def get_morph_occurrences(
     mock_am_config: AnkiMorphsConfig,

ankimorphs/generators/generators_window.py

@@ -16,6 +16,7 @@
     QFileDialog,
     QHeaderView,
     QMainWindow,
+    Qt,
     QTableWidget,
     QTableWidgetItem,
 )
@@ -57,15 +58,14 @@ def __init__(
 
         self.checkboxes: list[QCheckBox] = []
         self._input_files: list[Path] = []
+        self._input_dir_root: Path
         self._morphemizers: list[Morphemizer] = morphemizer_utils.get_all_morphemizers()
         self._setup_morphemizers()
+        self._setup_line_edits()
         self._setup_checkboxes()
-        self._input_dir_root: Path
-
         self._setup_table(self.ui.numericalTableWidget)
         self._setup_table(self.ui.percentTableWidget)
         self._setup_buttons()
-        self._setup_input_field()
         self._setup_geometry()
 
         self.am_extra_settings.endGroup()
@@ -124,13 +124,27 @@ def _setup_buttons(self) -> None:
 
         self.ui.loadFilesPushButton.setFocus()  # quality of life
 
-    def _setup_input_field(self) -> None:
+    def _setup_line_edits(self) -> None:
         stored_input_dir: str = self.am_extra_settings.value(
             extra_settings_keys.GeneratorsWindowKeys.INPUT_DIR, type=str
         )
+
+        self.am_extra_settings.beginGroup(
+            extra_settings_keys.GeneratorsWindowKeys.PREPROCESS
+        )
+        stored_chars_to_ignore: str = self.am_extra_settings.value(
+            extra_settings_keys.PreprocessKeys.CHARS_TO_IGNORE, type=str
+        )
+        self.am_extra_settings.endGroup()
+
         if stored_input_dir is not None:
             self.ui.inputDirLineEdit.setText(stored_input_dir)
 
+        if stored_chars_to_ignore is not None:
+            self.ui.customCharactersLineEdit.setText(stored_chars_to_ignore)
+
+        self.ui.customCharactersLineEdit.setDisabled(True)
+
         self.ui.inputDirLineEdit.textEdited.connect(
             partial(self.ui.loadFilesPushButton.setEnabled, True)
         )
@@ -225,15 +239,29 @@ def _setup_preprocess_checkboxes(self) -> None:
         stored_ignore_numbers: bool = self.am_extra_settings.value(
             extra_settings_keys.PreprocessKeys.IGNORE_NUMBERS, type=bool
         )
+        stored_ignore_custom_chars: bool = self.am_extra_settings.value(
+            extra_settings_keys.PreprocessKeys.IGNORE_CUSTOM_CHARS, type=bool
+        )
 
         self.am_extra_settings.endGroup()
 
+        self.ui.customCharactersCheckBox.stateChanged.connect(
+            self._toggle_disable_custom_characters_line_edit
+        )
+
         self.ui.squareBracketsCheckBox.setChecked(stored_ignore_square_brackets)
         self.ui.roundBracketsCheckBox.setChecked(stored_ignore_round_brackets)
         self.ui.slimRoundBracketsCheckBox.setChecked(stored_ignore_slim_round_brackets)
         self.ui.namesMorphemizerCheckBox.setChecked(stored_ignore_names_morphemizer)
         self.ui.namesFileCheckBox.setChecked(stored_ignore_names_in_file)
         self.ui.numbersCheckBox.setChecked(stored_ignore_numbers)
+        self.ui.customCharactersCheckBox.setChecked(stored_ignore_custom_chars)
+
+    def _toggle_disable_custom_characters_line_edit(self) -> None:
+        if self.ui.customCharactersCheckBox.checkState() == Qt.CheckState.Unchecked:
+            self.ui.customCharactersLineEdit.setDisabled(True)
+        else:
+            self.ui.customCharactersLineEdit.setEnabled(True)
 
     def _on_select_folder_clicked(self) -> None:
         input_dir: str = QFileDialog.getExistingDirectory(

ankimorphs/text_preprocessing.py

@@ -1,3 +1,5 @@
+from __future__ import annotations
+
 import re
 from typing import Any
 
@@ -9,7 +11,7 @@
 round_brackets_regex = re.compile(r"（[^）]*）")
 slim_round_brackets_regexp = re.compile(r"\([^)]*\)")
 
-translation_table: dict[int, Any] = {}
+global_translation_table: dict[int, Any] = {}
 
 
 def update_translation_table() -> None:
@@ -19,13 +21,18 @@ def update_translation_table() -> None:
 
     Note: this function is executed on startup and when settings are saved
     """
-    global translation_table
-    translation_table = str.maketrans(
+    global global_translation_table
+    global_translation_table = str.maketrans(
         "", "", AnkiMorphsConfig().preprocess_custom_characters_to_ignore
     )
 
 
-def get_processed_text(am_config: AnkiMorphsConfig, text: str) -> str:
+def get_processed_text(
+    am_config: AnkiMorphsConfig,
+    text: str,
+    translation_table: dict[int, int | None] | None = None,
+) -> str:
+
     if am_config.preprocess_ignore_bracket_contents:
         text = square_brackets_regex.sub("", text)
 
@@ -39,6 +46,8 @@ def get_processed_text(am_config: AnkiMorphsConfig, text: str) -> str:
         text = re.sub(r"\d", "", text)
 
     if am_config.preprocess_ignore_custom_characters:
+        if translation_table is None:
+            translation_table = global_translation_table
         # str.translate() removes characters in a single pass, which is
         # much more efficient than str.replace()
         text = text.translate(translation_table)