added spaCy manager

Author: mortii

.github/workflows/build.yml

@@ -9,7 +9,7 @@ jobs:
     runs-on: ${{ matrix.os }}
     strategy:
       matrix:
-        python_version: [3.9, 3.13]
+        python_version: [3.13]
         os: [ubuntu-latest, macOS-latest, windows-latest]
       fail-fast: false
     steps:

ankimorphs/__init__.py

@@ -61,6 +61,7 @@
 from .recalc import recalc_main
 from .settings import settings_dialog
 from .settings.settings_dialog import SettingsDialog
+from .spacy_manager import SpacyManagerDialog
 from .tag_selection_dialog import TagSelectionDialog
 from .toolbar_stats import MorphToolbarStats
 
@@ -73,7 +74,7 @@
 
 
 def main() -> None:
-    # Support anki version 2.1.50 and above
+    # Support anki version 25.07.3 and above
     # Place hooks in the order they are executed
 
     gui_hooks.top_toolbar_did_init_links.append(init_toolbar_items)
@@ -212,6 +213,10 @@ def register_addon_dialogs() -> None:
         name=am_globals.KNOWN_MORPHS_EXPORTER_DIALOG_NAME,
         creator=KnownMorphsExporterDialog,
     )
+    aqt.dialogs.register_dialog(
+        name=am_globals.SPACY_MANAGER_DIALOG_NAME,
+        creator=SpacyManagerDialog,
+    )
 
 
 def redraw_toolbar() -> None:
@@ -235,6 +240,7 @@ def init_tool_menu_and_actions() -> None:
     generators_action = create_generators_dialog_action(am_config)
     progression_action = create_progression_dialog_action(am_config)
     known_morphs_exporter_action = create_known_morphs_exporter_action(am_config)
+    spacy_manager_action = create_spacy_manager_dialog_action()
     reset_tags_action = create_tag_reset_action()
     guide_action = create_guide_action()
     changelog_action = create_changelog_action()
@@ -245,6 +251,7 @@ def init_tool_menu_and_actions() -> None:
     am_tool_menu.addAction(generators_action)
     am_tool_menu.addAction(progression_action)
     am_tool_menu.addAction(known_morphs_exporter_action)
+    am_tool_menu.addAction(spacy_manager_action)
     am_tool_menu.addAction(reset_tags_action)
     am_tool_menu.addAction(guide_action)
     am_tool_menu.addAction(changelog_action)
@@ -594,6 +601,17 @@ def create_known_morphs_exporter_action(am_config: AnkiMorphsConfig) -> QAction:
     return action
 
 
+def create_spacy_manager_dialog_action() -> QAction:
+    action = QAction("&spaCy Manager", mw)
+    action.triggered.connect(
+        partial(
+            aqt.dialogs.open,
+            name=am_globals.SPACY_MANAGER_DIALOG_NAME,
+        )
+    )
+    return action
+
+
 def create_test_action() -> QAction:
     keys = QKeySequence("Ctrl+T")
     action = QAction("&Test", mw)

ankimorphs/ankimorphs_globals.py

@@ -5,7 +5,7 @@
 """
 
 # Semantic Versioning https://semver.org/
-__version__ = "4.0.2"
+__version__ = "5.0.0"
 
 DEV_MODE: bool = False
 
@@ -19,6 +19,7 @@
 GENERATOR_DIALOG_NAME: str = "am_generator_dialog"
 PROGRESSION_DIALOG_NAME: str = "am_progression_dialog"
 KNOWN_MORPHS_EXPORTER_DIALOG_NAME: str = "am_known_morphs_exporter_dialog"
+SPACY_MANAGER_DIALOG_NAME: str = "am_spacy_manager_dialog"
 
 # The static names of the extra fields
 EXTRA_FIELD_ALL_MORPHS: str = "am-all-morphs"

ankimorphs/extra_settings/ankimorphs_extra_settings.py

@@ -16,6 +16,7 @@
     KnownMorphsExporterKeys,
     PreprocessKeys,
     ProgressionWindowKeys,
+    SpacyManagerWindowKeys,
 )
 
 
@@ -74,6 +75,13 @@ def save_known_morphs_exporter_settings(
         self.endGroup()
         # fmt: on
 
+    def spacy_manager_window_settings(self, geometry: QByteArray) -> None:
+        # fmt: off
+        self.beginGroup(keys.Dialogs.SPACY_MANAGER_WINDOW)
+        self.setValue(SpacyManagerWindowKeys.WINDOW_GEOMETRY, geometry)
+        self.endGroup()
+        # fmt: on
+
     def save_progression_window_settings(
         self, ui: Ui_ProgressionWindow, geometry: QByteArray
     ) -> None:

ankimorphs/extra_settings/extra_settings_keys.py

@@ -5,6 +5,7 @@ class General:
 class Dialogs:
     GENERATORS_WINDOW = "generators_window"
     KNOWN_MORPHS_EXPORTER = "known_morphs_exporter"
+    SPACY_MANAGER_WINDOW = "spacy_manager_window"
     PROGRESSION_WINDOW = "progression_window"
     GENERATOR_OUTPUT_PRIORITY_FILE = "generator_output_priority_file"
     GENERATOR_OUTPUT_STUDY_PLAN = "generator_output_study_plan"
@@ -46,6 +47,10 @@ class KnownMorphsExporterKeys:
     OCCURRENCES = "occurrences"
 
 
+class SpacyManagerWindowKeys:
+    WINDOW_GEOMETRY = "window_geometry"
+
+
 class ProgressionWindowKeys:
     WINDOW_GEOMETRY = "window_geometry"
     PRIORITY_FILE = "priority_file"

ankimorphs/message_box_utils.py

@@ -9,6 +9,16 @@
 )
 
 
+def show_info_box(title: str, body: str, parent: QWidget) -> None:
+    info_box = QMessageBox(parent)
+    info_box.setWindowTitle(title)
+    info_box.setIcon(QMessageBox.Icon.Information)
+    info_box.setStandardButtons(QMessageBox.StandardButton.Ok)
+    info_box.setTextFormat(Qt.TextFormat.MarkdownText)
+    info_box.setText(body)
+    info_box.exec()
+
+
 def show_warning_box(title: str, body: str, parent: QWidget) -> bool:
     """
     Returns 'True' if user clicked 'Ok' button

ankimorphs/morphemizers/spacy_wrapper.py

@@ -2,12 +2,15 @@
 
 import functools
 import os.path
+import shutil
+import subprocess
 import sys
 from types import ModuleType
 from typing import Any
 
 from anki.utils import is_win
 from aqt import mw
+from aqt.package import venv_binary
 
 updated_python_path: bool = False
 testing_environment: bool = False
@@ -20,35 +23,71 @@
 _SpacyTokenizer: Any = None  # spacy.tokenizer
 _SpacyDoc: Any = None  #  spacy.tokens.doc
 
+
+# spaCy does not have a cli to query available languages or models, so we hardcode it.
+available_langs_and_models: dict[str, list[str]] = {
+    # fmt: off
+    "Catalan": ["ca_core_news_sm", "ca_core_news_md", "ca_core_news_lg"],
+    "Chinese": ["zh_core_web_sm", "zh_core_web_md", "zh_core_web_lg"],
+    "Croatian": ["hr_core_news_sm", "hr_core_news_md", "hr_core_news_lg"],
+    "Danish": ["da_core_news_sm", "da_core_news_md", "da_core_news_lg"],
+    "Dutch": ["nl_core_news_sm", "nl_core_news_md", "nl_core_news_lg"],
+    "English": ["en_core_web_sm", "en_core_web_md", "en_core_web_lg"],
+    "Finnish": ["fi_core_news_sm", "fi_core_news_md", "fi_core_news_lg"],
+    "French": ["fr_core_news_sm", "fr_core_news_md", "fr_core_news_lg"],
+    "German": ["de_core_news_sm", "de_core_news_md", "de_core_news_lg"],
+    "Greek": ["el_core_news_sm", "el_core_news_md", "el_core_news_lg"],
+    "Italian": ["it_core_news_sm", "it_core_news_md", "it_core_news_lg"],
+    "Japanese": ["ja_core_news_sm", "ja_core_news_md", "ja_core_news_lg"],
+    "Korean": ["ko_core_news_sm", "ko_core_news_md", "ko_core_news_lg"],
+    "Lithuanian": ["lt_core_news_sm", "lt_core_news_md", "lt_core_news_lg"],
+    "Macedonian": ["mk_core_news_sm", "mk_core_news_md", "mk_core_news_lg"],
+    "Norwegian Bokmål": ["nb_core_news_sm", "nb_core_news_md", "nb_core_news_lg"],
+    "Polish": ["pl_core_news_sm", "pl_core_news_md", "pl_core_news_lg"],
+    "Portuguese": ["pt_core_news_sm", "pt_core_news_md", "pt_core_news_lg"],
+    "Romanian": ["ro_core_news_sm", "ro_core_news_md", "ro_core_news_lg"],
+    "Russian": ["ru_core_news_sm", "ru_core_news_md", "ru_core_news_lg"],
+    "Slovenian": ["sl_core_news_sm", "sl_core_news_md", "sl_core_news_lg"],
+    "Spanish": ["es_core_news_sm", "es_core_news_md", "es_core_news_lg"],
+    "Swedish": ["sv_core_news_sm", "sv_core_news_md", "sv_core_news_lg"],
+    "Ukrainian": ["uk_core_news_sm", "uk_core_news_md", "uk_core_news_lg"],
+    # fmt: on
+}
+
+
 LANGUAGE_PIPE_CONFIGS: dict[str, set[str]] = {
-    "ja": {""},  # Japanese uses SudachiPy
-    "nb": {"tok2vec", "attribute_ruler", "lemmatizer", "morphologizer", "ner"},
+    "ca": {"tok2vec", "morphologizer", "lemmatizer"},
+    "zh": {"tok2vec", "tagger", "attribute_ruler"},
+    "hr": {"tok2vec", "lemmatizer", "morphologizer"},
     "da": {"tok2vec", "morphologizer", "lemmatizer"},
-    "de": {"tok2vec", "lemmatizer", "morphologizer"},
-    "fr": {"tok2vec", "lemmatizer", "morphologizer"},
-    "en": {"tok2vec", "tagger", "attribute_ruler", "lemmatizer", "morphologizer"},
-    "es": {"tok2vec", "lemmatizer", "morphologizer"},
-    "sv": {"tok2vec", "lemmatizer", "morphologizer"},
     "nl": {"tok2vec", "lemmatizer", "morphologizer"},
-    "hr": {"tok2vec", "lemmatizer", "morphologizer"},
+    "en": {"tok2vec", "tagger", "attribute_ruler", "lemmatizer", "morphologizer"},
     "fi": {"tok2vec", "lemmatizer", "morphologizer"},
+    "fr": {"tok2vec", "lemmatizer", "morphologizer"},
+    "de": {"tok2vec", "lemmatizer", "morphologizer"},
     "el": {"tok2vec", "lemmatizer", "morphologizer"},
     "it": {"tok2vec", "lemmatizer", "morphologizer"},
+    "ja": {""},  # Japanese uses SudachiPy
+    "ko": {"tok2vec", "morphologizer", "lemmatizer"},
     "lt": {"tok2vec", "lemmatizer", "morphologizer"},
     "mk": {"tok2vec", "lemmatizer", "morphologizer"},
+    "nb": {"tok2vec", "attribute_ruler", "lemmatizer", "morphologizer", "ner"},
     "pl": {"tok2vec", "lemmatizer", "morphologizer"},
     "pt": {"tok2vec", "lemmatizer", "morphologizer"},
     "ro": {"tok2vec", "tagger", "morphologizer", "lemmatizer", "attribute_ruler"},
-    "sl": {"tok2vec", "lemmatizer", "morphologizer"},
-    "ca": {"tok2vec", "morphologizer", "lemmatizer"},
     "ru": {"tok2vec", "morphologizer", "lemmatizer"},
+    "sl": {"tok2vec", "lemmatizer", "morphologizer"},
+    "es": {"tok2vec", "lemmatizer", "morphologizer"},
+    "sv": {"tok2vec", "lemmatizer", "morphologizer"},
     "uk": {"tok2vec", "morphologizer", "lemmatizer"},
-    "ko": {"tok2vec", "morphologizer", "lemmatizer"},
-    "zh": {"tok2vec", "tagger", "attribute_ruler"},
 }
 
 
 def load_spacy_modules() -> None:
+    # We load the spacy modules in this complicated way to maintain at least
+    # some form of static type checking, and to minimize error checking
+    # and exception handling
+
     global updated_python_path
     global successful_import
     global _spacy
@@ -59,14 +98,9 @@ def load_spacy_modules() -> None:
 
     # dev environments should already have spaCy, so this can be skipped
     if not updated_python_path and not testing_environment:
-        # Anki only looks into its own directories for python packages,
-        # to add other lookup folders we have to change the sys path.
-        # In the guide we instruct the users to install the spacy
-        # virtual environment into the addons21 folder as 'spacyenv',
-        # that way we can get the path based on the anki mw.pm.
-
         assert mw is not None
-        spacy_path = os.path.join(mw.pm.addonFolder(), "spacyenv")
+
+        spacy_path = _get_am_spacy_venv_path()
 
         if is_win is True:
             spacy_bin_path = os.path.join(spacy_path, "Scripts")
@@ -80,6 +114,7 @@ def load_spacy_modules() -> None:
                 "site-packages",
             )
 
+        # appending to the path is less disruptive than prepending
         sys.path.append(spacy_bin_path)
         sys.path.append(spacy_site_packages_path)
         updated_python_path = True
@@ -116,6 +151,79 @@ def get_installed_models() -> list[str]:
     return [f"{model_name}" for model_name in _spacy_utils.get_installed_models()]
 
 
+def _get_am_spacy_venv_python() -> str:
+    if is_win:
+        return os.path.join(_get_am_spacy_venv_path(), "Scripts", "python", ".exe")
+    return os.path.join(_get_am_spacy_venv_path(), "bin", "python")
+
+
+def _get_am_spacy_venv_path() -> str:
+    python_version = f"{sys.version_info.major}_{sys.version_info.minor}"
+    return os.path.join(mw.pm.addonFolder(), f"spacy-venv-python-{python_version}")
+
+
+def create_spacy_venv() -> None:
+    """
+    We create a dedicated venv to avoid polluting the anki launcher environment
+    """
+
+    spacy_venv_path = _get_am_spacy_venv_path()
+    python_path: str | None = venv_binary("python")
+    assert python_path is not None
+
+    subprocess.run([python_path, "-m", "venv", spacy_venv_path], check=True)
+
+    if is_win:
+        spacy_venv_python = os.path.join(spacy_venv_path, "Scripts", "python", ".exe")
+    else:
+        spacy_venv_python = os.path.join(spacy_venv_path, "bin", "python")
+
+    # make sure pip, setuptools, and wheel are up to date
+    subprocess.run(
+        [
+            spacy_venv_python,
+            "-m",
+            "pip",
+            "install",
+            "--upgrade",
+            "pip",
+            "setuptools",
+            "wheel",
+        ],
+        check=True,
+    )
+
+    # six is necessary for some models
+    subprocess.run(
+        [spacy_venv_python, "-m", "pip", "install", "--upgrade", "spacy", "six"],
+        check=True,
+    )
+
+
+def delete_spacy_venv() -> None:
+    shutil.rmtree(_get_am_spacy_venv_path())
+
+
+def install_model(model_name: str) -> None:
+    assert successful_import
+    assert _spacy is not None
+
+    subprocess.run(
+        [_get_am_spacy_venv_python(), "-m", "spacy", "download", model_name], check=True
+    )
+
+
+def uninstall_model(model_name: str) -> None:
+    assert successful_import
+    assert _spacy is not None
+
+    # the -y flag prevents a confirmation prompt
+    subprocess.run(
+        [_get_am_spacy_venv_python(), "-m", "pip", "uninstall", "-y", model_name],
+        check=True,
+    )
+
+
 # the cache needs to have a max size to maintain garbage collection
 @functools.lru_cache(maxsize=131072)
 def get_nlp(spacy_model_name: str):  # type: ignore[no-untyped-def] # pylint:disable=too-many-branches, too-many-statements