From fd5d1b9ad8f1f18e47ab010353bf19b5a55e5b05 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 14:59:52 +0100 Subject: [PATCH 1/6] small changes --- .pre-commit-config.yaml | 10 +++++----- Code-of-Conduct.md | 2 +- README.md | 5 +++-- pyproject.toml | 2 +- src/maturin_import_hook/__main__.py | 4 ++-- 5 files changed, 12 insertions(+), 11 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index ffa2f17..f98b421 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,6 +1,6 @@ repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - id: check-toml @@ -8,12 +8,12 @@ repos: - id: trailing-whitespace - id: mixed-line-ending - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.4 + rev: v0.4.9 hooks: - id: ruff-format - id: ruff - repo: https://github.com/pre-commit/mirrors-mypy - rev: v1.9.0 + rev: v1.10.0 hooks: # note: mypy runs in an isolated environment and so has no access to third party packages - id: mypy @@ -21,10 +21,10 @@ repos: pass_filenames: false additional_dependencies: ["pytest"] - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell - repo: https://github.com/igorshubovych/markdownlint-cli - rev: v0.39.0 + rev: v0.41.0 hooks: - id: markdownlint-fix diff --git a/Code-of-Conduct.md b/Code-of-Conduct.md index 86fe59f..9f0b4c0 100644 --- a/Code-of-Conduct.md +++ b/Code-of-Conduct.md @@ -6,7 +6,7 @@ In the interest of fostering an open and welcoming environment, we as contributors and maintainers pledge to making participation in our project and our community a harassment-free experience for everyone, regardless of age, body size, disability, ethnicity, gender identity and expression, level of experience, -education, socio-economic status, nationality, personal appearance, race, +education, socioeconomic status, nationality, personal appearance, race, religion, or sexual identity and orientation. ## Our Standards diff --git a/README.md b/README.md index 8b344d6..296ccea 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,8 @@ The import hook also provides conveniences such as ## Usage -After installing `maturin`, install the import hook into a python virtual environment with: +After installing [maturin](https://www.maturin.rs/installation), install the import hook into a python virtual +environment with: ```shell pip install maturin_import_hook @@ -35,7 +36,7 @@ maturin_import_hook.install() # when a rust package that is installed in editable mode is imported, # that package will be automatically recompiled if necessary. -import pyo3_pure +import example_maturin_package # when a .rs file is imported a project will be created for it in the # maturin build cache and the resulting library will be loaded. diff --git a/pyproject.toml b/pyproject.toml index ce62ad7..6ac45bc 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "maturin-import-hook" -description = "import hook to load rust projects built with maturin" +description = "Import hook to load rust projects built with maturin" authors = [ {name = "Matthew Broadway", email = "mattdbway@gmail.com"} ] diff --git a/src/maturin_import_hook/__main__.py b/src/maturin_import_hook/__main__.py index 3f8fb31..90e9ece 100644 --- a/src/maturin_import_hook/__main__.py +++ b/src/maturin_import_hook/__main__.py @@ -23,12 +23,12 @@ def _action_version(format_name: str) -> None: try: maturin_version = subprocess.check_output(["maturin", "--version"]).decode().strip() - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): maturin_version = "?" try: rustc_version = subprocess.check_output(["rustc", "--version"]).decode().strip() - except subprocess.CalledProcessError: + except (subprocess.CalledProcessError, FileNotFoundError): rustc_version = "?" try: From 59756732a7e16e9539fbe7cab5f01e7369dd802a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 15:00:12 +0100 Subject: [PATCH 2/6] removed documentation that moved to maturin repo --- docs/basics.md | 126 +------------------------------------------------ 1 file changed, 1 insertion(+), 125 deletions(-) diff --git a/docs/basics.md b/docs/basics.md index 1d48615..1edcd05 100644 --- a/docs/basics.md +++ b/docs/basics.md @@ -1,127 +1,3 @@ # Basics -`maturin_import_hook` is a package that provides the capability for python `import` statements to trigger a rebuild -when importing a maturin project that is installed in editable mode (eg with `maturin develop` or `pip install -e`). -This makes development much more convenient as it brings the workflow of -developing Rust modules closer to the workflow of developing regular python modules. - -The hook supports importing editable-installed pure Rust and mixed Rust/Python project -layouts as well as importing standalone `.rs` files. - -## Installation - -Install into a virtual environment then install so that the import hook is always active. - -```shell -pip install maturin_import_hook -python -m maturin_import_hook site install # install into the active environment -``` - -Alternatively, instead of using `site install`, put calls to `maturin_import_hook.install()` into any script where you -want to use the import hook. - -## Usage - -```python -import maturin_import_hook - -# install the import hook with default settings. -# this must be called before importing any maturin project -maturin_import_hook.install() - -# when a rust package that is installed in editable mode is imported, -# that package will be automatically recompiled if necessary. -import pyo3_pure - -# when a .rs file is imported a project will be created for it in the -# maturin build cache and the resulting library will be loaded. -# -# assuming subpackage/my_rust_script.rs defines a pyo3 module: -import subpackage.my_rust_script -``` - -The maturin project importer and the rust file importer can be used separately - -```python -from maturin_import_hook import rust_file_importer -rust_file_importer.install() - -from maturin_import_hook import project_importer -project_importer.install() -``` - -The import hook can be configured to control its behaviour - -```python -import maturin_import_hook -from maturin_import_hook.settings import MaturinSettings - -maturin_import_hook.install( - enable_project_importer=True, - enable_rs_file_importer=True, - settings=MaturinSettings( - release=True, - strip=True, - # ... - ), - show_warnings=True, - # ... -) -``` - -Since the import hook is intended for use in development environments and not for -production environments, it may be a good idea to put the call to `maturin_import_hook.install()` -into `site-packages/sitecustomize.py` of your development virtual environment -([documentation](https://docs.python.org/3/library/site.html)). This will -enable the hook for every script run by that interpreter without calling `maturin_import_hook.install()` -in every script, meaning the scripts do not need alteration before deployment. - -Installation into `sitecustomize.py` can be managed with the import hook cli using -`python -m maturin_import_hook site install`. The CLI can also manage uninstallation. - -## CLI - -The package provides a CLI interface for getting information such as the location and size of the build cache and -managing the installation into `sitecustomize.py`. For details, run: - -```shell -python -m maturin_import_hook --help -``` - -## Environment Variables - -The import hook can be disabled by setting `MATURIN_IMPORT_HOOK_ENABLED=0`. This can be used to disable -the import hook in production if you want to leave calls to `import_hook.install()` in place. - -Build files will be stored in an appropriate place for the current system but can be overridden -by setting `MATURIN_BUILD_DIR`. These files can be deleted without causing any issues (unless a build is in progress). -The precedence for storing build files is: - -- `MATURIN_BUILD_DIR` -- `/maturin_build_cache` -- `/maturin_build_cache` - - e.g. `~/.cache/maturin_build_cache` on POSIX - -See the location being used with the CLI: `python -m maturin_import_hook cache info` - -## Logging - -By default the `maturin_import_hook` logger does not propagate to the root logger. This is so that `INFO` level messages -are shown to the user without them having to configure logging (`INFO` level is normally not visible). The import hook -also has extensive `DEBUG` level logging that generally would be more noise than useful. So by not propagating, `DEBUG` -messages from the import hook are not shown even if the root logger has `DEBUG` level visible. - -If you prefer, `maturin_import_hook.reset_logger()` can be called to undo the default configuration and propagate -the messages as normal. - -When debugging issues with the import hook, you should first call `reset_logger()` then configure the root logger -to show `DEBUG` messages. You can also run with the environment variable `RUST_LOG=maturin=debug` to get more -information from maturin. - -```python -import logging -logging.basicConfig(format='%(name)s [%(levelname)s] %(message)s', level=logging.DEBUG) -import maturin_import_hook -maturin_import_hook.reset_logger() -maturin_import_hook.install() -``` +See [maturin.rs](https://maturin.rs/import_hook). From 0866441c4e84628601aa3c8b9759f3006f28a6f2 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 15:11:03 +0100 Subject: [PATCH 3/6] added force and preset options to site install --- src/maturin_import_hook/__main__.py | 23 ++++++++--- src/maturin_import_hook/_site.py | 40 +++++++++++++++---- tests/test_import_hook/test_site.py | 60 ++++++++++++++++++++++++++--- 3 files changed, 104 insertions(+), 19 deletions(-) diff --git a/src/maturin_import_hook/__main__.py b/src/maturin_import_hook/__main__.py index 90e9ece..90fdbb4 100644 --- a/src/maturin_import_hook/__main__.py +++ b/src/maturin_import_hook/__main__.py @@ -83,16 +83,17 @@ def _action_site_info(format_name: str) -> None: _print_info( { - "has_sitecustomize": sitecustomize_path.exists(), + "sitecustomize_exists": sitecustomize_path.exists(), + "sitecustomize_path": str(sitecustomize_path), "import_hook_installed": has_automatic_installation(sitecustomize_path), }, format_name, ) -def _action_site_install() -> None: +def _action_site_install(preset_name: str, force: bool) -> None: sitecustomize_path = get_sitecustomize_path() - insert_automatic_installation(sitecustomize_path) + insert_automatic_installation(sitecustomize_path, preset_name, force) def _action_site_uninstall() -> None: @@ -157,9 +158,21 @@ def _main() -> None: site_info.add_argument( "-f", "--format", choices=["text", "json"], default="text", help="the format to output the data in" ) - site_sub_actions.add_parser( + install = site_sub_actions.add_parser( "install", help="install the import hook into site-packages/sitecustomize.py so that it starts automatically" ) + install.add_argument( + "-f", + "--force", + action="store_true", + help="whether to overwrite any existing managed import hook installation in sitecustomize.py", + ) + install.add_argument( + "--preset", + default="debug", + choices=["debug", "release"], + help="the settings preset for the import hook to use when building packages. Defaults to 'debug'.", + ) site_sub_actions.add_parser("uninstall", help="uninstall the import hook from site-packages/sitecustomize.py") args = parser.parse_args() @@ -179,7 +192,7 @@ def _main() -> None: if args.sub_action == "info": _action_site_info(args.format) elif args.sub_action == "install": - _action_site_install() + _action_site_install(args.preset, args.force) elif args.sub_action == "uninstall": _action_site_uninstall() else: diff --git a/src/maturin_import_hook/_site.py b/src/maturin_import_hook/_site.py index 4892e9e..c97f5bc 100644 --- a/src/maturin_import_hook/_site.py +++ b/src/maturin_import_hook/_site.py @@ -1,17 +1,28 @@ import site from pathlib import Path +from textwrap import dedent from maturin_import_hook._logging import logger MANAGED_INSTALL_START = "# " MANAGED_INSTALL_END = "# \n" -MANAGED_INSTALLATION = """ -# this section of code installs the maturin import hook into every interpreter. +MANAGED_INSTALL_COMMENT = """ +# the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` -import maturin_import_hook -maturin_import_hook.install() """ +MANAGED_INSTALLATION_PRESETS = { + "debug": dedent("""\ + import maturin_import_hook + maturin_import_hook.install() + """), + "release": dedent("""\ + import maturin_import_hook + from maturin_import_hook.settings import MaturinSettings + maturin_import_hook.install(MaturinSettings(release=True)) + """), +} + def get_sitecustomize_path() -> Path: site_packages = site.getsitepackages() @@ -56,16 +67,29 @@ def remove_automatic_installation(sitecustomize: Path) -> None: sitecustomize.unlink(missing_ok=True) -def insert_automatic_installation(sitecustomize: Path) -> None: +def insert_automatic_installation(sitecustomize: Path, preset_name: str, force: bool) -> None: + if preset_name not in MANAGED_INSTALLATION_PRESETS: + msg = f"Unknown managed installation preset name: '{preset_name}'" + raise ValueError(msg) + logger.info(f"installing automatic activation into '{sitecustomize}'") if has_automatic_installation(sitecustomize): - logger.info("already installed") - return + if force: + logger.info("already installed, but force=True. Overwriting...") + remove_automatic_installation(sitecustomize) + else: + logger.info("already installed. Aborting install") + return parts = [] if sitecustomize.exists(): parts.append(sitecustomize.read_text()) parts.append("\n") - parts.extend([MANAGED_INSTALL_START, MANAGED_INSTALLATION, MANAGED_INSTALL_END]) + parts.extend([ + MANAGED_INSTALL_START, + MANAGED_INSTALL_COMMENT, + MANAGED_INSTALLATION_PRESETS[preset_name], + MANAGED_INSTALL_END, + ]) code = "".join(parts) sitecustomize.write_text(code) diff --git a/tests/test_import_hook/test_site.py b/tests/test_import_hook/test_site.py index c0805bf..da2e5f6 100644 --- a/tests/test_import_hook/test_site.py +++ b/tests/test_import_hook/test_site.py @@ -1,6 +1,7 @@ from pathlib import Path from textwrap import dedent +import pytest from maturin_import_hook._site import ( has_automatic_installation, insert_automatic_installation, @@ -22,12 +23,12 @@ def test_automatic_site_installation(tmp_path: Path) -> None: assert not has_automatic_installation(sitecustomize) - insert_automatic_installation(sitecustomize) + insert_automatic_installation(sitecustomize, preset_name="debug", force=False) with capture_logs() as cap: - insert_automatic_installation(sitecustomize) + insert_automatic_installation(sitecustomize, preset_name="debug", force=False) logs = cap.getvalue() - assert "already installed" in logs + assert "already installed. Aborting install" in logs expected_code = dedent("""\ # some existing code @@ -35,7 +36,7 @@ def test_automatic_site_installation(tmp_path: Path) -> None: install() # another import hook # - # this section of code installs the maturin import hook into every interpreter. + # the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` import maturin_import_hook maturin_import_hook.install() @@ -68,13 +69,60 @@ def test_automatic_site_installation(tmp_path: Path) -> None: assert "no installation found" in logs +def test_automatic_site_installation_force_overwrite(tmp_path: Path) -> None: + sitecustomize = tmp_path / "sitecustomize.py" + header = dedent("""\ + # some existing code + print(123) + install() # another import hook + """) + + sitecustomize.write_text(header) + + insert_automatic_installation(sitecustomize, preset_name="debug", force=False) + + sitecustomize.write_text(sitecustomize.read_text() + "\n\n# more code") + + with capture_logs() as cap: + insert_automatic_installation(sitecustomize, preset_name="release", force=True) + logs = cap.getvalue() + assert "already installed, but force=True. Overwriting..." in logs + + expected_code = dedent("""\ + # some existing code + print(123) + install() # another import hook + + + + # more code + # + # the following commands install the maturin import hook during startup. + # see: `python -m maturin_import_hook site` + import maturin_import_hook + from maturin_import_hook.settings import MaturinSettings + maturin_import_hook.install(MaturinSettings(release=True)) + # + """) + + assert sitecustomize.read_text() == expected_code + assert has_automatic_installation(sitecustomize) + + +def test_automatic_site_installation_invalid_preset(tmp_path: Path) -> None: + sitecustomize = tmp_path / "sitecustomize.py" + with pytest.raises(ValueError, match="Unknown managed installation preset name: 'foo'"): + insert_automatic_installation(sitecustomize, preset_name="foo", force=False) + assert not sitecustomize.exists() + + def test_automatic_site_installation_empty(tmp_path: Path) -> None: sitecustomize = tmp_path / "sitecustomize.py" - insert_automatic_installation(sitecustomize) + insert_automatic_installation(sitecustomize, preset_name="debug", force=False) expected_code = dedent("""\ # - # this section of code installs the maturin import hook into every interpreter. + # the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` import maturin_import_hook maturin_import_hook.install() From b9be6ecb785ee891272e0e9bca9ffeee1ef7c33a Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 16:03:58 +0100 Subject: [PATCH 4/6] added option to install to usercustomize.py --- src/maturin_import_hook/__main__.py | 62 ++++++++++++----- src/maturin_import_hook/_site.py | 66 ++++++++++++------- src/maturin_import_hook/project_importer.py | 4 ++ src/maturin_import_hook/rust_file_importer.py | 4 ++ tests/test_import_hook/test_site.py | 26 ++++++-- 5 files changed, 116 insertions(+), 46 deletions(-) diff --git a/src/maturin_import_hook/__main__.py b/src/maturin_import_hook/__main__.py index 90fdbb4..6f9559d 100644 --- a/src/maturin_import_hook/__main__.py +++ b/src/maturin_import_hook/__main__.py @@ -3,12 +3,14 @@ import json import platform import shutil +import site import subprocess from pathlib import Path from maturin_import_hook._building import get_default_build_dir from maturin_import_hook._site import ( get_sitecustomize_path, + get_usercustomize_path, has_automatic_installation, insert_automatic_installation, remove_automatic_installation, @@ -80,25 +82,30 @@ def _action_cache_clear(interactive: bool) -> None: def _action_site_info(format_name: str) -> None: sitecustomize_path = get_sitecustomize_path() + usercustomize_path = get_usercustomize_path() _print_info( { - "sitecustomize_exists": sitecustomize_path.exists(), "sitecustomize_path": str(sitecustomize_path), - "import_hook_installed": has_automatic_installation(sitecustomize_path), + "sitecustomize_exists": sitecustomize_path.exists(), + "sitecustomize_import_hook_installed": has_automatic_installation(sitecustomize_path), + "user_site_enabled": str(site.ENABLE_USER_SITE), + "usercustomize_path": str(usercustomize_path), + "usercustomize_exists": usercustomize_path.exists(), + "usercustomize_import_hook_installed": has_automatic_installation(usercustomize_path), }, format_name, ) -def _action_site_install(preset_name: str, force: bool) -> None: - sitecustomize_path = get_sitecustomize_path() - insert_automatic_installation(sitecustomize_path, preset_name, force) +def _action_site_install(*, user: bool, preset_name: str, force: bool) -> None: + module_path = get_usercustomize_path() if user else get_sitecustomize_path() + insert_automatic_installation(module_path, preset_name, force) -def _action_site_uninstall() -> None: - sitecustomize_path = get_sitecustomize_path() - remove_automatic_installation(sitecustomize_path) +def _action_site_uninstall(*, user: bool) -> None: + module_path = get_usercustomize_path() if user else get_sitecustomize_path() + remove_automatic_installation(module_path) def _ask_yes_no(question: str) -> bool: @@ -149,23 +156,31 @@ def _main() -> None: site_action = subparsers.add_parser( "site", - help="manage installation of the import hook into site-packages/sitecustomize.py (so it starts automatically)", + help=( + "manage installation of the import hook into site-packages/sitecustomize.py " + "or usercustomize.py (so it starts automatically)" + ), ) site_sub_actions = site_action.add_subparsers(dest="sub_action") site_info = site_sub_actions.add_parser( - "info", help="information about the current status of installation into sitecustomize" + "info", help="information about the current status of installation into sitecustomize/usercustomize" ) site_info.add_argument( "-f", "--format", choices=["text", "json"], default="text", help="the format to output the data in" ) + install = site_sub_actions.add_parser( - "install", help="install the import hook into site-packages/sitecustomize.py so that it starts automatically" + "install", + help=( + "install the import hook into site-packages/sitecustomize.py " + "or usercustomize.py so that it starts automatically" + ), ) install.add_argument( "-f", "--force", action="store_true", - help="whether to overwrite any existing managed import hook installation in sitecustomize.py", + help="whether to overwrite any existing managed import hook installation", ) install.add_argument( "--preset", @@ -173,7 +188,24 @@ def _main() -> None: choices=["debug", "release"], help="the settings preset for the import hook to use when building packages. Defaults to 'debug'.", ) - site_sub_actions.add_parser("uninstall", help="uninstall the import hook from site-packages/sitecustomize.py") + install.add_argument( + "--user", + action="store_true", + help="whether to install into usercustomize.py instead of sitecustomize.py. " + "Note that usercustomize.py is shared between virtualenvs of the same interpreter version and is not loaded " + "unless the virtualenv is created with the `--system-site-packages` argument. Use `site info` to check " + "whether usercustomize.py is loaded the current interpreter.", + ) + + uninstall = site_sub_actions.add_parser( + "uninstall", + help="uninstall the import hook from site-packages/sitecustomize.py or site-packages/usercustomize.py", + ) + uninstall.add_argument( + "--user", + action="store_true", + help="whether to uninstall from usercustomize.py instead of sitecustomize.py", + ) args = parser.parse_args() @@ -192,9 +224,9 @@ def _main() -> None: if args.sub_action == "info": _action_site_info(args.format) elif args.sub_action == "install": - _action_site_install(args.preset, args.force) + _action_site_install(user=args.user, preset_name=args.preset, force=args.force) elif args.sub_action == "uninstall": - _action_site_uninstall() + _action_site_uninstall(user=args.user) else: site_action.print_help() else: diff --git a/src/maturin_import_hook/_site.py b/src/maturin_import_hook/_site.py index c97f5bc..655b3b4 100644 --- a/src/maturin_import_hook/_site.py +++ b/src/maturin_import_hook/_site.py @@ -13,13 +13,21 @@ MANAGED_INSTALLATION_PRESETS = { "debug": dedent("""\ - import maturin_import_hook - maturin_import_hook.install() + try: + import maturin_import_hook + except ImportError: + pass + else: + maturin_import_hook.install() """), "release": dedent("""\ - import maturin_import_hook - from maturin_import_hook.settings import MaturinSettings - maturin_import_hook.install(MaturinSettings(release=True)) + try: + import maturin_import_hook + from maturin_import_hook.settings import MaturinSettings + except ImportError: + pass + else: + maturin_import_hook.install(MaturinSettings(release=True)) """), } @@ -36,54 +44,62 @@ def get_sitecustomize_path() -> Path: return Path(site_packages[0]) / "sitecustomize.py" -def has_automatic_installation(sitecustomize: Path) -> bool: - if not sitecustomize.is_file(): +def get_usercustomize_path() -> Path: + user_site_packages = site.getusersitepackages() + if user_site_packages is None: + msg = "could not find usercustomize.py (user site-packages not found)" + raise FileNotFoundError(msg) + return Path(user_site_packages) / "usercustomize.py" + + +def has_automatic_installation(module_path: Path) -> bool: + if not module_path.is_file(): return False - code = sitecustomize.read_text() + code = module_path.read_text() return MANAGED_INSTALL_START in code -def remove_automatic_installation(sitecustomize: Path) -> None: - logger.info(f"removing automatic activation from '{sitecustomize}'") - if not has_automatic_installation(sitecustomize): +def remove_automatic_installation(module_path: Path) -> None: + logger.info(f"removing automatic activation from '{module_path}'") + if not has_automatic_installation(module_path): logger.info("no installation found") return - code = sitecustomize.read_text() + code = module_path.read_text() managed_start = code.find(MANAGED_INSTALL_START) if managed_start == -1: - msg = f"failed to find managed install start marker in '{sitecustomize}'" + msg = f"failed to find managed install start marker in '{module_path}'" raise RuntimeError(msg) managed_end = code.find(MANAGED_INSTALL_END) if managed_end == -1: - msg = f"failed to find managed install start marker in '{sitecustomize}'" + msg = f"failed to find managed install start marker in '{module_path}'" raise RuntimeError(msg) code = code[:managed_start] + code[managed_end + len(MANAGED_INSTALL_END) :] if code.strip(): - sitecustomize.write_text(code) + module_path.write_text(code) else: logger.info("module is now empty. Removing file.") - sitecustomize.unlink(missing_ok=True) + module_path.unlink(missing_ok=True) -def insert_automatic_installation(sitecustomize: Path, preset_name: str, force: bool) -> None: +def insert_automatic_installation(module_path: Path, preset_name: str, force: bool) -> None: if preset_name not in MANAGED_INSTALLATION_PRESETS: msg = f"Unknown managed installation preset name: '{preset_name}'" raise ValueError(msg) - logger.info(f"installing automatic activation into '{sitecustomize}'") - if has_automatic_installation(sitecustomize): + logger.info(f"installing automatic activation into '{module_path}'") + if has_automatic_installation(module_path): if force: logger.info("already installed, but force=True. Overwriting...") - remove_automatic_installation(sitecustomize) + remove_automatic_installation(module_path) else: - logger.info("already installed. Aborting install") + logger.info("already installed. Aborting install.") return parts = [] - if sitecustomize.exists(): - parts.append(sitecustomize.read_text()) + if module_path.exists(): + parts.append(module_path.read_text()) parts.append("\n") parts.extend([ MANAGED_INSTALL_START, @@ -92,4 +108,6 @@ def insert_automatic_installation(sitecustomize: Path, preset_name: str, force: MANAGED_INSTALL_END, ]) code = "".join(parts) - sitecustomize.write_text(code) + module_path.parent.mkdir(parents=True, exist_ok=True) + module_path.write_text(code) + logger.info("automatic activation written successfully.") diff --git a/src/maturin_import_hook/project_importer.py b/src/maturin_import_hook/project_importer.py index acdcdb1..7f3367c 100644 --- a/src/maturin_import_hook/project_importer.py +++ b/src/maturin_import_hook/project_importer.py @@ -641,3 +641,7 @@ def uninstall() -> None: with contextlib.suppress(ValueError): sys.meta_path.remove(IMPORTER) IMPORTER = None + + +def is_installed() -> bool: + return IMPORTER is not None and IMPORTER in sys.meta_path diff --git a/src/maturin_import_hook/rust_file_importer.py b/src/maturin_import_hook/rust_file_importer.py index 92e4bb0..c50d0fd 100644 --- a/src/maturin_import_hook/rust_file_importer.py +++ b/src/maturin_import_hook/rust_file_importer.py @@ -420,3 +420,7 @@ def uninstall() -> None: with contextlib.suppress(ValueError): sys.meta_path.remove(IMPORTER) IMPORTER = None + + +def is_installed() -> bool: + return IMPORTER is not None and IMPORTER in sys.meta_path diff --git a/tests/test_import_hook/test_site.py b/tests/test_import_hook/test_site.py index da2e5f6..bc8a915 100644 --- a/tests/test_import_hook/test_site.py +++ b/tests/test_import_hook/test_site.py @@ -38,8 +38,12 @@ def test_automatic_site_installation(tmp_path: Path) -> None: # # the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` - import maturin_import_hook - maturin_import_hook.install() + try: + import maturin_import_hook + except ImportError: + pass + else: + maturin_import_hook.install() # """) @@ -99,9 +103,13 @@ def test_automatic_site_installation_force_overwrite(tmp_path: Path) -> None: # # the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` - import maturin_import_hook - from maturin_import_hook.settings import MaturinSettings - maturin_import_hook.install(MaturinSettings(release=True)) + try: + import maturin_import_hook + from maturin_import_hook.settings import MaturinSettings + except ImportError: + pass + else: + maturin_import_hook.install(MaturinSettings(release=True)) # """) @@ -124,8 +132,12 @@ def test_automatic_site_installation_empty(tmp_path: Path) -> None: # # the following commands install the maturin import hook during startup. # see: `python -m maturin_import_hook site` - import maturin_import_hook - maturin_import_hook.install() + try: + import maturin_import_hook + except ImportError: + pass + else: + maturin_import_hook.install() # """) From 6b250c7792eaae5e19f58e08e180a4721993a1e4 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 17:12:19 +0100 Subject: [PATCH 5/6] use uv for creating virtual environments and installing packages when available --- tests/requirements.txt | 2 +- tests/runner.py | 43 ++++++++++--- .../test_import_hook/test_project_importer.py | 60 ++++++++++++++----- 3 files changed, 81 insertions(+), 24 deletions(-) diff --git a/tests/requirements.txt b/tests/requirements.txt index 9d0879b..6252065 100644 --- a/tests/requirements.txt +++ b/tests/requirements.txt @@ -1,5 +1,5 @@ -e .. -virtualenv +uv maturin==1.5.0 pytest junit2html diff --git a/tests/runner.py b/tests/runner.py index c5e7bcf..db58b66 100644 --- a/tests/runner.py +++ b/tests/runner.py @@ -100,16 +100,33 @@ def _run_test_in_environment( sys.exit(proc.returncode) +def _pip_install_command(interpreter_path: Path) -> list[str]: + if shutil.which("uv") is not None: + log.info("using uv to install packages") + return [ + "uv", + "pip", + "install", + "--python", + str(interpreter_path), + ] + else: + log.info("using pip to install packages") + return [ + str(interpreter_path), + "-m", + "pip", + "install", + "--disable-pip-version-check", + ] + + def _create_test_venv(python: Path, venv_dir: Path) -> VirtualEnv: venv = VirtualEnv.new(venv_dir, python) log.info("installing test requirements into virtualenv") proc = subprocess.run( [ - str(venv.interpreter_path), - "-m", - "pip", - "install", - "--disable-pip-version-check", + *_pip_install_command(venv.interpreter_path), "-r", "requirements.txt", ], @@ -120,13 +137,25 @@ def _create_test_venv(python: Path, venv_dir: Path) -> VirtualEnv: if proc.returncode != 0: log.error(proc.stdout.decode()) log.error(proc.stderr.decode()) - msg = "pip install failed" + msg = "package installation failed" raise RuntimeError(msg) log.debug("%s", proc.stdout.decode()) log.info("test environment ready") return venv +def _create_virtual_env_command(interpreter_path: Path, venv_path: Path) -> list[str]: + if shutil.which("uv") is not None: + log.info("using uv to create virtual environments") + return ["uv", "venv", "--seed", "--python", str(interpreter_path), str(venv_path)] + elif shutil.which("virtualenv") is not None: + log.info("using virtualenv to create virtual environments") + return ["virtualenv", "--python", str(interpreter_path), str(venv_path)] + else: + log.info("using venv to create virtual environments") + return [str(interpreter_path), "-m", "venv", str(venv_path)] + + class VirtualEnv: def __init__(self, root: Path) -> None: self._root = root.resolve() @@ -140,7 +169,7 @@ def new(root: Path, interpreter_path: Path) -> VirtualEnv: if not interpreter_path.exists(): raise FileNotFoundError(interpreter_path) log.info("creating test virtualenv at '%s' from '%s'", root, interpreter_path) - cmd = ["virtualenv", "--python", str(interpreter_path), str(root)] + cmd = _create_virtual_env_command(interpreter_path, root) proc = subprocess.run(cmd, capture_output=True, check=True) log.debug("%s", proc.stdout.decode()) assert root.is_dir() diff --git a/tests/test_import_hook/test_project_importer.py b/tests/test_import_hook/test_project_importer.py index f40556d..42fdb43 100644 --- a/tests/test_import_hook/test_project_importer.py +++ b/tests/test_import_hook/test_project_importer.py @@ -10,6 +10,7 @@ from collections.abc import Iterator from pathlib import Path from textwrap import dedent +from typing import Optional import pytest from maturin_import_hook.project_importer import DefaultProjectFileSearcher, _load_dist_info @@ -1342,30 +1343,47 @@ def _rebuilt_message(project_name: str) -> str: return f'rebuilt and loaded package "{with_underscores(project_name)}"' +_UV_AVAILABLE: Optional[bool] = None + + +def uv_available() -> bool: + """whether the `uv` command is installed""" + global _UV_AVAILABLE + if _UV_AVAILABLE is None: + _UV_AVAILABLE = shutil.which("uv") is not None + return _UV_AVAILABLE + + def _uninstall(*project_names: str) -> None: log.info("uninstalling %s", sorted(project_names)) - subprocess.check_call([ - sys.executable, - "-m", - "pip", - "uninstall", - "--disable-pip-version-check", - "-y", - *project_names, - ]) + if uv_available(): + cmd = ["uv", "pip", "uninstall", "--python", str(sys.executable), *project_names] + else: + cmd = [ + sys.executable, + "-m", + "pip", + "uninstall", + "--disable-pip-version-check", + "-y", + *project_names, + ] + subprocess.check_call(cmd) def _get_installed_package_names() -> set[str]: - packages = json.loads( - subprocess.check_output([ + if uv_available(): + cmd = ["uv", "pip", "list", "--python", sys.executable, "--format", "json"] + else: + cmd = [ sys.executable, "-m", "pip", "--disable-pip-version-check", "list", "--format=json", - ]).decode() - ) + ] + packages = json.loads(subprocess.check_output(cmd).decode()) return {package["name"] for package in packages} @@ -1382,7 +1400,11 @@ def _install_editable(project_dir: Path) -> None: def _install_non_editable(project_dir: Path) -> None: log.info("installing %s in non-editable mode", project_dir.name) - subprocess.check_call([sys.executable, "-m", "pip", "install", "--disable-pip-version-check", str(project_dir)]) + if uv_available(): + cmd = ["uv", "pip", "install", "--python", sys.executable, str(project_dir)] + else: + cmd = [sys.executable, "-m", "pip", "install", "--disable-pip-version-check", str(project_dir)] + subprocess.check_call(cmd) def _is_installed_as_pth(project_name: str) -> bool: @@ -1415,14 +1437,20 @@ def _is_editable_installed_correctly(project_name: str, project_dir: Path, is_mi installed_editable_with_direct_url, ) + if uv_available(): + # TODO(matt): use uv once the --files option is supported https://github.com/astral-sh/uv/issues/2526 + cmd = [sys.executable, "-m", "pip", "show", "--disable-pip-version-check", "-f", project_name] + else: + cmd = [sys.executable, "-m", "pip", "show", "--disable-pip-version-check", "-f", project_name] + proc = subprocess.run( - [sys.executable, "-m", "pip", "show", "--disable-pip-version-check", "-f", project_name], + cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False, ) output = "None" if proc.stdout is None else proc.stdout.decode() - log.info("pip output (returned %s):\n%s", proc.returncode, output) + log.info("command output (returned %s):\n%s", proc.returncode, output) return installed_editable_with_direct_url and (installed_as_pth == is_mixed) From f71d92ac8c3db96e9002d8967a50ec640e562133 Mon Sep 17 00:00:00 2001 From: Matthew Broadway Date: Sun, 16 Jun 2024 17:51:20 +0100 Subject: [PATCH 6/6] added link to documentation --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 296ccea..361fed1 100644 --- a/README.md +++ b/README.md @@ -8,6 +8,8 @@ components take effect automatically like changes to python components do. The import hook also provides conveniences such as [importlib.reload()](https://docs.python.org/3/library/importlib.html#importlib.reload) support for maturin projects. +Documentation can be found at [maturin.rs](https://www.maturin.rs/import_hook). + ## Usage After installing [maturin](https://www.maturin.rs/installation), install the import hook into a python virtual