From aabd6a68023adf59eef3c882b7283efaef488c8b Mon Sep 17 00:00:00 2001 From: chrysle Date: Sun, 8 Oct 2023 15:27:26 +0200 Subject: [PATCH 1/5] Drop `wheel` and `setuptools` from shared libs And enable running the remaining shared library `pip` using `pipx run`. --- CHANGELOG.md | 2 ++ docs/how-pipx-works.md | 8 ++++---- src/pipx/commands/install.py | 7 ++++++- src/pipx/commands/run.py | 7 ++++++- src/pipx/shared_libs.py | 6 ++---- src/pipx/venv.py | 34 +++++++++++++++++++++------------- tests/test_run.py | 7 +++++++ 7 files changed, 48 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index da15e33efd..0d529d50de 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## dev +- Drop `setuptools` and `wheel` from the shared libraries +- Allow running `pip` with `pipx run` - Make usage message in `pipx run` show `package_or_url`, so extra will be printed out as well - Add `--force-reinstall` to pip arguments when `--force` was passed - Use the py launcher, if available, to select Python version with the `--python` option diff --git a/docs/how-pipx-works.md b/docs/how-pipx-works.md index 622543d47a..17a75efac5 100644 --- a/docs/how-pipx-works.md +++ b/docs/how-pipx-works.md @@ -3,8 +3,8 @@ When installing a package and its binaries on linux (`pipx install package`) pipx will - create directory `~/.local/share/pipx/venvs/PACKAGE` -- create or re-use a shared virtual environment that contains shared packaging libraries `pip`, `setuptools` and `wheel` in `~/.local/share/pipx/shared/` -- ensure all packaging libraries are updated to their latest versions +- create or re-use a shared virtual environment that contains shared packaging library `pip` in `~/.local/share/pipx/shared/` +- ensure the library is updated to its latest version - create a Virtual Environment in `~/.local/share/pipx/venvs/PACKAGE` that uses the shared pip mentioned above but otherwise is isolated (pipx uses a [.pth file]( https://docs.python.org/3/library/site.html) to do this) - install the desired package in the Virtual Environment - expose binaries at `~/.local/bin` that point to new binaries in `~/.local/share/pipx/venvs/PACKAGE/bin` (such as `~/.local/bin/black` -> `~/.local/share/pipx/venvs/black/bin/black`) @@ -12,8 +12,8 @@ When installing a package and its binaries on linux (`pipx install package`) pip When running a binary (`pipx run BINARY`), pipx will -- create or re-use a shared virtual environment that contains shared packaging libraries `pip`, `setuptools` and `wheel` in `~/.local/share/pipx/shared/` -- ensure all packaging libraries are updated to their latest versions +- create or re-use a shared virtual environment that contains the shared packaging library `pip` +- ensure the library is updated to its latest version - create a temporary directory (or reuse a cached virtual environment for this package) with a name based on a hash of the attributes that make the run reproducible. This includes things like the package name, spec, python version, and pip arguments. - create a Virtual Environment inside it with `python -m venv` - install the desired package in the Virtual Environment diff --git a/src/pipx/commands/install.py b/src/pipx/commands/install.py index 8d6ae2b94a..abad14c186 100644 --- a/src/pipx/commands/install.py +++ b/src/pipx/commands/install.py @@ -58,7 +58,12 @@ def install( return EXIT_CODE_INSTALL_VENV_EXISTS try: - venv.create_venv(venv_args, pip_args) + # Enable installing shared library `pip` with `pipx` + override_shared = False + + if package_name == "pip": + override_shared = True + venv.create_venv(venv_args, pip_args, override_shared) for dep in preinstall_packages or []: dep_name = package_name_from_spec( dep, python, pip_args=pip_args, verbose=verbose diff --git a/src/pipx/commands/run.py b/src/pipx/commands/run.py index 15428e081c..383eb35079 100644 --- a/src/pipx/commands/run.py +++ b/src/pipx/commands/run.py @@ -218,7 +218,6 @@ def _download_and_run( verbose: bool, ) -> NoReturn: venv = Venv(venv_dir, python=python, verbose=verbose) - venv.create_venv(venv_args, pip_args) if venv.pipx_metadata.main_package.package is not None: package_name = venv.pipx_metadata.main_package.package @@ -227,6 +226,12 @@ def _download_and_run( package_or_url, python, pip_args=pip_args, verbose=verbose ) + override_shared = False + + if package_name == "pip": + override_shared = True + + venv.create_venv(venv_args, pip_args, override_shared) venv.install_package( package_name=package_name, package_or_url=package_or_url, diff --git a/src/pipx/shared_libs.py b/src/pipx/shared_libs.py index 5a752c9bd9..625ad98f06 100644 --- a/src/pipx/shared_libs.py +++ b/src/pipx/shared_libs.py @@ -26,8 +26,8 @@ def __init__(self) -> None: self.root = constants.PIPX_SHARED_LIBS self.bin_path, self.python_path = get_venv_paths(self.root) self.pip_path = self.bin_path / ("pip" if not WINDOWS else "pip.exe") - # i.e. bin_path is ~/.local/pipx/shared/bin - # i.e. python_path is ~/.local/pipx/shared/python + # i.e. bin_path is ~/.local/share/pipx/shared/bin + # i.e. python_path is ~/.local/share/pipx/shared/python self._site_packages: Optional[Path] = None self.has_been_updated_this_run = False self.has_been_logged_this_run = False @@ -107,8 +107,6 @@ def upgrade( *_pip_args, "--upgrade", "pip", - "setuptools", - "wheel", ] ) subprocess_post_check(upgrade_process) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 28189cc510..70921e86dc 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -156,24 +156,32 @@ def main_package_name(self) -> str: else: return self.pipx_metadata.main_package.package - def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: + def create_venv( + self, venv_args: List[str], pip_args: List[str], override_shared: bool = False + ) -> None: + """ + override_shared -- Override installing shared libraries to the pipx shared directory (default False) + """ with animate("creating virtual environment", self.do_animation): - cmd = [self.python, "-m", "venv", "--without-pip"] + cmd = [self.python, "-m", "venv"] + if not override_shared: + cmd.append("--without-pip") venv_process = run_subprocess(cmd + venv_args + [str(self.root)]) subprocess_post_check(venv_process) shared_libs.create(self.verbose) - pipx_pth = get_site_packages(self.python_path) / PIPX_SHARED_PTH - # write path pointing to the shared libs site-packages directory - # example pipx_pth location: - # ~/.local/pipx/venvs/black/lib/python3.8/site-packages/pipx_shared.pth - # example shared_libs.site_packages location: - # ~/.local/pipx/shared/lib/python3.6/site-packages - # - # https://docs.python.org/3/library/site.html - # A path configuration file is a file whose name has the form 'name.pth'. - # its contents are additional items (one per line) to be added to sys.path - pipx_pth.write_text(f"{shared_libs.site_packages}\n", encoding="utf-8") + if not override_shared: + pipx_pth = get_site_packages(self.python_path) / PIPX_SHARED_PTH + # write path pointing to the shared libs site-packages directory + # example pipx_pth location: + # ~/.local/share/pipx/venvs/black/lib/python3.8/site-packages/pipx_shared.pth + # example shared_libs.site_packages location: + # ~/.local/share/pipx/shared/lib/python3.6/site-packages + # + # https://docs.python.org/3/library/site.html + # A path configuration file is a file whose name has the form 'name.pth'. + # its contents are additional items (one per line) to be added to sys.path + pipx_pth.write_text(f"{shared_libs.site_packages}\n", encoding="utf-8") self.pipx_metadata.venv_args = venv_args self.pipx_metadata.python_version = self.get_python_version() diff --git a/tests/test_run.py b/tests/test_run.py index f70c140eac..98ad312718 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -334,3 +334,10 @@ def test_run_with_windows_python_version(caplog, pipx_temp_env, tmp_path): ) run_pipx_cli_exit(["run", script.as_uri(), "--python", "3.11"]) assert "3.11" in out.read_text() + + +@mock.patch("os.execvpe", new=execvpe_mock) +def test_run_shared_lib_as_app(pipx_temp_env, monkeypatch, capfd): + run_pipx_cli_exit(["run", "pip", "--help"]) + captured = capfd.readouterr() + assert "pip [options]\n" in captured.out From 51a010bad444633cec58b57c3a69b534355f4360 Mon Sep 17 00:00:00 2001 From: chrysle Date: Tue, 24 Oct 2023 21:15:09 +0200 Subject: [PATCH 2/5] Update changelog and remove line ending from test assert --- CHANGELOG.md | 3 ++- tests/test_run.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0d529d50de..b31b16d3ea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## dev -- Drop `setuptools` and `wheel` from the shared libraries +- Drop `setuptools` and `wheel` from the shared libraries. This results in less time consumption when the libraries are +automatically upgraded. - Allow running `pip` with `pipx run` - Make usage message in `pipx run` show `package_or_url`, so extra will be printed out as well - Add `--force-reinstall` to pip arguments when `--force` was passed diff --git a/tests/test_run.py b/tests/test_run.py index 98ad312718..064c785e8c 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -340,4 +340,4 @@ def test_run_with_windows_python_version(caplog, pipx_temp_env, tmp_path): def test_run_shared_lib_as_app(pipx_temp_env, monkeypatch, capfd): run_pipx_cli_exit(["run", "pip", "--help"]) captured = capfd.readouterr() - assert "pip [options]\n" in captured.out + assert "pip [options]" in captured.out From 64b445e02a58baa83a1fe3b876f89a65af69ec59 Mon Sep 17 00:00:00 2001 From: chrysle Date: Sun, 8 Oct 2023 15:27:26 +0200 Subject: [PATCH 3/5] Drop `wheel` and `setuptools` from shared libs And enable running the remaining shared library `pip` using `pipx run`. --- CHANGELOG.md | 2 ++ docs/how-pipx-works.md | 8 ++++---- src/pipx/commands/install.py | 7 ++++++- src/pipx/commands/run.py | 7 ++++++- src/pipx/shared_libs.py | 6 ++---- src/pipx/venv.py | 34 +++++++++++++++++++++------------- tests/test_run.py | 7 +++++++ 7 files changed, 48 insertions(+), 23 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9ca9c78450..fb80dcb285 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,7 @@ ## dev +- Drop `setuptools` and `wheel` from the shared libraries +- Allow running `pip` with `pipx run` - Support PEP 723 run requirements in `pipx run`. - Imply `--include-apps` when running `pipx inject --include-deps` - Add `--with-suffix` for `pipx inject` command diff --git a/docs/how-pipx-works.md b/docs/how-pipx-works.md index 622543d47a..17a75efac5 100644 --- a/docs/how-pipx-works.md +++ b/docs/how-pipx-works.md @@ -3,8 +3,8 @@ When installing a package and its binaries on linux (`pipx install package`) pipx will - create directory `~/.local/share/pipx/venvs/PACKAGE` -- create or re-use a shared virtual environment that contains shared packaging libraries `pip`, `setuptools` and `wheel` in `~/.local/share/pipx/shared/` -- ensure all packaging libraries are updated to their latest versions +- create or re-use a shared virtual environment that contains shared packaging library `pip` in `~/.local/share/pipx/shared/` +- ensure the library is updated to its latest version - create a Virtual Environment in `~/.local/share/pipx/venvs/PACKAGE` that uses the shared pip mentioned above but otherwise is isolated (pipx uses a [.pth file]( https://docs.python.org/3/library/site.html) to do this) - install the desired package in the Virtual Environment - expose binaries at `~/.local/bin` that point to new binaries in `~/.local/share/pipx/venvs/PACKAGE/bin` (such as `~/.local/bin/black` -> `~/.local/share/pipx/venvs/black/bin/black`) @@ -12,8 +12,8 @@ When installing a package and its binaries on linux (`pipx install package`) pip When running a binary (`pipx run BINARY`), pipx will -- create or re-use a shared virtual environment that contains shared packaging libraries `pip`, `setuptools` and `wheel` in `~/.local/share/pipx/shared/` -- ensure all packaging libraries are updated to their latest versions +- create or re-use a shared virtual environment that contains the shared packaging library `pip` +- ensure the library is updated to its latest version - create a temporary directory (or reuse a cached virtual environment for this package) with a name based on a hash of the attributes that make the run reproducible. This includes things like the package name, spec, python version, and pip arguments. - create a Virtual Environment inside it with `python -m venv` - install the desired package in the Virtual Environment diff --git a/src/pipx/commands/install.py b/src/pipx/commands/install.py index 218bf7d39e..d0f1de38c4 100644 --- a/src/pipx/commands/install.py +++ b/src/pipx/commands/install.py @@ -73,7 +73,12 @@ def install( return EXIT_CODE_INSTALL_VENV_EXISTS try: - venv.create_venv(venv_args, pip_args) + # Enable installing shared library `pip` with `pipx` + override_shared = False + + if package_name == "pip": + override_shared = True + venv.create_venv(venv_args, pip_args, override_shared) for dep in preinstall_packages or []: dep_name = package_name_from_spec( dep, python, pip_args=pip_args, verbose=verbose diff --git a/src/pipx/commands/run.py b/src/pipx/commands/run.py index 575565c129..c57fabf83e 100644 --- a/src/pipx/commands/run.py +++ b/src/pipx/commands/run.py @@ -225,7 +225,6 @@ def _download_and_run( verbose: bool, ) -> NoReturn: venv = Venv(venv_dir, python=python, verbose=verbose) - venv.create_venv(venv_args, pip_args) if venv.pipx_metadata.main_package.package is not None: package_name = venv.pipx_metadata.main_package.package @@ -234,6 +233,12 @@ def _download_and_run( package_or_url, python, pip_args=pip_args, verbose=verbose ) + override_shared = False + + if package_name == "pip": + override_shared = True + + venv.create_venv(venv_args, pip_args, override_shared) venv.install_package( package_name=package_name, package_or_url=package_or_url, diff --git a/src/pipx/shared_libs.py b/src/pipx/shared_libs.py index 5a752c9bd9..625ad98f06 100644 --- a/src/pipx/shared_libs.py +++ b/src/pipx/shared_libs.py @@ -26,8 +26,8 @@ def __init__(self) -> None: self.root = constants.PIPX_SHARED_LIBS self.bin_path, self.python_path = get_venv_paths(self.root) self.pip_path = self.bin_path / ("pip" if not WINDOWS else "pip.exe") - # i.e. bin_path is ~/.local/pipx/shared/bin - # i.e. python_path is ~/.local/pipx/shared/python + # i.e. bin_path is ~/.local/share/pipx/shared/bin + # i.e. python_path is ~/.local/share/pipx/shared/python self._site_packages: Optional[Path] = None self.has_been_updated_this_run = False self.has_been_logged_this_run = False @@ -107,8 +107,6 @@ def upgrade( *_pip_args, "--upgrade", "pip", - "setuptools", - "wheel", ] ) subprocess_post_check(upgrade_process) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index 28189cc510..70921e86dc 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -156,24 +156,32 @@ def main_package_name(self) -> str: else: return self.pipx_metadata.main_package.package - def create_venv(self, venv_args: List[str], pip_args: List[str]) -> None: + def create_venv( + self, venv_args: List[str], pip_args: List[str], override_shared: bool = False + ) -> None: + """ + override_shared -- Override installing shared libraries to the pipx shared directory (default False) + """ with animate("creating virtual environment", self.do_animation): - cmd = [self.python, "-m", "venv", "--without-pip"] + cmd = [self.python, "-m", "venv"] + if not override_shared: + cmd.append("--without-pip") venv_process = run_subprocess(cmd + venv_args + [str(self.root)]) subprocess_post_check(venv_process) shared_libs.create(self.verbose) - pipx_pth = get_site_packages(self.python_path) / PIPX_SHARED_PTH - # write path pointing to the shared libs site-packages directory - # example pipx_pth location: - # ~/.local/pipx/venvs/black/lib/python3.8/site-packages/pipx_shared.pth - # example shared_libs.site_packages location: - # ~/.local/pipx/shared/lib/python3.6/site-packages - # - # https://docs.python.org/3/library/site.html - # A path configuration file is a file whose name has the form 'name.pth'. - # its contents are additional items (one per line) to be added to sys.path - pipx_pth.write_text(f"{shared_libs.site_packages}\n", encoding="utf-8") + if not override_shared: + pipx_pth = get_site_packages(self.python_path) / PIPX_SHARED_PTH + # write path pointing to the shared libs site-packages directory + # example pipx_pth location: + # ~/.local/share/pipx/venvs/black/lib/python3.8/site-packages/pipx_shared.pth + # example shared_libs.site_packages location: + # ~/.local/share/pipx/shared/lib/python3.6/site-packages + # + # https://docs.python.org/3/library/site.html + # A path configuration file is a file whose name has the form 'name.pth'. + # its contents are additional items (one per line) to be added to sys.path + pipx_pth.write_text(f"{shared_libs.site_packages}\n", encoding="utf-8") self.pipx_metadata.venv_args = venv_args self.pipx_metadata.python_version = self.get_python_version() diff --git a/tests/test_run.py b/tests/test_run.py index 6a6708d50a..7835eebe81 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -339,3 +339,10 @@ def test_run_with_windows_python_version(caplog, pipx_temp_env, tmp_path): ) run_pipx_cli_exit(["run", script.as_uri(), "--python", "3.11"]) assert "3.11" in out.read_text() + + +@mock.patch("os.execvpe", new=execvpe_mock) +def test_run_shared_lib_as_app(pipx_temp_env, monkeypatch, capfd): + run_pipx_cli_exit(["run", "pip", "--help"]) + captured = capfd.readouterr() + assert "pip [options]\n" in captured.out From fd4205c7f772995ebdcca8345dced196fcd24d80 Mon Sep 17 00:00:00 2001 From: chrysle Date: Tue, 24 Oct 2023 21:15:09 +0200 Subject: [PATCH 4/5] Update changelog and remove line ending from test assert --- CHANGELOG.md | 3 ++- tests/test_run.py | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index fb80dcb285..5371db415a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ ## dev -- Drop `setuptools` and `wheel` from the shared libraries +- Drop `setuptools` and `wheel` from the shared libraries. This results in less time consumption when the libraries are +automatically upgraded. - Allow running `pip` with `pipx run` - Support PEP 723 run requirements in `pipx run`. - Imply `--include-apps` when running `pipx inject --include-deps` diff --git a/tests/test_run.py b/tests/test_run.py index 7835eebe81..60a24c40a8 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -345,4 +345,4 @@ def test_run_with_windows_python_version(caplog, pipx_temp_env, tmp_path): def test_run_shared_lib_as_app(pipx_temp_env, monkeypatch, capfd): run_pipx_cli_exit(["run", "pip", "--help"]) captured = capfd.readouterr() - assert "pip [options]\n" in captured.out + assert "pip [options]" in captured.out From 913e823f10b14e9f6f3be5e03825e819c9336274 Mon Sep 17 00:00:00 2001 From: chrysle Date: Fri, 1 Dec 2023 06:55:14 +0100 Subject: [PATCH 5/5] Simplify test for `pip` and set minimum version Co-authored-by: Sviatoslav Sydorenko --- src/pipx/commands/install.py | 5 +---- src/pipx/commands/run.py | 5 +---- src/pipx/shared_libs.py | 2 +- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/src/pipx/commands/install.py b/src/pipx/commands/install.py index d0f1de38c4..ab76778551 100644 --- a/src/pipx/commands/install.py +++ b/src/pipx/commands/install.py @@ -74,10 +74,7 @@ def install( try: # Enable installing shared library `pip` with `pipx` - override_shared = False - - if package_name == "pip": - override_shared = True + override_shared = package_name == "pip" venv.create_venv(venv_args, pip_args, override_shared) for dep in preinstall_packages or []: dep_name = package_name_from_spec( diff --git a/src/pipx/commands/run.py b/src/pipx/commands/run.py index c57fabf83e..ce84efc8b6 100644 --- a/src/pipx/commands/run.py +++ b/src/pipx/commands/run.py @@ -233,10 +233,7 @@ def _download_and_run( package_or_url, python, pip_args=pip_args, verbose=verbose ) - override_shared = False - - if package_name == "pip": - override_shared = True + override_shared = package_name == "pip" venv.create_venv(venv_args, pip_args, override_shared) venv.install_package( diff --git a/src/pipx/shared_libs.py b/src/pipx/shared_libs.py index 625ad98f06..cfe938dbcb 100644 --- a/src/pipx/shared_libs.py +++ b/src/pipx/shared_libs.py @@ -106,7 +106,7 @@ def upgrade( "install", *_pip_args, "--upgrade", - "pip", + "pip >= 23.1", ] ) subprocess_post_check(upgrade_process)