diff --git a/changelog.d/1069.feature.md b/changelog.d/1069.feature.md new file mode 100644 index 000000000..5633390e9 --- /dev/null +++ b/changelog.d/1069.feature.md @@ -0,0 +1 @@ +Pipe output from pip to pipx output during installation of packages diff --git a/src/pipx/animate.py b/src/pipx/animate.py index ed1dcd6f3..38d57d35c 100644 --- a/src/pipx/animate.py +++ b/src/pipx/animate.py @@ -1,8 +1,9 @@ import shutil import sys from contextlib import contextmanager +from multiprocessing import Queue from threading import Event, Thread -from typing import Generator, List +from typing import Generator, List, Optional from pipx.constants import WINDOWS from pipx.emojis import EMOJI_SUPPORT @@ -30,7 +31,9 @@ def _env_supports_animation() -> bool: @contextmanager -def animate(message: str, do_animation: bool, *, delay: float = 0) -> Generator[None, None, None]: +def animate( + message: str, do_animation: bool, *, delay: float = 0, stream: Optional[Queue] = None +) -> Generator[None, None, None]: if not do_animation or not _env_supports_animation(): # No animation, just a single print of message sys.stderr.write(f"{message}...\n") @@ -55,6 +58,7 @@ def animate(message: str, do_animation: bool, *, delay: float = 0) -> Generator[ "delay": delay, "period": period, "animate_at_beginning_of_line": animate_at_beginning_of_line, + "stream": stream, } t = Thread(target=print_animation, kwargs=thread_kwargs) @@ -75,19 +79,23 @@ def print_animation( delay: float, period: float, animate_at_beginning_of_line: bool, + stream: Queue, ) -> None: (term_cols, _) = shutil.get_terminal_size(fallback=(9999, 24)) event.wait(delay) + last_received_stream = "" while not event.wait(0): for s in symbols: + if stream is not None and not stream.empty(): + last_received_stream = f": {stream.get_nowait().strip()}" if animate_at_beginning_of_line: max_message_len = term_cols - len(f"{s} ... ") - cur_line = f"{s} {message:.{max_message_len}}" + cur_line = f"{s} {f'{message}{last_received_stream}':.{max_message_len}}" if len(message) > max_message_len: cur_line += "..." else: max_message_len = term_cols - len("... ") - cur_line = f"{message:.{max_message_len}}{s}" + cur_line = f"{f'{message}{last_received_stream}':.{max_message_len}}{s}" clear_line() sys.stderr.write(cur_line) diff --git a/src/pipx/util.py b/src/pipx/util.py index ff83f6645..016a5f648 100644 --- a/src/pipx/util.py +++ b/src/pipx/util.py @@ -1,3 +1,4 @@ +import importlib import logging import os import random @@ -8,6 +9,7 @@ import sys import textwrap from dataclasses import dataclass +from multiprocessing import Queue from pathlib import Path from typing import ( Any, @@ -165,6 +167,7 @@ def run_subprocess( log_stdout: bool = True, log_stderr: bool = True, run_dir: Optional[str] = None, + stream: Optional[Queue] = None, ) -> "subprocess.CompletedProcess[str]": """Run arbitrary command as subprocess, capturing stderr and stout""" env = dict(os.environ) @@ -180,16 +183,47 @@ def run_subprocess( # TODO: Switch to using `-P` / PYTHONSAFEPATH instead of running in # separate directory in Python 3.11 - completed_process = subprocess.run( + # Modified version of subprocess.run + # This allows us to read stdout and stderr in real time + with subprocess.Popen( cmd_str_list, env=env, stdout=subprocess.PIPE if capture_stdout else None, - stderr=subprocess.PIPE if capture_stderr else None, + stderr=subprocess.PIPE if capture_stdout else None, encoding="utf-8", text=True, - check=False, cwd=run_dir, - ) + ) as process: + try: + stdout, stderr = "", "" + while True: + out = process.stdout.readline() if process.stdout is not None else None + err = process.stderr.readline() if process.stderr is not None else None + if out: + stdout += out + if stream is not None: + stream.put_nowait(out) + if err: + stderr += err + if not out and not err: + break + except subprocess.TimeoutExpired as exc: + process.kill() + if importlib.util.find_spec("msvcrt") is None: + process.wait() + else: + communication_result = process.communicate() + exc.stdout, exc.stderr = bytes(communication_result[0], "utf-8"), bytes(communication_result[1], "utf-8") + raise + except: + process.kill() + raise + retcode = process.poll() + + if type(retcode) is int: + completed_process = subprocess.CompletedProcess(process.args, retcode, stdout, stderr) + else: + completed_process = subprocess.CompletedProcess(process.args, 0, stdout, stderr) if capture_stdout and log_stdout: logger.debug(f"stdout: {completed_process.stdout}".rstrip()) diff --git a/src/pipx/venv.py b/src/pipx/venv.py index be761d529..e20de9ab2 100644 --- a/src/pipx/venv.py +++ b/src/pipx/venv.py @@ -3,6 +3,7 @@ import re import shutil import time +from multiprocessing import Queue from pathlib import Path from typing import TYPE_CHECKING, Dict, Generator, List, NoReturn, Optional, Set @@ -244,8 +245,10 @@ def install_package( # check syntax and clean up spec and pip_args (package_or_url, pip_args) = parse_specifier_for_install(package_or_url, pip_args) + pip_out_stream: Queue = Queue() + logger.info("Installing %s", package_descr := full_package_description(package_name, package_or_url)) - with animate(f"installing {package_descr}", self.do_animation): + with animate(f"installing {package_descr}", self.do_animation, stream=pip_out_stream): # do not use -q with `pip install` so subprocess_post_check_pip_errors # has more information to analyze in case of failure. cmd = [ @@ -253,13 +256,16 @@ def install_package( "-m", "pip", "--no-input", + "--no-cache-dir", "install", *pip_args, package_or_url, ] # no logging because any errors will be specially logged by # subprocess_post_check_handle_pip_error() - pip_process = run_subprocess(cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root)) + pip_process = run_subprocess( + cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root), stream=pip_out_stream + ) subprocess_post_check_handle_pip_error(pip_process) if pip_process.returncode: raise PipxError(f"Error installing {full_package_description(package_name, package_or_url)}.") @@ -287,10 +293,12 @@ def install_package( def install_unmanaged_packages(self, requirements: List[str], pip_args: List[str]) -> None: """Install packages in the venv, but do not record them in the metadata.""" + pip_out_stream: Queue = Queue() + # Note: We want to install everything at once, as that lets # pip resolve conflicts correctly. logger.info("Installing %s", package_descr := ", ".join(requirements)) - with animate(f"installing {package_descr}", self.do_animation): + with animate(f"installing {package_descr}", self.do_animation, stream=pip_out_stream): # do not use -q with `pip install` so subprocess_post_check_pip_errors # has more information to analyze in case of failure. cmd = [ @@ -304,7 +312,9 @@ def install_unmanaged_packages(self, requirements: List[str], pip_args: List[str ] # no logging because any errors will be specially logged by # subprocess_post_check_handle_pip_error() - pip_process = run_subprocess(cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root)) + pip_process = run_subprocess( + cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root), stream=pip_out_stream + ) subprocess_post_check_handle_pip_error(pip_process) if pip_process.returncode: raise PipxError(f"Error installing {', '.join(requirements)}.")