#1069 Pipe pip progress to pipx

changelog.d/1069.feature.md

@@ -0,0 +1 @@
+Pipe output from pip to pipx output during installation of packages

src/pipx/animate.py

@@ -1,8 +1,9 @@
 import shutil
 import sys
 from contextlib import contextmanager
+from multiprocessing import Queue
 from threading import Event, Thread
-from typing import Generator, List
+from typing import Generator, List, Optional
 
 from pipx.constants import WINDOWS
 from pipx.emojis import EMOJI_SUPPORT
@@ -30,7 +31,9 @@ def _env_supports_animation() -> bool:
 
 
 @contextmanager
-def animate(message: str, do_animation: bool, *, delay: float = 0) -> Generator[None, None, None]:
+def animate(
+    message: str, do_animation: bool, *, delay: float = 0, stream: Optional[Queue] = None
+) -> Generator[None, None, None]:
     if not do_animation or not _env_supports_animation():
         # No animation, just a single print of message
         sys.stderr.write(f"{message}...\n")
@@ -55,6 +58,7 @@ def animate(message: str, do_animation: bool, *, delay: float = 0) -> Generator[
         "delay": delay,
         "period": period,
         "animate_at_beginning_of_line": animate_at_beginning_of_line,
+        "stream": stream,
     }
 
     t = Thread(target=print_animation, kwargs=thread_kwargs)
@@ -75,19 +79,23 @@ def print_animation(
     delay: float,
     period: float,
     animate_at_beginning_of_line: bool,
+    stream: Queue,
 ) -> None:
     (term_cols, _) = shutil.get_terminal_size(fallback=(9999, 24))
     event.wait(delay)
+    last_received_stream = ""
     while not event.wait(0):
         for s in symbols:
+            if stream is not None and not stream.empty():
+                last_received_stream = f": {stream.get_nowait().strip()}"
             if animate_at_beginning_of_line:
                 max_message_len = term_cols - len(f"{s} ... ")
-                cur_line = f"{s} {message:.{max_message_len}}"
+                cur_line = f"{s} {f'{message}{last_received_stream}':.{max_message_len}}"
                 if len(message) > max_message_len:
                     cur_line += "..."
             else:
                 max_message_len = term_cols - len("... ")
-                cur_line = f"{message:.{max_message_len}}{s}"
+                cur_line = f"{f'{message}{last_received_stream}':.{max_message_len}}{s}"
 
             clear_line()
             sys.stderr.write(cur_line)

src/pipx/util.py

@@ -1,3 +1,4 @@
+import importlib
 import logging
 import os
 import random
@@ -8,6 +9,7 @@
 import sys
 import textwrap
 from dataclasses import dataclass
+from multiprocessing import Queue
 from pathlib import Path
 from typing import (
     Any,
@@ -165,6 +167,7 @@ def run_subprocess(
     log_stdout: bool = True,
     log_stderr: bool = True,
     run_dir: Optional[str] = None,
+    stream: Optional[Queue] = None,
 ) -> "subprocess.CompletedProcess[str]":
     """Run arbitrary command as subprocess, capturing stderr and stout"""
     env = dict(os.environ)
@@ -180,16 +183,47 @@ def run_subprocess(
 
     # TODO: Switch to using `-P` / PYTHONSAFEPATH instead of running in
     # separate directory in Python 3.11
-    completed_process = subprocess.run(
+    #   Modified version of subprocess.run
+    #   This allows us to read stdout and stderr in real time
+    with subprocess.Popen(
         cmd_str_list,
         env=env,
         stdout=subprocess.PIPE if capture_stdout else None,
-        stderr=subprocess.PIPE if capture_stderr else None,
+        stderr=subprocess.PIPE if capture_stdout else None,
         encoding="utf-8",
         text=True,
-        check=False,
         cwd=run_dir,
-    )
+    ) as process:
+        try:
+            stdout, stderr = "", ""
+            while True:
+                out = process.stdout.readline() if process.stdout is not None else None
+                err = process.stderr.readline() if process.stderr is not None else None
+                if out:
+                    stdout += out
+                    if stream is not None:
+                        stream.put_nowait(out)
+                if err:
+                    stderr += err
+                if not out and not err:
+                    break
+        except subprocess.TimeoutExpired as exc:
+            process.kill()
+            if importlib.util.find_spec("msvcrt") is None:
+                process.wait()
+            else:
+                communication_result = process.communicate()
+                exc.stdout, exc.stderr = bytes(communication_result[0], "utf-8"), bytes(communication_result[1], "utf-8")
+            raise
+        except:
+            process.kill()
+            raise
+        retcode = process.poll()
+
+    if type(retcode) is int:
+        completed_process = subprocess.CompletedProcess(process.args, retcode, stdout, stderr)
+    else:
+        completed_process = subprocess.CompletedProcess(process.args, 0, stdout, stderr)
 
     if capture_stdout and log_stdout:
         logger.debug(f"stdout: {completed_process.stdout}".rstrip())

src/pipx/venv.py

@@ -3,6 +3,7 @@
 import re
 import shutil
 import time
+from multiprocessing import Queue
 from pathlib import Path
 from typing import TYPE_CHECKING, Dict, Generator, List, NoReturn, Optional, Set
 
@@ -244,22 +245,27 @@ def install_package(
         # check syntax and clean up spec and pip_args
         (package_or_url, pip_args) = parse_specifier_for_install(package_or_url, pip_args)
 
+        pip_out_stream: Queue = Queue()
+
         logger.info("Installing %s", package_descr := full_package_description(package_name, package_or_url))
-        with animate(f"installing {package_descr}", self.do_animation):
+        with animate(f"installing {package_descr}", self.do_animation, stream=pip_out_stream):
             # do not use -q with `pip install` so subprocess_post_check_pip_errors
             #   has more information to analyze in case of failure.
             cmd = [
                 str(self.python_path),
                 "-m",
                 "pip",
                 "--no-input",
+                "--no-cache-dir",
                 "install",
                 *pip_args,
                 package_or_url,
             ]
             # no logging because any errors will be specially logged by
             #   subprocess_post_check_handle_pip_error()
-            pip_process = run_subprocess(cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root))
+            pip_process = run_subprocess(
+                cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root), stream=pip_out_stream
+            )
         subprocess_post_check_handle_pip_error(pip_process)
         if pip_process.returncode:
             raise PipxError(f"Error installing {full_package_description(package_name, package_or_url)}.")
@@ -287,10 +293,12 @@ def install_package(
     def install_unmanaged_packages(self, requirements: List[str], pip_args: List[str]) -> None:
         """Install packages in the venv, but do not record them in the metadata."""
 
+        pip_out_stream: Queue = Queue()
+
         # Note: We want to install everything at once, as that lets
         # pip resolve conflicts correctly.
         logger.info("Installing %s", package_descr := ", ".join(requirements))
-        with animate(f"installing {package_descr}", self.do_animation):
+        with animate(f"installing {package_descr}", self.do_animation, stream=pip_out_stream):
             # do not use -q with `pip install` so subprocess_post_check_pip_errors
             #   has more information to analyze in case of failure.
             cmd = [
@@ -304,7 +312,9 @@ def install_unmanaged_packages(self, requirements: List[str], pip_args: List[str
             ]
             # no logging because any errors will be specially logged by
             #   subprocess_post_check_handle_pip_error()
-            pip_process = run_subprocess(cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root))
+            pip_process = run_subprocess(
+                cmd, log_stdout=False, log_stderr=False, run_dir=str(self.root), stream=pip_out_stream
+            )
         subprocess_post_check_handle_pip_error(pip_process)
         if pip_process.returncode:
             raise PipxError(f"Error installing {', '.join(requirements)}.")