Support adhoc Pip versions in development. (#3011)

jsirois · web-flow · commit 659e813a2686 · 2025-11-15T11:04:01.000-08:00
Enables #2177.
diff --git a/CHANGES.md b/CHANGES.md
@@ -1,5 +1,13 @@
 # Release Notes
 
+## 2.70.0
+
+This release adds a feature for Pex developers. If you want to experiment with a new version of Pip
+you can now specify `_PEX_PIP_VERSION=adhoc _PEX_PIP_ADHOC_REQUIREMENT=...`. N.B.: This feature is
+for Pex development only.
+
+* Support adhoc Pip versions in development. (#3011)
+
 ## 2.69.2
 
 This release fixes handling of scoped repos. Previously, validation against duplicate scopes was too
diff --git a/pex/cache/dirs.py b/pex/cache/dirs.py
@@ -614,7 +614,10 @@ def iter_all(cls, pex_root=ENV):
         from pex.pip.version import PipVersion
 
         for base_dir in glob.glob(CacheDir.PIP.path("*", pex_root=pex_root)):
-            version = PipVersion.for_value(os.path.basename(base_dir))
+            dir_name = os.path.basename(base_dir)
+            version = (
+                PipVersion.ADHOC if dir_name.startswith("adhoc") else PipVersion.for_value(dir_name)
+            )
             cache_dir = os.path.join(base_dir, "pip_cache")
             for pex_dir in glob.glob(os.path.join(base_dir, "pip.pex", "*", "*")):
                 yield cls(path=pex_dir, version=version, base_dir=base_dir, cache_dir=cache_dir)
@@ -629,7 +632,7 @@ def create(
 
         from pex.third_party import isolated
 
-        base_dir = CacheDir.PIP.path(str(version))
+        base_dir = CacheDir.PIP.path(version.cache_dir_name())
         return cls(
             path=os.path.join(base_dir, "pip.pex", isolated().pex_hash, fingerprint),
             version=version,
diff --git a/pex/enum.py b/pex/enum.py
@@ -72,9 +72,13 @@ def __reduce__(self):
             )
             return _get_or_create, (module, enum_type, type(self).__name__, self.value)
 
-        def __init__(self, value):
-            # type: (str) -> None
-            values = Enum.Value._values_by_type[type(self)]
+        def __init__(
+            self,
+            value,  # type: str
+            enum_type=None,  # type: Optional[Type[Enum.Value]]
+        ):
+            # type: (...) -> None
+            values = Enum.Value._values_by_type[enum_type or type(self)]
             self.value = value
             self.ordinal = len(values)
             values.append(weakref.ref(self))
diff --git a/pex/pip/version.py b/pex/pip/version.py
@@ -4,6 +4,7 @@
 from __future__ import absolute_import
 
 import functools
+import hashlib
 import os
 import sys
 from textwrap import dedent
@@ -42,6 +43,22 @@ def overridden(cls):
                         break
         return cast("Optional[PipVersionValue]", getattr(cls, "_overridden"))
 
+    @staticmethod
+    def _to_requirement(
+        project_name,  # type: str
+        project_version,  # type: Union[str, Version]
+    ):
+        # type: (...) -> Requirement
+
+        if not project_version:
+            return Requirement.parse(project_name)
+
+        return Requirement.parse(
+            "{project_name}=={project_version}".format(
+                project_name=project_name, project_version=project_version
+            )
+        )
+
     def __init__(
         self,
         version,  # type: str
@@ -54,34 +71,34 @@ def __init__(
         hidden=False,  # type: bool
     ):
         # type: (...) -> None
-        super(PipVersionValue, self).__init__(name or version)
-
-        def to_requirement(
-            project_name,  # type: str
-            project_version,  # type: str
-        ):
-            # type: (...) -> Requirement
-            return Requirement.parse(
-                "{project_name}=={project_version}".format(
-                    project_name=project_name, project_version=project_version
-                )
-            )
+        super(PipVersionValue, self).__init__(name or version, enum_type=PipVersionValue)
 
         self.version = Version(version)
-        self.requirement = (
-            Requirement.parse(requirement) if requirement else to_requirement("pip", version)
-        )
+        self._requirement = requirement
         self.setuptools_version = setuptools_version
         self.setuptools_requirement = (
             Requirement.parse(setuptools_requirement)
             if setuptools_requirement
-            else to_requirement("setuptools", setuptools_version)
+            else self._to_requirement("setuptools", setuptools_version)
         )
         self.wheel_version = wheel_version
-        self.wheel_requirement = to_requirement("wheel", wheel_version)
+        self.wheel_requirement = self._to_requirement("wheel", wheel_version)
         self.requires_python = SpecifierSet(requires_python) if requires_python else None
         self.hidden = hidden
 
+    def cache_dir_name(self):
+        # type: () -> str
+        return self.value
+
+    @property
+    def requirement(self):
+        # type: () -> Requirement
+        return (
+            Requirement.parse(self._requirement)
+            if self._requirement
+            else self._to_requirement("pip", self.version)
+        )
+
     @property
     def requirements(self):
         # type: () -> Iterable[Requirement]
@@ -124,9 +141,9 @@ def __get__(self, obj, objtype=None):
 class LatestCompatiblePipVersion(object):
     def __get__(self, obj, objtype=None):
         # type: (...) -> PipVersionValue
-        if not hasattr(self, "_latest"):
-            self._latest = PipVersion.latest_compatible()
-        return self._latest
+        if not hasattr(self, "_latest_compatible"):
+            self._latest_compatible = PipVersion.latest_compatible()
+        return self._latest_compatible
 
 
 class DefaultPipVersion(object):
@@ -171,6 +188,39 @@ def __get__(self, obj, objtype=None):
         return self._default
 
 
+class Adhoc(PipVersionValue):
+    def __init__(self):
+        super(Adhoc, self).__init__(
+            name="adhoc",
+            version="99",
+            setuptools_version="",
+            wheel_version="",
+            requires_python="",
+            hidden=True,
+        )
+        self._adhoc_requirement = None  # type: Optional[Requirement]
+
+    def cache_dir_name(self):
+        # type: () -> str
+        return "adhoc-{fingerprint}".format(
+            fingerprint=hashlib.sha1(str(self.requirement).encode("utf-8")).hexdigest()
+        )
+
+    @property
+    def requirement(self):
+        # type: () -> Requirement
+
+        if self._adhoc_requirement is None:
+            requirement = os.environ.get("_PEX_PIP_ADHOC_REQUIREMENT")
+            if not requirement:
+                raise ValueError(
+                    "You must set a value for the _PEX_PIP_ADHOC_REQUIREMENT environment variable "
+                    "to use an adhoc Pip version."
+                )
+            self._adhoc_requirement = Requirement.parse(requirement)
+        return self._adhoc_requirement
+
+
 class PipVersion(Enum["PipVersionValue"]):
     @classmethod
     def values(cls):
@@ -376,6 +426,8 @@ def latest_compatible(cls, target=None):
         requires_python=">=3.9,<3.16",
     )
 
+    ADHOC = Adhoc()
+
     VENDORED = v20_3_4_patched
     LATEST = LatestPipVersion()
     LATEST_COMPATIBLE = LatestCompatiblePipVersion()
diff --git a/pex/venv/venv_pex.py b/pex/venv/venv_pex.py
@@ -172,6 +172,7 @@ def maybe_log(*message):
             "_PEX_HTTP_SERVER_TIMEOUT",
             "_PEX_PEXPECT_TIMEOUT",
             "_PEX_PIP_VERSION",
+            "_PEX_PIP_ADHOC_REQUIREMENT",
             "_PEX_REQUIRES_PYTHON",
             "_PEX_TEST_DEV_ROOT",
             "_PEX_TEST_POS_ARGS",
diff --git a/pex/version.py b/pex/version.py
@@ -1,4 +1,4 @@
 # Copyright 2015 Pex project contributors.
 # Licensed under the Apache License, Version 2.0 (see LICENSE).
 
-__version__ = "2.69.2"
+__version__ = "2.70.0"
diff --git a/tests/integration/test_pip.py b/tests/integration/test_pip.py
@@ -0,0 +1,80 @@
+# Copyright 2025 Pex project contributors.
+# Licensed under the Apache License, Version 2.0 (see LICENSE).
+
+from __future__ import absolute_import
+
+import re
+import sys
+
+import pytest
+
+from testing import make_env, run_pex_command
+from testing.pytest_utils.tmp import Tempdir
+
+
+@pytest.mark.skipif(
+    sys.version_info[:2] < (3, 9),
+    reason="The adhoc Pip version used in the test requires Python>=3.9.",
+)
+def test_adhoc_nominal(tmpdir):
+    # type: (Tempdir) -> None
+
+    pex_root = tmpdir.join("pex-root")
+    pip_log = tmpdir.join("pip.log")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "--pip-log",
+            pip_log,
+            "cowsay==5",
+            "-c",
+            "cowsay",
+            "--",
+            "Moo!",
+        ],
+        env=make_env(
+            _PEX_PIP_VERSION="adhoc",
+            # N.B.: This is a custom version of Pip that prints "Pex Adhoc Proof!" to STDERR just
+            # before running the rest of Pip.
+            _PEX_PIP_ADHOC_REQUIREMENT="pip @ git+https://github.com/pex-tool/pip@2c03ed1a2d60b57f",
+        ),
+    ).assert_success(
+        expected_output_re=r"^.*{message}.*$".format(message=re.escape("| Moo! |")),
+        re_flags=re.DOTALL | re.MULTILINE,
+    )
+
+    with open(pip_log) as fp:
+        assert (
+            "Pex Adhoc Proof!" in fp.read()
+        ), "Expected proof we were running the right adhoc Pip version."
+
+
+def test_adhoc_missing_req(tmpdir):
+    # type: (Tempdir) -> None
+
+    pex_root = tmpdir.join("pex-root")
+    run_pex_command(
+        args=[
+            "--pex-root",
+            pex_root,
+            "--runtime-pex-root",
+            pex_root,
+            "cowsay==5",
+            "-c",
+            "cowsay",
+            "--",
+            "Moo!",
+        ],
+        env=make_env(_PEX_PIP_VERSION="adhoc"),
+    ).assert_failure(
+        expected_error_re=r"^.*{message}.*$".format(
+            message=re.escape(
+                "You must set a value for the _PEX_PIP_ADHOC_REQUIREMENT environment variable to "
+                "use an adhoc Pip version."
+            )
+        ),
+        re_flags=re.DOTALL | re.MULTILINE,
+    )
