feat: uv lock rule instead of genrule (#2657)

This change re-implements the `uv pip compile` as a set of rules instead
of
using a `genrule`. This makes the setup more RBE friendly and it also
fixes
some of existing issues in the exec tools toolchain.

The `lock` macro in the `//python/uv:lock.bzl` now creates three public
targets: `<name>`, `<name>.update` and `<name>.run`. The first will
provide you
with the locked `requirements.txt` file that is used in the
`<name>.update`
executable target when updating the in-source copy of the file. The
`<name>.run` provides an executable target that hardcodes all of the
`uv` args
from the `<name>` rule in a shell script and allows user to debug the
execution
and add extra arguments at the command line.

The `test` target is no longer included, but users can define it
themselves
with the help of `native_test`.

Things that I could not test and would benefit from the community help:
* Windows support - the repository has a rudimentary script, but I am
almost
  sure that it is likely not working, so PRs there are welcome.
* The integration tests are not running on RBE because of the current
RBE
  cluster setup. If you see issues in your RBE setup, PRs are welcome.
* `keyring` integration to pull packages from private index servers is
untested
  as of now, but I see no reason why it should not work.

Work towards #1325
Work towards #1975
Related #2663

Author: aignas

CHANGELOG.md

@@ -53,6 +53,10 @@ Unreleased changes template.
 
 {#v0-0-0-changed}
 ### Changed
+* (toolchain) The `exec` configuration toolchain now has the forwarded
+  `exec_interpreter` now also forwards the `ToolchainInfo` provider. This is
+  for increased compatibility with the `RBE` setups where access to the `exec`
+  configuration interpreter is needed.
 * (toolchains) Use the latest astrahl-sh toolchain release [20250317] for Python versions:
     * 3.9.21
     * 3.10.16
@@ -75,6 +79,14 @@ Unreleased changes template.
 
 {#v0-0-0-added}
 ### Added
+* (uv) A {obj}`lock` rule that is the replacement for the
+  {obj}`compile_pip_requirements`. This may still have rough corners
+  so please report issues with it in the
+  [#1975](https://github.com/bazel-contrib/rules_python/issues/1975).
+  Main highlights - the locking can be done within a build action or outside
+  it, there is no more automatic `test` target (but it can be added on the user
+  side by using `native_test`). For customizing the `uv` version that is used,
+  please check the {obj}`uv.configure` tag class.
 * Add support for riscv64 linux platform.
 * (toolchains) Add python 3.13.2 and 3.12.9 toolchains
 

docs/BUILD.bazel

@@ -176,8 +176,12 @@ lock(
     name = "requirements",
     srcs = ["pyproject.toml"],
     out = "requirements.txt",
-    upgrade = True,
-    visibility = ["//private:__pkg__"],
+    args = [
+        "--emit-index-url",
+        "--universal",
+        "--upgrade",
+    ],
+    visibility = ["//:__subpackages__"],
 )
 
 # Temporary compatibility aliases for some other projects depending on the old

examples/BUILD.bazel

@@ -21,5 +21,10 @@ lock(
     name = "bzlmod_requirements_3_9",
     srcs = ["bzlmod/requirements.in"],
     out = "bzlmod/requirements_lock_3_9.txt",
+    args = [
+        "--emit-index-url",
+        "--universal",
+        "--python-version=3.9",
+    ],
     python_version = "3.9.19",
 )

examples/bzlmod/requirements_lock_3_9.txt

@@ -46,7 +46,7 @@ imagesize==1.4.1 \
     --hash=sha256:0d8d18d08f840c19d0ee7ca1fd82490fdc3729b7ac93f49870406ddde8ef8d8b \
     --hash=sha256:69150444affb9cb0d5cc5a92b3676f0b2fb7cd9ae39e947a5e11a36b4497cd4a
     # via sphinx
-importlib-metadata==8.4.0 ; python_version < '3.10' \
+importlib-metadata==8.4.0 ; python_full_version < '3.10' \
     --hash=sha256:66f342cc6ac9818fc6ff340576acd24d65ba0b3efabb2b4ac08b598965a4a2f1 \
     --hash=sha256:9a547d3bc3608b025f93d403fdd1aae741c24fbb8314df4b155675742ce303c5
     # via sphinx
@@ -316,15 +316,15 @@ tabulate==0.9.0 \
     --hash=sha256:0095b12bf5966de529c0feb1fa08671671b3368eec77d7ef7ab114be2c068b3c \
     --hash=sha256:024ca478df22e9340661486f85298cff5f6dcdba14f3813e8830015b9ed1948f
     # via -r examples/bzlmod/requirements.in
-tomli==2.0.1 ; python_version < '3.11' \
+tomli==2.0.1 ; python_full_version < '3.11' \
     --hash=sha256:939de3e7a6161af0c887ef91b7d41a53e7c5a1ca976325f429cb46ea9bc30ecc \
     --hash=sha256:de526c12914f0c550d15924c62d72abc48d6fe7364aa87328337a31007fe8a4f
     # via pylint
 tomlkit==0.11.6 \
     --hash=sha256:07de26b0d8cfc18f871aec595fda24d95b08fef89d147caa861939f37230bf4b \
     --hash=sha256:71b952e5721688937fb02cf9d354dbcf0785066149d2855e44531ebdd2b65d73
     # via pylint
-typing-extensions==4.12.2 ; python_version < '3.10' \
+typing-extensions==4.12.2 ; python_full_version < '3.10' \
     --hash=sha256:04e5ca0351e0f3f85c6853954072df659d0d13fac324d0072316b67d7794700d \
     --hash=sha256:1a7ead55c7e559dd4dee8856e3a88b41225abfe1ce8df57b7c13915fe121ffb8
     # via
@@ -480,7 +480,7 @@ yamllint==1.28.0 \
     --hash=sha256:89bb5b5ac33b1ade059743cf227de73daa34d5e5a474b06a5e17fc16583b0cf2 \
     --hash=sha256:9e3d8ddd16d0583214c5fdffe806c9344086721f107435f68bad990e5a88826b
     # via -r examples/bzlmod/requirements.in
-zipp==3.20.0 ; python_version < '3.10' \
+zipp==3.20.0 ; python_full_version < '3.10' \
     --hash=sha256:0145e43d89664cfe1a2e533adc75adafed82fe2da404b4bbb6b026c0157bdb31 \
     --hash=sha256:58da6168be89f0be59beb194da1250516fdaa062ccebd30127ac65d30045e10d
     # via importlib-metadata

private/BUILD.bazel

@@ -15,6 +15,7 @@ multirun(
     ] + [
         "//docs:requirements.update",
     ],
+    tags = ["manual"],
 )
 
 # NOTE: The requirements for the pip dependencies may sometimes break the build
@@ -24,4 +25,5 @@ multirun(
 alias(
     name = "whl_library_requirements.update",
     actual = "//tools/private/update_deps:update_pip_deps",
+    tags = ["manual"],
 )

python/private/BUILD.bazel

@@ -361,6 +361,7 @@ bzl_library(
     name = "py_exec_tools_toolchain_bzl",
     srcs = ["py_exec_tools_toolchain.bzl"],
     deps = [
+        ":common_bzl",
         ":py_exec_tools_info_bzl",
         ":sentinel_bzl",
         ":toolchain_types_bzl",

python/private/py_exec_tools_info.bzl

@@ -24,15 +24,26 @@ When running it in an action, use `DefaultInfo.files_to_run` to ensure all its
 files are appropriately available. An exec interpreter may not be available,
 e.g. if all the exec tools are prebuilt binaries.
 
-NOTE: this interpreter is really only for use when a build tool cannot use
+:::{note}
+this interpreter is really only for use when a build tool cannot use
 the Python toolchain itself. When possible, prefeer to define a `py_binary`
 instead and use it via a `cfg=exec` attribute; this makes it much easier
 to setup the runtime environment for the binary. See also:
 `py_interpreter_program` rule.
+:::
 
-NOTE: What interpreter is used depends on the toolchain constraints. Ensure
-the proper target constraints are being applied when obtaining this from
-the toolchain.
+:::{note}
+What interpreter is used depends on the toolchain constraints. Ensure the
+proper target constraints are being applied when obtaining this from the
+toolchain.
+:::
+
+:::{warning}
+This does not work correctly in case of RBE, please use exec_runtime instead.
+
+Once https://github.com/bazelbuild/bazel/issues/23620 is resolved this warning
+may be removed.
+:::
 """,
         "precompiler": """
 :type: Target | None

python/private/py_exec_tools_toolchain.bzl

@@ -29,13 +29,15 @@ def _py_exec_tools_toolchain_impl(ctx):
     if SentinelInfo in ctx.attr.exec_interpreter:
         exec_interpreter = None
 
-    return [platform_common.ToolchainInfo(
-        exec_tools = PyExecToolsInfo(
-            exec_interpreter = exec_interpreter,
-            precompiler = ctx.attr.precompiler,
+    return [
+        platform_common.ToolchainInfo(
+            exec_tools = PyExecToolsInfo(
+                exec_interpreter = exec_interpreter,
+                precompiler = ctx.attr.precompiler,
+            ),
+            **extra_kwargs
         ),
-        **extra_kwargs
-    )]
+    ]
 
 py_exec_tools_toolchain = rule(
     implementation = _py_exec_tools_toolchain_impl,
@@ -51,6 +53,11 @@ This provides `ToolchainInfo` with the following attributes:
     attrs = {
         "exec_interpreter": attr.label(
             default = "//python/private:current_interpreter_executable",
+            providers = [
+                DefaultInfo,
+                # Add the toolchain provider so that we can forward provider fields.
+                platform_common.ToolchainInfo,
+            ],
             cfg = "exec",
             doc = """
 An interpreter that is directly usable in the exec configuration
@@ -69,6 +76,11 @@ handle all the necessary transitions and runtime setup to invoke a program.
 :::
 
 See {obj}`PyExecToolsInfo.exec_interpreter` for further docs.
+
+:::{versionchanged} VERSION_NEXT_FEATURE
+From now on the provided target also needs to provide `platform_common.ToolchainInfo`
+so that the toolchain `py_runtime` field can be correctly forwarded.
+:::
 """,
         ),
         "precompiler": attr.label(

python/private/sentinel.bzl

@@ -25,6 +25,10 @@ SentinelInfo = provider(
 
 def _sentinel_impl(ctx):
     _ = ctx  # @unused
-    return [SentinelInfo()]
+    return [
+        SentinelInfo(),
+        # Also output ToolchainInfo to allow it to be used for noop toolchains
+        platform_common.ToolchainInfo(),
+    ]
 
 sentinel = rule(implementation = _sentinel_impl)

python/uv/lock.bzl

@@ -14,7 +14,33 @@
 
 """The `uv` locking rule.
 
-EXPERIMENTAL: This is experimental and may be removed without notice
+Differences with the legacy {obj}`compile_pip_requirements` rule:
+- This is implemented as a rule that performs locking in a build action.
+- Additionally one can use the runnable target.
+- Uses `uv`.
+- This does not error out if the output file does not exist yet.
+- Supports transitions out of the box.
+
+Note, this does not provide a `test` target, if you would like to add a test
+target that always does the locking automatically to ensure that the
+`requirements.txt` file is up-to-date, add something similar to:
+
+```starlark
+load("@bazel_skylib//rules:native_binary.bzl", "native_test")
+load("@rules_python//python/uv:lock.bzl", "lock")
+
+lock(
+    name = "requirements",
+    srcs = ["pyproject.toml"],
+)
+
+native_test(
+    name = "requirements_test",
+    src = "requirements.update",
+)
+```
+
+EXPERIMENTAL: This is experimental and may be changed without notice.
 """
 
 load("//python/uv/private:lock.bzl", _lock = "lock")

python/uv/private/BUILD.bazel

@@ -13,6 +13,15 @@
 # limitations under the License.
 
 load("@bazel_skylib//:bzl_library.bzl", "bzl_library")
+load("//python/private:bzlmod_enabled.bzl", "BZLMOD_ENABLED")  # buildifier: disable=bzl-visibility
+
+exports_files(
+    srcs = [
+        "lock_copier.py",
+    ],
+    # only because this is used from a macro to template
+    visibility = ["//visibility:public"],
+)
 
 filegroup(
     name = "distribution",
@@ -31,9 +40,13 @@ bzl_library(
     srcs = ["lock.bzl"],
     visibility = ["//python/uv:__subpackages__"],
     deps = [
+        ":toolchain_types_bzl",
         "//python:py_binary_bzl",
         "//python/private:bzlmod_enabled_bzl",
-        "@bazel_skylib//rules:write_file",
+        "//python/private:full_version_bzl",
+        "//python/private:toolchain_types_bzl",
+        "@bazel_skylib//lib:shell",
+        "@pythons_hub//:versions_bzl",
     ],
 )
 
@@ -81,3 +94,13 @@ bzl_library(
         "//python/private:text_util_bzl",
     ],
 )
+
+filegroup(
+    name = "lock_template",
+    srcs = select({
+        "@platforms//os:windows": ["lock.bat"],
+        "//conditions:default": ["lock.sh"],
+    }),
+    target_compatible_with = [] if BZLMOD_ENABLED else ["@platforms//:incompatible"],
+    visibility = ["//visibility:public"],
+)

python/uv/private/lock.bat

@@ -0,0 +1,7 @@
+if defined BUILD_WORKSPACE_DIRECTORY (
+    set "out=%BUILD_WORKSPACE_DIRECTORY%\{{src_out}}"
+) else (
+    exit /b 1
+)
+
+"{{args}}" --output-file "%out%" %*