pw_build: Add pw_py_importable_runfile

Adds a rule that exposes runfiles as importable Python libraries to make them easier to use. Bug: b/388526733 Change-Id: Ic1f36077ba492fc06f5540bad320ac41b7558887 Reviewed-on: https://pigweed-review.googlesource.com/c/pigweed/pigweed/+/267335 Presubmit-Verified: CQ Bot Account <[email protected]> Lint: Lint 🤖 <[email protected]> Pigweed-Auto-Submit: Armando Montanez <[email protected]> Reviewed-by: Ted Pudlik <[email protected]> Commit-Queue: Auto-Submit <[email protected]>
google · Feb 13, 2025 · 335316a · 335316a
1 parent d1c3488
commit 335316a
Show file tree

Hide file tree

Showing 11 changed files with 409 additions and 2 deletions.
diff --git a/docs/style/python.rst b/docs/style/python.rst
@@ -17,6 +17,8 @@ Pigweed officially supports :ref:`a few Python versions
 versions. The only exception is :ref:`module-pw_env_setup`, which must also
 support Python 2 and 3.6.
 
+.. _docs-style-python-extend-generated-import-paths:
+
 -------------------------------------------------------
 Extend the import path of packages with generated files
 -------------------------------------------------------

diff --git a/pw_build/BUILD.bazel b/pw_build/BUILD.bazel
@@ -20,6 +20,7 @@ load("//pw_build:load_phase_test.bzl", "pw_string_comparison_test", "pw_string_l
 load("//pw_build:pw_cc_binary.bzl", "pw_cc_binary_with_map")
 load("//pw_build:pw_cc_blob_library.bzl", "pw_cc_blob_info", "pw_cc_blob_library")
 load("//pw_build:pw_copy_and_patch_file_test.bzl", "pw_copy_and_patch_file_test")
+load("//pw_build:pw_py_importable_runfile.bzl", "pw_py_importable_runfile")
 load("//pw_unit_test:pw_cc_test.bzl", "pw_cc_test")
 
 package(default_visibility = ["//visibility:public"])
@@ -273,6 +274,21 @@ pw_copy_and_patch_file_test(
     patch_file = "test_data/pw_copy_and_patch_file/basic.patch",
 )
 
+pw_py_importable_runfile(
+    name = "test_runfile",
+    testonly = True,
+    target = "test_data/test_runfile.txt",
+    visibility = ["//pw_build:__subpackages__"],
+)
+
+pw_py_importable_runfile(
+    name = "test_runfile_remapped",
+    testonly = True,
+    import_location = "pw_build.another_test_runfile",
+    target = "test_data/test_runfile.txt",
+    visibility = ["//pw_build:__subpackages__"],
+)
+
 filegroup(
     name = "doxygen",
     srcs = [

diff --git a/pw_build/bazel.rst b/pw_build/bazel.rst
@@ -389,6 +389,75 @@ in the bazel dependency `@external-sdk//`.
        patch_file = "changes.patch",
    )
 
+pw_py_importable_runfile
+========================
+An importable ``py_library`` that makes loading runfiles easier.
+
+When using Bazel runfiles from Python,
+`Rlocation() <https://rules-python.readthedocs.io/en/latest/api/py/runfiles/runfiles.runfiles.html#runfiles.runfiles.Runfiles.Rlocation>`__
+takes two arguments:
+
+1. The ``path`` of the runfiles. This is the apparent repo name joined with
+   the path within that repo.
+2. The ``source_repo`` to evaluate ``path`` from. This is related to how
+   apparent repo names and canonical repo names are handled by Bazel.
+
+Unfortunately, it's easy to get these arguments wrong.
+
+This generated Python library short-circuits this problem by letting Bazel
+generate the correct arguments to ``Rlocation()`` so users don't even have
+to think about what to pass.
+
+For example:
+
+.. code-block:: python
+
+   # In @bloaty//:BUILD.bazel, or wherever is convenient:
+   pw_py_importable_runfile(
+       name = "bloaty_runfiles",
+       target = "//:bin/bloaty",
+       import_location = "bloaty.bloaty_binary",
+       visibility = ["//visibility:public"],
+   )
+
+   # Using the pw_py_importable_runfile from a py_binary in a
+   # BUILD.bazel file:
+   py_binary(
+       name = "my_binary",
+       srcs = ["my_binary.py"],
+       main = "my_binary.py",
+       deps = ["@bloaty//:bloaty_runfiles"],
+   )
+
+   # In my_binary.py:
+   import bloaty.bloaty_binary
+   from python.runfiles import runfiles  # type: ignore
+
+   r = runfiles.Create()
+   bloaty_path = r.Rlocation(*bloaty.bloaty_binary.RLOCATION)
+
+.. note::
+
+   Because this exposes runfiles as importable Python modules,
+   the import paths of the generated libraries may collide with existing
+   Python libraries. When this occurs, you need to
+   :ref:`docs-style-python-extend-generated-import-paths`.
+
+Attrs
+-----
+
+.. list-table::
+   :header-rows: 1
+
+   * - Name
+     - Description
+   * - import_location
+     - The final Python import path of the generated module. By default, this is ``path.to.package.label_name``.
+   * - target
+     - The file this library exposes as runfiles.
+   * - \*\*kwargs
+     - Common attributes to forward both underlying targets.
+
 Platform compatibility rules
 ============================
 Macros and rules related to platform compatibility are provided in

diff --git a/pw_build/pw_py_importable_runfile.bzl b/pw_build/pw_py_importable_runfile.bzl
@@ -0,0 +1,168 @@
+# Copyright 2025 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""A rule defining importable Python modules that simplify locating runfiles."""
+
+load("@rules_python//python:defs.bzl", "py_library")
+
+_TEMPLATE = """# FILE GENERATED BY {label}, DO NOT EDIT!
+
+from pw_build.python_runfiles import PythonRunfilesLabelAdapter
+
+RLOCATION = PythonRunfilesLabelAdapter(
+    runfiles_path = "{runfiles_path}",
+    source_repo = {source_repo},
+)
+"""
+
+def _generated_runfile_import_impl(ctx):
+    import_path = "/".join(
+        [ctx.attr.import_dir] + ctx.attr.import_location.split("."),
+    )
+
+    generated_import = ctx.actions.declare_file(import_path + ".py")
+
+    if ctx.file.target:
+        f = ctx.file.target
+    else:
+        target_files = ctx.attr.target[DefaultInfo].files.to_list()
+        if len(target_files) != 1:
+            fail(
+                ctx.attr.target.label,
+                "provides multiple files, which is currently not supported by",
+                "pw_py_importable_runfile. Please ensure",
+                ctx.label,
+                "specifies a `target` with a single file",
+            )
+        f = ctx.attr.target[DefaultInfo].files.to_list()[0]
+
+    target_repo_name = ctx.attr.target.label.repo_name
+    if not target_repo_name:
+        target_repo_name = ctx.attr.module_name
+
+    # It's valid for this to be None. For example, the repo_name of the
+    # root repo defaults to None to support the canonical name of "@@".
+    current_repo_name = ctx.label.repo_name
+    current_repo_name = '"{}"'.format(current_repo_name) if current_repo_name else None
+
+    runfile_path = f.short_path
+    runfile_path = "{}/{}".format(target_repo_name, runfile_path)
+
+    ctx.actions.write(
+        output = generated_import,
+        content = _TEMPLATE.format(
+            label = ctx.label,
+            runfiles_path = runfile_path,
+            source_repo = current_repo_name,
+        ),
+    )
+
+    runfiles = ctx.runfiles([ctx.file.target])
+    runfiles = runfiles.merge(ctx.attr.target[DefaultInfo].data_runfiles)
+
+    return DefaultInfo(
+        files = depset(direct = [generated_import]),
+        runfiles = runfiles,
+    )
+
+_generated_runfile_import = rule(
+    implementation = _generated_runfile_import_impl,
+    attrs = {
+        "import_dir": attr.string(mandatory = True),
+        "import_location": attr.string(mandatory = True),
+        "module_name": attr.string(),
+        "target": attr.label(mandatory = True, allow_single_file = True),
+    },
+)
+
+def pw_py_importable_runfile(*, name, target = None, import_location = None, **kwargs):
+    """An importable py_library that makes loading runfiles easier.
+
+    When using Bazel runfiles from Python, ``Rlocation()`` takes two arguments:
+
+    1. The ``path`` of the runfiles. This is the apparent repo name joined with
+       the path within that repo.
+    2. The ``source_repo`` to evaluate ``path`` from. This is related to how
+       apparent repo names and canonical repo names are handled by Bazel.
+
+    Unfortunately, it's easy to get these arguments wrong.
+
+    This generated Python library short-circuits this problem by letting Bazel
+    generate the correct arguments to ``Rlocation()`` so users don't even have
+    to think about what to pass.
+
+    For example:
+
+        ```
+        # In @bloaty//:BUILD.bazel, or wherever is convenient:
+        pw_py_importable_runfile(
+            name = "bloaty_runfiles",
+            target = "//:bin/bloaty",
+            import_location = "bloaty.bloaty_binary",
+        )
+
+        # Using the pw_py_importable_runfile from a py_binary in a
+        # BUILD.bazel file:
+        py_binary(
+            name = "my_binary",
+            srcs = ["my_binary.py"],
+            main = "my_binary.py",
+            deps = ["@bloaty//:bloaty_runfiles"],
+        )
+
+        # In my_binary.py:
+        import bloaty.bloaty_binary
+        from python.runfiles import runfiles  # type: ignore
+
+        r = runfiles.Create()
+        bloaty_path = r.Rlocation(*bloaty.bloaty_binary.RLOCATION)
+        ```
+
+    Note: Because this exposes runfiles as importable Python modules,
+    the import paths of the generated libraries may collide with existing
+    Python libraries. When this occurs, you need to extend the import path
+    of modules with generated files.
+
+    Args:
+        name: name of the target.
+        import_location: The final Python import path of the generated module.
+            By default, this is ``path.to.package.label_name``.
+        target: The file this library exposes as runfiles.
+        **kwargs: Common attributes to forward both underlying targets.
+    """
+    _generated_py_file = "{}._generated_py_import".format(name)
+    _virtual_import_dir = "_{}_virtual_imports".format((name))
+    if not import_location:
+        import_location = "/".join((
+            native.package_relative_label(":{}".format(name)).package,
+            name,
+        ))
+    _generated_runfile_import(
+        name = _generated_py_file,
+        target = target,
+        import_location = import_location,
+        import_dir = _virtual_import_dir,
+        module_name = native.module_name(),
+        **kwargs
+    )
+    py_library(
+        name = name,
+        imports = [_virtual_import_dir],
+        srcs = [":" + _generated_py_file],
+        data = [target],
+        deps = [
+            Label("//pw_build/py:python_runfiles"),
+            Label("@rules_python//python/runfiles"),
+        ],
+        **kwargs
+    )
diff --git a/pw_build/py/BUILD.bazel b/pw_build/py/BUILD.bazel
@@ -81,6 +81,12 @@ py_library(
     ],
 )
 
+py_library(
+    name = "python_runfiles",
+    srcs = ["pw_build/python_runfiles.py"],
+    imports = ["."],
+)
+
 pw_py_binary(
     name = "generate_cc_blob_library",
     srcs = [
@@ -215,6 +221,18 @@ pw_py_test(
     ],
 )
 
+pw_py_test(
+    name = "py_runfiles_test",
+    size = "small",
+    srcs = [
+        "py_runfiles_test.py",
+    ],
+    deps = [
+        "//pw_build:test_runfile",
+        "//pw_build:test_runfile_remapped",
+    ],
+)
+
 pw_py_test(
     name = "build_recipe_test",
     size = "small",

diff --git a/pw_build/py/BUILD.gn b/pw_build/py/BUILD.gn
@@ -58,6 +58,7 @@ pw_python_package("py") {
     "pw_build/project_builder_prefs.py",
     "pw_build/project_builder_presubmit_runner.py",
     "pw_build/python_package.py",
+    "pw_build/python_runfiles.py",
     "pw_build/python_runner.py",
     "pw_build/wrap_ninja.py",
     "pw_build/zip.py",
@@ -86,3 +87,9 @@ pw_python_package("py") {
   mypy_ini = "$dir_pigweed/.mypy.ini"
   ruff_toml = "$dir_pigweed/.ruff.toml"
 }
+
+# These are intentionally excluded from the GN build since they're
+# Bazel-only.
+group("bazel_tests") {
+  data = [ "py_runfiles_test.py" ]
+}
diff --git a/pw_build/py/pw_build/__init__.py b/pw_build/py/pw_build/__init__.py
@@ -0,0 +1,18 @@
+# Copyright 2025 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Pigweed's core build-system integration libraries."""
+
+from pkgutil import extend_path  # type: ignore
+
+__path__ = extend_path(__path__, __name__)  # type: ignore
diff --git a/pw_build/py/pw_build/python_runfiles.py b/pw_build/py/pw_build/python_runfiles.py
@@ -0,0 +1,34 @@
+# Copyright 2025 The Pigweed Authors
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may not
+# use this file except in compliance with the License. You may obtain a copy of
+# the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations under
+# the License.
+"""Adapters and helpers for Bazel runfiles handling."""
+
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass
+class PythonRunfilesLabelAdapter:
+    """A object that represents Bazel-provided runtime file resource.
+
+    This isn't intended for direct instantiation in a codebase. Bazel will
+    use this as the generated type for pw_py_importable_runfile libraries.
+    """
+
+    runfiles_path: Path
+    source_repo: str
+
+    def __iter__(self):
+        """Custom iterator to support passing an adapter to Rlocation()."""
+        yield self.runfiles_path
+        yield self.source_repo