diff --git a/.bazelignore b/.bazelignore index 5e8a62da496..e4cb6071322 100644 --- a/.bazelignore +++ b/.bazelignore @@ -1 +1,2 @@ # NB: don't ignore the modules/ folder as it contains inputs to bazel test targets. +tools/bzlmod_migration_test_examples \ No newline at end of file diff --git a/.gitignore b/.gitignore index e3dbd9f39c4..6cf783c6175 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,7 @@ tools/__pycache__ tools/node_modules +tools/bzlmod_migration_test_examples/*/bazel-* +tools/bzlmod_migration_test_examples/*/__pycache__ /bazel-* MODULE.bazel.lock temp_test_repos diff --git a/tools/bzlmod_migration_test_examples/maven_extensions/.bazelrc b/tools/bzlmod_migration_test_examples/maven_extensions/.bazelrc new file mode 100644 index 00000000000..3b741d11037 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/maven_extensions/.bazelrc @@ -0,0 +1 @@ +build --java_runtime_version=21 \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/maven_extensions/.bazelversion b/tools/bzlmod_migration_test_examples/maven_extensions/.bazelversion new file mode 100644 index 00000000000..bbd8e9206ee --- /dev/null +++ b/tools/bzlmod_migration_test_examples/maven_extensions/.bazelversion @@ -0,0 +1 @@ +7.6.1 \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/maven_extensions/BUILD b/tools/bzlmod_migration_test_examples/maven_extensions/BUILD new file mode 100644 index 00000000000..af64c4e9f06 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/maven_extensions/BUILD @@ -0,0 +1,11 @@ +load("@rules_java//java:defs.bzl", "java_binary") + +java_binary( + name = "px_deps_bin", + main_class = "org.antlr.v4.Tool", + runtime_deps = [ + "@px_deps//:org_antlr_antlr4", + "@px_deps//:com_google_jimfs_jimfs", + "@px_deps//:com_google_truth_truth", + ], +) \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/maven_extensions/WORKSPACE b/tools/bzlmod_migration_test_examples/maven_extensions/WORKSPACE new file mode 100644 index 00000000000..f75bfa7d332 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/maven_extensions/WORKSPACE @@ -0,0 +1,31 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_java", + sha256 = "c999cdbb4e8414d49c4117bb73800cff95c438c15da075531ae004275ab23144", + urls = ["https://github.com/bazelbuild/rules_java/releases/download/7.12.0/rules_java-7.12.0.tar.gz"], +) + +http_archive( + name = "rules_jvm_external", + sha256 = "23fe83890a77ac1a3ee143e2306ec12da4a845285b14ea13cb0df1b1e23658fe", + strip_prefix = "rules_jvm_external-4.3", + urls = ["https://github.com/bazelbuild/rules_jvm_external/archive/refs/tags/4.3.tar.gz"], +) + +load("@rules_jvm_external//:defs.bzl", "maven_install") + +maven_install( + name = "px_deps", + artifacts = [ + "org.antlr:antlr4:4.11.1", + "com.google.jimfs:jimfs:1.2", + "com.google.truth:truth:1.4.0", + ], + repositories = [ + "https://repo1.maven.org/maven2", + ], +) + +load("@px_deps//:defs.bzl", pinned_maven_install = "pinned_maven_install") +pinned_maven_install() diff --git a/tools/bzlmod_migration_test_examples/maven_extensions/migration_test.py b/tools/bzlmod_migration_test_examples/maven_extensions/migration_test.py new file mode 100644 index 00000000000..d5ad2d6dd19 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/maven_extensions/migration_test.py @@ -0,0 +1,102 @@ +import unittest +import subprocess +import os +from unittest import main + + +class BazelBuildTest(unittest.TestCase): + """ + A test suite for verifying Bzlmod migration tool for maven extensions. + """ + + _CREATED_FILES = [ + "MODULE.bazel", + "MODULE.bazel.lock", + "WORKSPACE.bzlmod", + "migration_info.md", + "query_direct_deps", + "resolved_deps.py", + ] + + def _cleanup_created_files(self): + """ + Remove files which were created by migration tool. + """ + for file_name in self._CREATED_FILES: + file_path = os.path.join(os.getcwd(), file_name) + if os.path.exists(file_path): + os.remove(file_path) + + def _run_command(self, command, expected_failure=False): + """ + Helper function to run a command and return its result. + It captures `stdout`, `stderr` and `returncode` for debugging. + """ + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result + except FileNotFoundError: + self.fail("Command not found.") + except subprocess.CalledProcessError as e: + if expected_failure: + return e + self.fail(f"Command failed with exit code {e.returncode}:\nSTDOUT:\n{e.stdout}\nSTDERR:\n{e.stderr}") + + def _print_message(self, message): + GREEN = "\033[92m" + RESET = "\033[0m" + print(f"{GREEN}{message}{RESET}") + + def modify_build_file(self, old, new): + with open("BUILD", "r") as f: + original_content = f.read() + with open("BUILD", "w") as f: + modified_content = str(original_content).replace(old, new) + f.write(modified_content) + + def test_migration_of_module_deps(self): + self._cleanup_created_files() + + # Verify bazel build is successful with enabled workspace + print("\n--- Running bazel build with enabled workspace ---") + result = self._run_command(["bazel", "build", "--nobuild", "--enable_workspace", "--noenable_bzlmod", "//..."]) + assert result.returncode == 0 + self._print_message("Success.") + + # Run migration script + print("\n--- Running migration script ---") + result = self._run_command(["../../migrate_to_bzlmod.py", "-t=/..."], expected_failure=True) + assert result.returncode == 2 + assert "`px_deps` is a maven extension" in result.stdout + assert "no such package '@@[unknown repo 'px_deps'" in result.stderr + assert os.path.exists( + "migration_info.md" + ), "File 'migration_info.md' should be created during migration, but it doesn't exist." + self._print_message("Expected error: User need to modify `@px_deps` into `@maven`.") + + # Verify Bzlmod have error + print("\n--- Running bazel build with enabled bzlmod ---") + result = self._run_command( + ["bazel", "build", "--noenable_workspace", "--enable_bzlmod", "//..."], expected_failure=True + ) + assert result.returncode == 1 + self._print_message("Expected Bzlmod failure since manual change of BUILD file is needed") + + # Modify BUILD file + self.modify_build_file("px_deps", "maven") + print("\n--- Modifying BUILD file ---") + self._print_message("Success.") + + # Verify MODULE.bazel was created successfully + print("\n--- Running bazel build with enabled bzlmod ---") + result = self._run_command(["bazel", "build", "--noenable_workspace", "--enable_bzlmod", "//..."]) + assert result.returncode == 0 + self._print_message("Success with modified BUILD file.") + + # Restore BUILD file to the initial content + self.modify_build_file("maven", "px_deps") + self._cleanup_created_files() + + +if __name__ == "__main__": + main() diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelrc b/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelrc new file mode 100644 index 00000000000..3b741d11037 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelrc @@ -0,0 +1 @@ +build --java_runtime_version=21 \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelversion b/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelversion new file mode 100644 index 00000000000..bbd8e9206ee --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/.bazelversion @@ -0,0 +1 @@ +7.6.1 \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/BUILD b/tools/bzlmod_migration_test_examples/simple_module_deps/BUILD new file mode 100644 index 00000000000..cfd38c4001e --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/BUILD @@ -0,0 +1,48 @@ +load("@bazel_skylib//:bzl_library.bzl", "bzl_library") +load("@rules_cc//cc:defs.bzl", "cc_library", "cc_binary") +load("@rules_java//java:defs.bzl", "java_library", "java_binary") +load("@rules_shell//shell:sh_library.bzl", "sh_library") +load("@com_google_protobuf//bazel:cc_proto_library.bzl", "cc_proto_library") +load("@com_google_protobuf//bazel:proto_library.bzl", "proto_library") + +cc_library( + name = "cc_lib", + srcs = ["test.cc"], + visibility = ["//visibility:public"], +) + +cc_binary( + name = "cc_bin", + deps = [":cc_lib"], +) + +java_library( + name = "java_lib", + srcs = ["Test.java"], + visibility = ["//visibility:public"], +) + +java_binary( + name = "java_bin", + main_class = "Test", + runtime_deps = [":java_lib"], +) + +bzl_library( + name = "rules-java-docs", + srcs = [ + "@rules_java//java:defs.bzl", + ], +) + +sh_library(name='sh_lib') + +proto_library( + name = "proto_lib", + srcs = ["test.proto"] +) + +cc_proto_library( + name = "cc_proto_lib", + deps = [":proto_lib"] +) diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/Test.java b/tools/bzlmod_migration_test_examples/simple_module_deps/Test.java new file mode 100644 index 00000000000..3f43fe4b836 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/Test.java @@ -0,0 +1,4 @@ +public class Test { + public static void main(String[] args) { + } +} \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/WORKSPACE b/tools/bzlmod_migration_test_examples/simple_module_deps/WORKSPACE new file mode 100644 index 00000000000..a04e0ddad94 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/WORKSPACE @@ -0,0 +1,52 @@ +load("@bazel_tools//tools/build_defs/repo:http.bzl", "http_archive") + +http_archive( + name = "rules_java", + sha256 = "c999cdbb4e8414d49c4117bb73800cff95c438c15da075531ae004275ab23144", + urls = ["https://github.com/bazelbuild/rules_java/releases/download/7.12.0/rules_java-7.12.0.tar.gz"], +) + +http_archive( + name = "rules_shell", + sha256 = "3e114424a5c7e4fd43e0133cc6ecdfe54e45ae8affa14fadd839f29901424043", + strip_prefix = "rules_shell-0.4.0", + url = "https://github.com/bazelbuild/rules_shell/releases/download/v0.4.0/rules_shell-v0.4.0.tar.gz", +) + +http_archive( + name = "rules_cc", + sha256 = "f4aadd8387f381033a9ad0500443a52a0cea5f8ad1ede4369d3c614eb7b2682e", + strip_prefix = "rules_cc-0.0.15", + urls = [ + "https://github.com/bazelbuild/rules_cc/releases/download/0.0.15/rules_cc-0.0.15.tar.gz", + ], +) + +http_archive( + name = "bazel_skylib", + urls = [ + "https://github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + "https://mirror.bazel.build/github.com/bazelbuild/bazel-skylib/releases/download/1.1.1/bazel-skylib-1.1.1.tar.gz", + ], + sha256 = "c6966ec828da198c5d9adbaa94c05e3a1c7f21bd012a0b29ba8ddbccb2c93b0d", +) + +http_archive( + name = "com_google_protobuf", + sha256 = "a7e735f510520b41962d07459f6f5b99dd594c7ed4690bf1191b9924bec094a2", + strip_prefix = "protobuf-27.0", + urls = [ + "https://mirror.bazel.build/github.com/protocolbuffers/protobuf/archive/v27.0.zip", + "https://github.com/protocolbuffers/protobuf/archive/v27.0.zip", + ], +) + +http_archive( + name = "rules_python", + sha256 = "5868e73107a8e85d8f323806e60cad7283f34b32163ea6ff1020cf27abef6036", + strip_prefix = "rules_python-0.25.0", + url = "https://github.com/bazelbuild/rules_python/releases/download/0.25.0/rules_python-0.25.0.tar.gz", +) + +load("@com_google_protobuf//:protobuf_deps.bzl", "protobuf_deps") +protobuf_deps() diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/migration_test.py b/tools/bzlmod_migration_test_examples/simple_module_deps/migration_test.py new file mode 100755 index 00000000000..c4c4e84fdf5 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/migration_test.py @@ -0,0 +1,76 @@ +import unittest +import subprocess +import os +from unittest import main + + +class BazelBuildTest(unittest.TestCase): + """ + A test suite for verifying Bzlmod migration tool for simple module deps. + """ + + _CREATED_FILES = [ + "MODULE.bazel", + "MODULE.bazel.lock", + "WORKSPACE.bzlmod", + "migration_info.md", + "query_direct_deps", + "resolved_deps.py", + ] + + def _cleanup_created_files(self): + """ + Remove files which were created by migration tool. + """ + for file_name in self._CREATED_FILES: + file_path = os.path.join(os.getcwd(), file_name) + if os.path.exists(file_path): + os.remove(file_path) + + def _run_command(self, command): + """ + Helper function to run a command and return its result. + It captures `stdout`, `stderr` and `returncode` for debugging. + """ + try: + result = subprocess.run(command, capture_output=True, text=True, check=True) + return result + except FileNotFoundError: + self.fail("Command not found.") + except subprocess.CalledProcessError as e: + self.fail(f"Command failed with exit code {e.returncode}:\nSTDOUT:\n{e.stdout}\nSTDERR:\n{e.stderr}") + + def _print_success(self): + GREEN = "\033[92m" + RESET = "\033[0m" + print(f"{GREEN}Success{RESET}") + + def test_migration_of_module_deps(self): + self._cleanup_created_files() + + # Verify bazel build is successful with enabled workspace + print("\n--- Running bazel build with enabled workspace ---") + result = self._run_command(["bazel", "build", "--enable_workspace", "--noenable_bzlmod", "//..."]) + assert result.returncode == 0 + self._print_success() + + # Run migration script + print("\n--- Running migration script ---") + result = self._run_command(["../../migrate_to_bzlmod.py", "-t=/..."]) + assert result.returncode == 0 + assert os.path.exists( + "migration_info.md" + ), "File 'migration_info.md' should be created during migration, but it doesn't exist." + self._print_success() + + # Verify MODULE.bazel was created successfully + print("\n--- Running bazel build with enabled bzlmod ---") + result = self._run_command(["bazel", "build", "--noenable_workspace", "--enable_bzlmod", "//..."]) + assert result.returncode == 0 + self._print_success() + + self._cleanup_created_files() + + +if __name__ == "__main__": + main() diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/test.cc b/tools/bzlmod_migration_test_examples/simple_module_deps/test.cc new file mode 100644 index 00000000000..8d79f0f4b89 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/test.cc @@ -0,0 +1,5 @@ +#include + +int main() { + return 0; +} \ No newline at end of file diff --git a/tools/bzlmod_migration_test_examples/simple_module_deps/test.proto b/tools/bzlmod_migration_test_examples/simple_module_deps/test.proto new file mode 100644 index 00000000000..1fcf0648948 --- /dev/null +++ b/tools/bzlmod_migration_test_examples/simple_module_deps/test.proto @@ -0,0 +1,6 @@ +syntax = "proto3"; + +package test; + +message Test { +} \ No newline at end of file diff --git a/tools/migrate_to_bzlmod.py b/tools/migrate_to_bzlmod.py index 6ad53672bf2..675dd550250 100755 --- a/tools/migrate_to_bzlmod.py +++ b/tools/migrate_to_bzlmod.py @@ -29,13 +29,18 @@ from registry import RegistryClient # The registry client points to the bazel central registry repo -REGISTRY_CLIENT = RegistryClient(pathlib.Path(__file__).parent.parent) +REGISTRY_CLIENT = RegistryClient(pathlib.Path(__file__).parent.joinpath("../")) USE_REPO_RULE_IDENTIFIER = "# -- use_repo_rule statements -- #" LOAD_IDENTIFIER = "# -- load statements -- #" REPO_IDENTIFIER = "# -- repo definitions -- #" BAZEL_DEP_IDENTIFIER = "# -- bazel_dep definitions -- #" +# Repos which are already translated to Bzlmod, but they could show up in the error messages as still needing translation. +# Example: Maven extension adds TODOs for the user, even thought the repo has been resolved in MODULE.bazel file. +IGNORED_REPOS = [] +# Keep information if it's the first time adding Maven extension since some parts of maven should be translated only once. +ALREADY_INTRODUCED_MAVEN_EXTENSION = False def abort_migration(): info("Abort migration...") @@ -112,6 +117,26 @@ def scratch_file(file_path, lines=None, mode="w"): return abspath +def append_to_file(filename, content): + """ + Creates a file with the given filename and content. + + Args: + filename (str): The name of the file to create. + content (str): The content to write to the file. + """ + try: + with open(filename, "a") as f: + f.write(content) + except OSError as e: + error(f"Error creating file '{filename}': {e}") + + +def append_migration_info(content): + """Adds content to the "migration_info" file in order to help users with details about the migration.""" + append_to_file("migration_info.md", content + "\n") + + def execute_command(args, cwd=None, env=None, shell=False, executable=None): info("Executing command: " + " ".join(args)) with tempfile.TemporaryFile() as stdout: @@ -135,7 +160,7 @@ def execute_command(args, cwd=None, env=None, shell=False, executable=None): def print_repo_definition(dep): - """Print the repository info to stdout and return the repository definition.""" + """Print the repository info to migration_info and return the repository definition.""" # Parse the repository rule class (rule name, and the label for the bzl file where the rule is defined.) rule_class = dep["original_rule_class"] if rule_class.find("%") != -1: @@ -180,15 +205,25 @@ def print_repo_definition(dep): repo_def.append(f" {key} = {value_str},") repo_def.append(")") - header = "----- Repository information for @%s in the WORKSPACE file -----" % dep["original_attributes"]["name"] - eprint(header) if "definition_information" in dep: eprint(dep["definition_information"]) - eprint("Repository definition:") - for line in repo_def: - eprint(line) - eprint("-" * len(header)) - + append_migration_info(f""" +
+ Click here to see where and how the repo was declared in the WORKSPACE file + +#### Location +```python +{dep["definition_information"]} +``` + +#### Definition +```python +{"\n".join(repo_def)} +``` + **Tip**: URLs usually show which version was used. +
+""") + append_migration_info("___") if file_label and file_label.startswith("@@"): file_label = file_label[1:] @@ -316,99 +351,187 @@ def url_match_source_repo(source_url, module_name): return matched -def address_unavailable_repo_error(repo, resolved_deps, workspace_name): - error(f"@{repo} is not visible in the Bzlmod build.") +def add_maven_extension(repo, maven_artifacts, resolved_deps, workspace_name): + global ALREADY_INTRODUCED_MAVEN_EXTENSION + append_migration_info("It has been introduced as a maven extension:\n") + add_rules_jvm_external = ALREADY_INTRODUCED_MAVEN_EXTENSION + + append_migration_info("```") + for maven_artifact in maven_artifacts: + parsed_data = json.loads(maven_artifact) + group = parsed_data["group"] + artifact = parsed_data["artifact"] + version = parsed_data["version"] + artifact = f"""maven.artifact( + group = "{group}", + artifact = "{artifact}", + version = "{version}" +) +""" + repo_def = [] + if not ALREADY_INTRODUCED_MAVEN_EXTENSION: + # Introduce maven extension only once + ALREADY_INTRODUCED_MAVEN_EXTENSION = True + repo_def.append( + 'maven = use_extension("@rules_jvm_external//:extensions.bzl", "maven")\n\n' + ) + repo_def.append(artifact + "\n") + repo_def.append('use_repo(maven, "maven")\n') + write_at = REPO_IDENTIFIER + else: + repo_def.append(artifact) + write_at = "maven.artifact(" + + write_at_given_place( + "MODULE.bazel", + "".join([""] + repo_def), + write_at, + ) + append_migration_info("".join([""] + repo_def)) + + append_migration_info("```") + IGNORED_REPOS.append(repo) + print( + f"{RED}TODO: {RESET}`" + + repo + + "` is a maven extension - Please modify `@" + + repo + + "//:` with `@maven//:`." + ) + append_migration_info( + "TODO: Please modify `@" + repo + "//:` with `@maven//:`.\n\t\t" + ) + + if not add_rules_jvm_external: + # Introduce rules_jvm_external only once + # Due to readability of `migration_info.md` file, it's necessary adding this repo before/after maven information. + address_unavailable_repo("rules_jvm_external", resolved_deps, workspace_name) + + +def address_unavailable_repo(repo, resolved_deps, workspace_name): + if repo in IGNORED_REPOS: + return False + + append_migration_info("## Migration of `" + repo + "`:") # Check if it's the original main repo name if repo == workspace_name: - error( + error_message = [] + error_message.append( f"Please remove the usages of referring your own repo via `@{repo}//`, " "targets should be referenced directly with `//`. " ) - eprint( + error_message.append( 'If it\'s used in a macro, you can use `Label("//foo/bar")` ' "to make sure it always points to your repo no matter where the macro is used." ) - eprint( + error_message.append( "You can temporarily work around this by adding `repo_name` attribute " "to the `module` directive in your MODULE.bazel file." ) - abort_migration() + error("\n".join(error_message)) + # TODO(kotlaja): Create more visible section for TODO. + append_migration_info("TODO: " + "\n".join(error_message)) # Print the repo definition in the original WORKSPACE file repo_def, file_label, rule_name = [], None, None - urls = [] + urls, maven_artifacts = [], [] for dep in resolved_deps: if dep["original_attributes"]["name"] == repo: repo_def, file_label, rule_name = print_repo_definition(dep) urls = dep["original_attributes"].get("urls", []) + if "artifacts" in dep["original_attributes"]: + maven_artifacts = dep["original_attributes"]["artifacts"] if dep["original_attributes"].get("url", None): urls.append(dep["original_attributes"]["url"]) if dep["original_attributes"].get("remote", None): urls.append(dep["original_attributes"]["remote"]) break + if not repo_def: - error( - f"Repository definition for {repo} isn't found in ./resolved_deps.py file, " - "please add `--force/-f` flag to force update it." + append_migration_info( + "Repository definition for `" + + repo + + "` is not found in ./resolved_deps.py file, please add `--force/-f` flag to force update it." ) - abort_migration() + return False # Check if a module is already available in the registry. - info(f"Finding Bazel module based on repo name ({repo}) and URLs: {urls}") found_module = None + potential_modules = [] for module_name in REGISTRY_CLIENT.get_all_modules(): - # Check if there is matching module name or a well known repo name for a matching module. - if repo == module_name or any(url_match_source_repo(url, module_name) for url in urls): + if repo == module_name: found_module = module_name + append_migration_info("Found perfect name match in BCR: `" + module_name + "`\n") + elif any(url_match_source_repo(url, module_name) for url in urls): + potential_modules.append(module_name) + if potential_modules: + append_migration_info("Found partially name matches in BCR: `" + "`, `".join(potential_modules) + ("`\n")) + if found_module == None and len(potential_modules) > 0: + found_module = potential_modules[0] if found_module: metadata = REGISTRY_CLIENT.get_metadata(found_module) version = metadata["versions"][-1] repo_name = "" if repo == found_module else f', repo_name = "{repo}"' bazel_dep_line = f'bazel_dep(name = "{found_module}", version = "{version}"{repo_name})' - info(f"Found module `{found_module}` in the registry, available versions are " + str(metadata["versions"])) - info(f"This can be introduced via a bazel_dep definition:") - eprint(f" {bazel_dep_line}") if yes_or_no( "Do you wish to add the bazel_dep definition to the MODULE.bazel file?", True, ): - info(f"Introducing @{repo} as a Bazel module.") + append_migration_info("It has been introduced as a Bazel module:\n") + append_migration_info("\t" + bazel_dep_line + "") write_at_given_place("MODULE.bazel", bazel_dep_line, BAZEL_DEP_IDENTIFIER) return True else: - info(f"{repo} isn't found in the registry.") + append_migration_info("It is not found in BCR. \n") + + # Support maven extensions. + if str(file_label).__contains__("rules_jvm_external") and maven_artifacts: + add_maven_extension(repo, maven_artifacts, resolved_deps, workspace_name) + return True # Ask user if the dependency should be introduced via use_repo_rule # Only ask if the repo is defined in @bazel_tools or the root module to avoid potential cycle. if ( file_label and file_label.startswith("//") - or file_label.startswith("@bazel_tools//") + or file_label + and file_label.startswith("@bazel_tools//") and yes_or_no( "Do you wish to introduce the repository with use_repo_rule in MODULE.bazel (requires Bazel 7.3 or later)?", True, ) ): + append_migration_info("\tIt has been introduced with `use_repo_rule`:\n") add_repo_with_use_repo_rule(repo, repo_def, file_label, rule_name) + return True + # Ask user if the dependency should be introduced via module extension # Only ask when file_label exists, which means it's a starlark repository rule. elif file_label and yes_or_no("Do you wish to introduce the repository with a module extension?", True): + append_migration_info("\tIt has been introduced as a module extension:\n") add_repo_to_module_extension(repo, repo_def, file_label, rule_name) + return True + elif rule_name == "local_repository" and repo != "bazel_tools": + append_migration_info("\tIt has been introduced as a module extension since it is local_repository rule:\n") + add_repo_to_module_extension(repo, repo_def, "@bazel_tools//tools/build_defs/repo:local.bzl", rule_name) + return True + # Ask user if this dep should be added to the WORKSPACE.bzlmod for later migration. elif yes_or_no( "Do you wish to add the repo definition to WORKSPACE.bzlmod for later migration?", True, ): repo_def = ["", "# TODO: Migrated to Bzlmod"] + repo_def - info(f"Introducing @{repo} in WORKSPACE.bzlmod file.") + append_migration_info("\tIntroducing dep in WORKSPACE.bzlmod for later migration as:") + append_migration_info("\t\t" + "".join(repo_def)) scratch_file("WORKSPACE.bzlmod", repo_def, mode="a") + return True else: - info("Please manually add this dependency ...") - abort_migration() - return True + append_migration_info("\tPlease manually add this dependency.") + return False def detect_bind_issue(stderr): @@ -529,6 +652,8 @@ def generate_resolved_file(targets, use_bazel_sync): "bazel", "build", "--nobuild", + "--noenable_bzlmod", + "--enable_workspace", "--experimental_repository_resolved_file=resolved_deps.py", ] + targets bazel_sync_comand = [ @@ -570,6 +695,64 @@ def load_resolved_deps(targets, use_bazel_sync, force): return resolved_deps +def parse_file(filename): + direct_deps = set() + previous_line_has_external = False + with open(filename, "r") as file: + for line in file: + # Parse for "@". + matches_at = re.findall(r"@(\w+)//", line) + for match_at in matches_at: + if match_at != "bazel_tools": + direct_deps.add(match_at) + + # Parse for "/external/{repo_name}/". + matches_external = re.findall(r"/external/(\w+)/", line) + if previous_line_has_external == False: + # Only first "/external/" is relevant. + for match in matches_external: + if match != "bazel_tools": + direct_deps.add(match) + previous_line_has_external = True if matches_external else False + + return direct_deps + + +def delete_file_if_exists(filename): + """Deletes a file if it exists.""" + if os.path.exists(filename): + try: + os.remove(filename) + except OSError as e: + print(f"Error deleting file '{filename}': {e}") + + +def query_direct_targets(args): + targets = args.target + direct_deps_file = "query_direct_deps" + delete_file_if_exists(direct_deps_file) + + for target in targets: + bazel_command = ["bazel", "query", "--noenable_bzlmod", "--enable_workspace", "--output=build"] + [target] + exit_code, stdout, stderr = execute_command(bazel_command) + if exit_code != 0 or not stdout: + error( + "Bazel query: `" + + " ".join(bazel_command) + + "` contains error:\n" + + stderr + + "\nDouble check if the target you've specified can be built successfully." + ) + abort_migration() + append_to_file(direct_deps_file, stdout) + + direct_deps = parse_file(direct_deps_file) + append_migration_info("## Direct dependencies:") + append_migration_info("* " + "\n* ".join(map(str, direct_deps))) + + return direct_deps + + def main(argv=None): if argv is None: argv = sys.argv[1:] @@ -626,6 +809,35 @@ def main(argv=None): yes_or_no.enable = args.interactive + delete_file_if_exists("migration_info.md") + append_migration_info("# Migration info") + append_migration_info("Command for local testing:") + append_migration_info("```\nbazel build --enable_bzlmod --noenable_workspace " + " ".join(args.target) + "\n```") + + # First part of the migration - Find direct deps with bazel query and add them in MODULE.bazel file. + print(f"{GREEN}\nFirst part of the migration - Resolve direct deps.") + direct_deps = query_direct_targets(args) + + resolved_repos = [] + unresolved_deps = [] + for direct_dep in direct_deps: + if address_unavailable_repo(direct_dep, resolved_deps, workspace_name): + resolved_repos.append(direct_dep) + else: + unresolved_deps.append(direct_dep) + + if unresolved_deps: + print(f"{RED}\nThese repos need manual support:") + for dep in unresolved_deps: + print(f"\t{RED}" + dep) + else: + print(f"\n{GREEN}All direct dependencies have been resolved.") + + print(f"{RESET}For details about the migration process, check {GREEN}`migration_info.md` {RESET}file.\n") + + # Second part of the migration - Build with bzlmod and fix potential errors. + print(f"\n{GREEN}Second part of the migration - Build with bzlmod and fix potential errors.\n") + while True: # Try to build with Bzlmod enabled targets = args.target @@ -634,6 +846,7 @@ def main(argv=None): "build", "--nobuild", "--enable_bzlmod", + "--noenable_workspace", ] + targets exit_code, _, stderr = execute_command(bazel_command) if exit_code == 0: @@ -653,7 +866,7 @@ def main(argv=None): # 1. Detect build failure caused by unavailable repository repo = detect_unavailable_repo_error(stderr) if repo: - if address_unavailable_repo_error(repo, resolved_deps, workspace_name): + if address_unavailable_repo(repo, resolved_deps, workspace_name): continue else: abort_migration()