Merge branch 'main' into cython-bindings-oss-build

Author: gmarzot

.buckconfig.d/external_cells/facebook/buck2-shims-meta/external_cell.buckconfig

@@ -6,4 +6,4 @@ gh_facebook_buck2_shims_meta = git
 
 [external_cell_gh_facebook_buck2_shims_meta]
 git_origin = https://github.com/facebook/buck2-shims-meta.git
-commit_hash = 5476ee7b2fc6e89f1dc625695c3457c42368e4d3
+commit_hash = df09226e8022572c3840179f33267099050db94f

.github/workflows/getdeps_linux.yml

@@ -468,6 +468,12 @@ option(PYTHON_EXTENSIONS
   "Build Python Bindings for Folly, requires Cython and (BUILD_SHARED_LIBS=ON)"
   OFF
 )
+set(PYTHON_PACKAGE_INSTALL_DIR "" CACHE STRING
+  "Installation directory for folly Python packages. \
+  If empty, CMAKE_INSTALL_PREFIX will be used. \
+  No effect if PYTHON_EXTENSIONS=OFF."
+)
+
 
 add_library(folly
   $<TARGET_OBJECTS:folly_base>

.github/workflows/getdeps_mac.yml

@@ -4,62 +4,62 @@
   "name": "buck2",
   "platforms": {
     "macos-aarch64": {
-      "size": 26678080,
+      "size": 26226254,
       "hash": "blake3",
-      "digest": "77a33acb816af93dc9a38d0bf26a1c0d044ede1d1e9bf80e68bd2c692e9b15b3",
+      "digest": "aa9470d7eb85a096eaf61f832c277d108b71d62c2ec70f7bf9a9d644955d8646",
       "format": "zst",
       "path": "buck2-aarch64-apple-darwin",
       "providers": [
         {
-          "url": "https://github.com/facebook/buck2/releases/download/2025-01-02/buck2-aarch64-apple-darwin.zst"
+          "url": "https://github.com/facebook/buck2/releases/download/2025-01-15/buck2-aarch64-apple-darwin.zst"
         }
       ]
     },
     "linux-aarch64": {
-      "size": 28584872,
+      "size": 28089853,
       "hash": "blake3",
-      "digest": "718ab212768932fa261284629b0c679639d06e7bff2d659f33eb1922121c4a9c",
+      "digest": "fb73b58584c052c0a867f6f6124f1ff5be5a6c20678f5fab613621fff96bb685",
       "format": "zst",
       "path": "buck2-aarch64-unknown-linux-musl",
       "providers": [
         {
-          "url": "https://github.com/facebook/buck2/releases/download/2025-01-02/buck2-aarch64-unknown-linux-musl.zst"
+          "url": "https://github.com/facebook/buck2/releases/download/2025-01-15/buck2-aarch64-unknown-linux-musl.zst"
         }
       ]
     },
     "macos-x86_64": {
-      "size": 28411693,
+      "size": 28041686,
       "hash": "blake3",
-      "digest": "c70e6b0a7d35ada00dbb0a6af45c9032322c8d6bb5da9ade57813db8fec9cfd1",
+      "digest": "d5be18e818d080b110d1d3fa0f8e5f5c25d05b4588ea442822166624800165d0",
       "format": "zst",
       "path": "buck2-x86_64-apple-darwin",
       "providers": [
         {
-          "url": "https://github.com/facebook/buck2/releases/download/2025-01-02/buck2-x86_64-apple-darwin.zst"
+          "url": "https://github.com/facebook/buck2/releases/download/2025-01-15/buck2-x86_64-apple-darwin.zst"
         }
       ]
     },
     "windows-x86_64": {
-      "size": 23990882,
+      "size": 23726878,
       "hash": "blake3",
-      "digest": "7dbef62c82d63a80c3a331b2455afce365c2a34c81a17533937f80989bc1f1dd",
+      "digest": "8ef066f8bf5d6ea8aef96f043d3bff06fc5fa361a296ace7090f9460a11d7ae4",
       "format": "zst",
       "path": "buck2-x86_64-pc-windows-msvc.exe",
       "providers": [
         {
-          "url": "https://github.com/facebook/buck2/releases/download/2025-01-02/buck2-x86_64-pc-windows-msvc.exe.zst"
+          "url": "https://github.com/facebook/buck2/releases/download/2025-01-15/buck2-x86_64-pc-windows-msvc.exe.zst"
         }
       ]
     },
     "linux-x86_64": {
-      "size": 30052738,
+      "size": 29531222,
       "hash": "blake3",
-      "digest": "6a4e4f1e4c27fafa799fd64b258b00d405344e3e9e8fab0ab0cc88477bcf9947",
+      "digest": "4d0c942dfd09d6a58aca43342f643f91eeaa8adb58b8d5a02a97ae5911518f1b",
       "format": "zst",
       "path": "buck2-x86_64-unknown-linux-musl",
       "providers": [
         {
-          "url": "https://github.com/facebook/buck2/releases/download/2025-01-02/buck2-x86_64-unknown-linux-musl.zst"
+          "url": "https://github.com/facebook/buck2/releases/download/2025-01-15/buck2-x86_64-unknown-linux-musl.zst"
         }
       ]
     }

.github/workflows/getdeps_windows.yml

@@ -245,16 +245,16 @@ def __init__(self, cache, loader, m):
         self.loader = loader
         self.cache = cache
 
-        self.cache_file_name = "-".join(
+        self.cache_key = "-".join(
             (
                 m.name,
                 self.ctx.get("os"),
                 self.ctx.get("distro") or "none",
                 self.ctx.get("distro_vers") or "none",
                 self.project_hash,
-                "buildcache.tgz",
             )
         )
+        self.cache_file_name = self.cache_key + "-buildcache.tgz"
 
     def is_cacheable(self):
         """We only cache third party projects"""
@@ -440,7 +440,13 @@ def run_project_cmd(self, args, loader, manifest):
         elif manager == "deb":
             packages = sorted(set(all_packages["deb"]))
             if packages:
-                cmd_args = ["sudo", "apt", "install", "-y"] + packages
+                cmd_args = [
+                    "sudo",
+                    "--preserve-env=http_proxy",
+                    "apt-get",
+                    "install",
+                    "-y",
+                ] + packages
         elif manager == "homebrew":
             packages = sorted(set(all_packages["homebrew"]))
             if packages:
@@ -552,6 +558,38 @@ def setup_project_cmd_parser(self, parser):
         )
 
 
+@cmd("query-paths", "print the paths for tooling to use")
+class QueryPathsCmd(ProjectCmdBase):
+    def run_project_cmd(self, args, loader, manifest):
+        if args.recursive:
+            manifests = loader.manifests_in_dependency_order()
+        else:
+            manifests = [manifest]
+
+        cache = cache_module.create_cache()
+        for m in manifests:
+            fetcher = loader.create_fetcher(m)
+            if isinstance(fetcher, SystemPackageFetcher):
+                # We are guaranteed that if the fetcher is set to
+                # SystemPackageFetcher then this item is completely
+                # satisfied by the appropriate system packages
+                continue
+            src_dir = fetcher.get_src_dir()
+            print(f"{m.name}_SOURCE={src_dir}")
+            inst_dir = loader.get_project_install_dir_respecting_install_prefix(m)
+            print(f"{m.name}_INSTALL={inst_dir}")
+            cached_project = CachedProject(cache, loader, m)
+            print(f"{m.name}_CACHE_KEY={cached_project.cache_key}")
+
+    def setup_project_cmd_parser(self, parser):
+        parser.add_argument(
+            "--recursive",
+            help="print the transitive deps also",
+            action="store_true",
+            default=False,
+        )
+
+
 @cmd("show-source-dir", "print the source dir for a given project")
 class ShowSourceDirCmd(ProjectCmdBase):
     def run_project_cmd(self, args, loader, manifest):
@@ -1001,6 +1039,10 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             manifest_ctx.set("test", "on")
         run_on = self.get_run_on(args)
 
+        tests_arg = "--no-tests "
+        if run_tests:
+            tests_arg = ""
+
         # Some projects don't do anything "useful" as a leaf project, only
         # as a dep for a leaf project. Check for those here; we don't want
         # to waste the effort scheduling them on CI.
@@ -1086,12 +1128,14 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 )
                 out.write("      shell: cmd\n")
 
-                # The git installation may not like long filenames, so tell it
-                # that we want it to use them!
                 out.write("    - name: Fix Git config\n")
-                out.write("      run: git config --system core.longpaths true\n")
-                out.write("    - name: Disable autocrlf\n")
-                out.write("      run: git config --system core.autocrlf false\n")
+                out.write("      run: >\n")
+                out.write("        git config --system core.longpaths true &&\n")
+                out.write("        git config --system core.autocrlf false &&\n")
+                # cxx crate needs symlinks enabled
+                out.write("        git config --system core.symlinks true\n")
+                # && is not supported on default windows powershell, so use cmd
+                out.write("      shell: cmd\n")
 
             out.write("    - uses: actions/checkout@v4\n")
 
@@ -1117,7 +1161,7 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 build_opts.allow_system_packages
                 and build_opts.host_type.get_package_manager()
             ):
-                sudo_arg = "sudo "
+                sudo_arg = "sudo --preserve-env=http_proxy "
                 allow_sys_arg = " --allow-system-packages"
                 if build_opts.host_type.get_package_manager() == "deb":
                     out.write("    - name: Update system package info\n")
@@ -1127,17 +1171,12 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 if build_opts.is_darwin():
                     # brew is installed as regular user
                     sudo_arg = ""
-                tests_arg = "--no-tests "
-                if run_tests:
-                    tests_arg = ""
-                out.write(
-                    f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive {manifest.name}\n"
-                )
+
+                system_deps_cmd = f"{sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive {manifest.name}"
                 if build_opts.is_linux() or build_opts.is_freebsd():
-                    out.write("    - name: Install packaging system deps\n")
-                    out.write(
-                        f"      run: {sudo_arg}python3 build/fbcode_builder/getdeps.py --allow-system-packages install-system-deps {tests_arg}--recursive patchelf\n"
-                    )
+                    system_deps_cmd += f" && {sudo_arg}{getdepscmd}{allow_sys_arg} install-system-deps {tests_arg}--recursive patchelf"
+                out.write(f"      run: {system_deps_cmd}\n")
+
                 required_locales = manifest.get(
                     "github.actions", "required_locales", ctx=manifest_ctx
                 )
@@ -1152,6 +1191,18 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                         out.write(f"    - name: Ensure {loc} locale present\n")
                         out.write(f"      run: {sudo_arg}locale-gen {loc}\n")
 
+            out.write("    - id: paths\n")
+            out.write("      name: Query paths\n")
+            if build_opts.is_windows():
+                out.write(
+                    f"      run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name}  >> $env:GITHUB_OUTPUT\n"
+                )
+                out.write("      shell: pwsh\n")
+            else:
+                out.write(
+                    f'      run: {getdepscmd}{allow_sys_arg} query-paths {tests_arg}--recursive --src-dir=. {manifest.name}  >> "$GITHUB_OUTPUT"\n'
+                )
+
             projects = loader.manifests_in_dependency_order()
 
             main_repo_url = manifest.get_repo_url(manifest_ctx)
@@ -1178,25 +1229,66 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
                 ctx = loader.ctx_gen.get_context(m.name)
                 if m.get_repo_url(ctx) != main_repo_url:
                     out.write("    - name: Fetch %s\n" % m.name)
+                    out.write(
+                        f"      if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
+                    )
                     out.write(
                         f"      run: {getdepscmd}{allow_sys_arg} fetch --no-tests {m.name}\n"
                     )
 
             for m in projects:
-                if m != manifest:
-                    if m.name == "rust":
-                        continue
+                if m == manifest or m.name == "rust":
+                    continue
+                src_dir_arg = ""
+                ctx = loader.ctx_gen.get_context(m.name)
+                if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
+                    # Its in the same repo, so src-dir is also .
+                    src_dir_arg = "--src-dir=. "
+                    has_same_repo_dep = True
+
+                if args.use_build_cache and not src_dir_arg:
+                    out.write(f"    - name: Restore {m.name} from cache\n")
+                    out.write(f"      id: restore_{m.name}\n")
+                    # only need to restore if would build it
+                    out.write(
+                        f"      if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
+                    )
+                    out.write("      uses: actions/cache/restore@v4\n")
+                    out.write("      with:\n")
+                    out.write(
+                        f"       path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
+                    )
+                    out.write(
+                        f"       key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
+                    )
+
+                out.write("    - name: Build %s\n" % m.name)
+                if not src_dir_arg:
+                    if args.use_build_cache:
+                        out.write(
+                            f"      if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
+                        )
                     else:
-                        src_dir_arg = ""
-                        ctx = loader.ctx_gen.get_context(m.name)
-                        if main_repo_url and m.get_repo_url(ctx) == main_repo_url:
-                            # Its in the same repo, so src-dir is also .
-                            src_dir_arg = "--src-dir=. "
-                            has_same_repo_dep = True
-                        out.write("    - name: Build %s\n" % m.name)
                         out.write(
-                            f"      run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
+                            f"      if: ${{{{ steps.paths.outputs.{m.name}_SOURCE }}}}\n"
                         )
+                out.write(
+                    f"      run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{src_dir_arg}{free_up_disk}--no-tests {m.name}\n"
+                )
+
+                if args.use_build_cache and not src_dir_arg:
+                    out.write(f"    - name: Save {m.name} to cache\n")
+                    out.write("      uses: actions/cache/save@v4\n")
+                    out.write(
+                        f"      if: ${{{{ steps.paths.outputs.{m.name}_SOURCE && ! steps.restore_{m.name}.outputs.cache-hit }}}}\n"
+                    )
+                    out.write("      with:\n")
+                    out.write(
+                        f"       path: ${{{{ steps.paths.outputs.{m.name}_INSTALL }}}}\n"
+                    )
+                    out.write(
+                        f"       key: ${{{{ steps.paths.outputs.{m.name}_CACHE_KEY }}}}-install\n"
+                    )
 
             out.write("    - name: Build %s\n" % manifest.name)
 
@@ -1213,12 +1305,8 @@ def write_job_for_platform(self, platform, args):  # noqa: C901
             if has_same_repo_dep:
                 no_deps_arg = "--no-deps "
 
-            no_tests_arg = ""
-            if not run_tests:
-                no_tests_arg = "--no-tests "
-
             out.write(
-                f"      run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{no_tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
+                f"      run: {getdepscmd}{allow_sys_arg} build {build_type_arg}{tests_arg}{no_deps_arg}--src-dir=. {manifest.name} {project_prefix}\n"
             )
 
             out.write("    - name: Copy artifacts\n")
@@ -1320,6 +1408,13 @@ def setup_project_cmd_parser(self, parser):
             action="store",
             default=None,
         )
+        parser.add_argument(
+            "--no-build-cache",
+            action="store_false",
+            default=True,
+            dest="use_build_cache",
+            help="Do not attempt to use the build cache.",
+        )
 
 
 def get_arg_var_name(args):

CMakeLists.txt

@@ -82,6 +82,14 @@ def recreate_dir(self, src, dst) -> None:
                 shutil.rmtree(dst)
         simple_copytree(src, dst)
 
+    def recreate_linked_dir(self, src, dst) -> None:
+        if os.path.isdir(dst):
+            if os.path.islink(dst):
+                os.remove(dst)
+            elif os.path.isdir(dst):
+                shutil.rmtree(dst)
+        os.symlink(src, dst)
+
     def cargo_config_file(self):
         build_source_dir = self.build_dir
         if self.cargo_config_file_subdir:
@@ -191,7 +199,9 @@ def _build(self, reconfigure) -> None:
                     ],
                 )
 
-        self.recreate_dir(build_source_dir, os.path.join(self.inst_dir, "source"))
+        self.recreate_linked_dir(
+            build_source_dir, os.path.join(self.inst_dir, "source")
+        )
 
     def run_tests(self, schedule_type, owner, test_filter, retry, no_testpilot) -> None:
         if test_filter:

buck2

@@ -244,7 +244,9 @@ def __init__(self, build_options, manifest, repo_url, rev, depth) -> None:
                     if not m:
                         raise Exception("Failed to parse rev from %s" % hash_file)
                     rev = m.group(1)
-                    print("Using pinned rev %s for %s" % (rev, repo_url))
+                    print(
+                        "Using pinned rev %s for %s" % (rev, repo_url), file=sys.stderr
+                    )
 
         self.rev = rev or "main"
         self.origin_repo = repo_url