Extend embuilder deferred building mode to ports (#23924)

This allows embuilder build ALL to mix the ports builds together with
the system_libs builds, for even faster build-linux speed.
Headers are still installed eagerly during ports.get() (to allow
dependencies to build their object files) but does not run ninja to build the ports
until the explicit step at the end.

Author: dschuff

embuilder.py

@@ -15,6 +15,7 @@
 import argparse
 import fnmatch
 import logging
+import os
 import sys
 import time
 from contextlib import contextmanager
@@ -171,8 +172,8 @@ def clear_port(port_name):
 
 
 def build_port(port_name):
-  with get_port_variant(port_name) as port_name:
-    ports.build_port(port_name, settings)
+  with get_port_variant(port_name) as port_name_base:
+    ports.build_port(port_name_base, settings)
 
 
 def get_system_tasks():
@@ -281,6 +282,9 @@ def main():
   if auto_tasks:
     print('Building targets: %s' % ' '.join(tasks))
 
+  if USE_NINJA:
+    os.environ['EMBUILDER_PORT_BUILD_DEFERRED'] = '1'
+
   for what in tasks:
     for old, new in legacy_prefixes.items():
       if what.startswith(old):

tools/cache.py

@@ -152,7 +152,7 @@ def get_lib(libname, *args, **kwargs):
 
 # Request a cached file. If it isn't in the cache, it will be created with
 # the given creator function
-def get(shortname, creator, what=None, force=False, quiet=False, deferred=False):
+def get(shortname, creator, what=None, force=False, quiet=False):
   ensure_setup()
   cachename = Path(cachedir, shortname)
   # Check for existence before taking the lock in case we can avoid the
@@ -177,8 +177,12 @@ def get(shortname, creator, what=None, force=False, quiet=False, deferred=False)
     logger.info(message)
     utils.safe_ensure_dirs(cachename.parent)
     creator(str(cachename))
-    if not deferred:
-      assert cachename.exists()
+    # In embuilder/deferred building mode, the library is not actually compiled at
+    # "creation" time; instead, the ninja files are built up incrementally, and
+    # compiled all at once with a single ninja invocation. So in that case we
+    # can't assert that the library was correctly built here.
+    if not os.getenv('EMBUILDER_PORT_BUILD_DEFERRED'):
+      assert cachename.is_file()
     if not quiet:
       logger.info(' - ok')
 

tools/ports/__init__.py

@@ -203,7 +203,8 @@ def build_port(src_dir, output_path, port_name, includes=[], flags=[], cxxflags=
       ninja_file = os.path.join(build_dir, 'build.ninja')
       system_libs.ensure_sysroot()
       system_libs.create_ninja_file(srcs, ninja_file, output_path, cflags=cflags)
-      system_libs.run_ninja(build_dir)
+      if not os.getenv('EMBUILDER_PORT_BUILD_DEFERRED'):
+        system_libs.run_ninja(build_dir)
     else:
       commands = []
       objects = []
@@ -577,9 +578,11 @@ def add_cflags(args, settings): # noqa: U100
   needed = get_needed_ports(settings)
 
   # Now get (i.e. build) the ports in dependency order.  This is important because the
-  # headers from one ports might be needed before we can build the next.
+  # headers from one port might be needed before we can build the next.
   for port in dependency_order(needed):
-    port.get(Ports, settings, shared)
+    # When using embuilder, don't build the dependencies
+    if not os.getenv('EMBUILDER_PORT_BUILD_DEFERRED'):
+      port.get(Ports, settings, shared)
     args += port.process_args(Ports)
 
 

tools/system_libs.py

@@ -431,8 +431,7 @@ def build(self):
     return cache.get(self.get_path(), self.do_build, force=USE_NINJA == 2, quiet=USE_NINJA)
 
   def generate(self):
-    return cache.get(self.get_path(), self.do_generate, force=USE_NINJA == 2, quiet=USE_NINJA,
-                     deferred=True)
+    return cache.get(self.get_path(), self.do_generate, force=USE_NINJA == 2, quiet=USE_NINJA)
 
   def get_link_flag(self):
     """