terraform test: Build mz-debug in the background

Saves ~7 min. We need to wait for terraform anyway

Author: def-

ci/plugins/mzcompose/hooks/pre-exit

@@ -134,14 +134,6 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ] && [ -z "${BUILDKITE_MZCOMPOSE_PLUGIN_SKIP_
     fi
 fi
 
-if [[ "$BUILDKITE_LABEL" =~ Terraform\ .* ]]; then
-  ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy || true
-  PATH="$PWD/test/terraform/azure-temporary/venv/bin:$PATH" VIRTUAL_ENV="$PWD/test/terraform/azure-temporary/venv" bin/ci-builder run stable terraform -chdir=test/terraform/azure-temporary destroy || true
-fi
-rm -rf ~/.kube # Remove potential state from E2E Terraform tests
-
 ci_unimportant_heading ":docker: Cleaning up after mzcompose"
 
 # docker-compose kill may fail attempting to kill containers
@@ -158,10 +150,10 @@ docker ps --all --quiet | xargs --no-run-if-empty docker rm --force --volumes
 
 if [[ "$BUILDKITE_LABEL" =~ Terraform\ .* ]]; then
   ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-upgrade destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/azure-temporary destroy || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy -auto-approve || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/aws-upgrade destroy -auto-approve || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy -auto-approve || true
+  bin/ci-builder run stable terraform -chdir=test/terraform/azure-temporary destroy -auto-approve || true
 fi
 rm -rf ~/.kube # Remove potential state from E2E Terraform tests
 

test/terraform/mzcompose.py

@@ -15,6 +15,7 @@
 import os
 import signal
 import subprocess
+import threading
 import time
 from collections.abc import Sequence
 from pathlib import Path
@@ -140,24 +141,44 @@ def get_tag(tag: str | None) -> str:
     return tag or f"v{ci_util.get_mz_version()}--pr.g{git.rev_parse('HEAD')}"
 
 
-def mz_debug(env: dict[str, str] | None = None) -> None:
+def build_mz_debug_async(env: dict[str, str] | None = None) -> None:
+    def run():
+        spawn.capture(
+            [
+                "cargo",
+                "build",
+                "--bin",
+                "mz-debug",
+            ],
+            cwd=MZ_ROOT,
+            env=env,
+        )
+
+    threading.Thread(target=run, daemon=True).start()
+
+
+def run_mz_debug(env: dict[str, str] | None = None) -> None:
     print("-- Running mz-debug")
-    run_ignore_error(
-        [
-            "cargo",
-            "run",
-            "--bin",
-            "mz-debug",
-            "--",
-            "self-managed",
-            "--k8s-namespace",
-            "materialize-environment",
-            "--k8s-namespace",
-            "materialize",
-        ],
-        cwd=MZ_ROOT,
-        env=env,
-    )
+    try:
+        # mz-debug (and its compilation) is rather noisy, so ignore the output
+        spawn.capture(
+            [
+                "cargo",
+                "run",
+                "--bin",
+                "mz-debug",
+                "--",
+                "self-managed",
+                "--k8s-namespace",
+                "materialize-environment",
+                "--k8s-namespace",
+                "materialize",
+            ],
+            cwd=MZ_ROOT,
+            env=env,
+        )
+    except:
+        pass
 
 
 class AWS:
@@ -667,6 +688,7 @@ def workflow_aws_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
     path = MZ_ROOT / "test" / "terraform" / "aws-temporary"
     aws = AWS(path)
     try:
+        build_mz_debug_async()
         aws.setup("aws-test", args.setup, tag)
         print("--- Running tests")
         with c.override(testdrive(no_reset=False)):
@@ -696,7 +718,7 @@ def workflow_aws_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
     finally:
         aws.cleanup()
 
-        mz_debug()
+        run_mz_debug()
 
         if args.cleanup:
             aws.destroy()
@@ -736,6 +758,7 @@ def workflow_aws_upgrade(c: Composition, parser: WorkflowArgumentParser) -> None
     path = MZ_ROOT / "test" / "terraform" / "aws-upgrade"
     aws = AWS(path)
     try:
+        build_mz_debug_async()
         aws.setup("aws-upgrade", args.setup, previous_tag)
         aws.upgrade(tag)
         # Try waiting a bit, otherwise connection error, should be handled better
@@ -769,7 +792,7 @@ def workflow_aws_upgrade(c: Composition, parser: WorkflowArgumentParser) -> None
     finally:
         aws.cleanup()
 
-        mz_debug()
+        run_mz_debug()
 
         if args.cleanup:
             aws.destroy()
@@ -943,6 +966,7 @@ def workflow_gcp_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
     os.environ["GOOGLE_APPLICATION_CREDENTIALS"] = str(gcloud_creds_path)
 
     try:
+        build_mz_debug_async()
         spawn.runv(["gcloud", "config", "set", "project", "materialize-ci"])
 
         spawn.runv(
@@ -1338,7 +1362,7 @@ def workflow_gcp_temporary(c: Composition, parser: WorkflowArgumentParser) -> No
         if balancerd_port_forward_process:
             os.killpg(os.getpgid(balancerd_port_forward_process.pid), signal.SIGTERM)
 
-        mz_debug()
+        run_mz_debug()
 
         if args.cleanup:
             print("--- Cleaning up")
@@ -1414,6 +1438,7 @@ def workflow_azure_temporary(c: Composition, parser: WorkflowArgumentParser) ->
     )
 
     try:
+        build_mz_debug_async()
         if os.getenv("CI"):
             username = os.getenv("AZURE_SERVICE_ACCOUNT_USERNAME")
             password = os.getenv("AZURE_SERVICE_ACCOUNT_PASSWORD")
@@ -1837,7 +1862,7 @@ def workflow_azure_temporary(c: Composition, parser: WorkflowArgumentParser) ->
         if balancerd_port_forward_process:
             os.killpg(os.getpgid(balancerd_port_forward_process.pid), signal.SIGTERM)
 
-        mz_debug(env=venv_env)
+        run_mz_debug(env=venv_env)
 
         if args.cleanup:
             print("--- Cleaning up")