terraform test: Add some interesting td files and make the cleanup more robust, speed up mz-debug build

ci/nightly/pipeline.template.yml

@@ -1209,6 +1209,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: aws-temporary
+            # Cleanup runs in pre-exit hook
+            args: [--no-cleanup]
       branches: "main v*.* lts-v* *aws* *tf* *terraform* *helm* *self-managed* *orchestratord*"
 
     - id: terraform-aws-upgrade
@@ -1225,6 +1227,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: aws-upgrade
+            # Cleanup runs in pre-exit hook
+            args: [--no-cleanup]
       branches: "main v*.* lts-v* *aws* *tf* *terraform* *helm* *self-managed* *orchestratord*"
 
     - id: terraform-gcp
@@ -1240,6 +1244,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: gcp-temporary
+            # Cleanup runs in pre-exit hook
+            args: [--no-cleanup]
       branches: "main v*.* lts-v* *gcp* *tf* *terraform* *helm* *self-managed* *orchestratord*"
 
     - id: terraform-azure
@@ -1255,6 +1261,8 @@ steps:
         - ./ci/plugins/mzcompose:
             composition: terraform
             run: azure-temporary
+            # Cleanup runs in pre-exit hook
+            args: [--no-cleanup]
       branches: "main v*.* lts-v* *azure* *tf* *terraform* *helm* *self-managed* *orchestratord*"
 
   - group: "Output consistency"

ci/plugins/mzcompose/hooks/pre-exit

@@ -134,14 +134,6 @@ if [ -n "${CI_COVERAGE_ENABLED:-}" ] && [ -z "${BUILDKITE_MZCOMPOSE_PLUGIN_SKIP_
     fi
 fi
 
-if [[ "$BUILDKITE_LABEL" =~ Terraform\ .* ]]; then
-  ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy || true
-  PATH="$PWD/test/terraform/azure-temporary/venv/bin:$PATH" VIRTUAL_ENV="$PWD/test/terraform/azure-temporary/venv" bin/ci-builder run stable terraform -chdir=test/terraform/azure-temporary destroy || true
-fi
-rm -rf ~/.kube # Remove potential state from E2E Terraform tests
-
 ci_unimportant_heading ":docker: Cleaning up after mzcompose"
 
 # docker-compose kill may fail attempting to kill containers
@@ -156,12 +148,14 @@ fi
 ci_collapsed_heading ":docker: Purging all existing docker containers and volumes, regardless of origin"
 docker ps --all --quiet | xargs --no-run-if-empty docker rm --force --volumes
 
-if [[ "$BUILDKITE_LABEL" =~ Terraform\ .* ]]; then
-  ci_unimportant_heading "terraform: Destroying leftover state in case job was cancelled or timed out..."
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/aws-upgrade destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/gcp-temporary destroy || true
-  bin/ci-builder run stable terraform -chdir=test/terraform/azure-temporary destroy || true
+if [ "$BUILDKITE_STEP_KEY" = "terraform-aws" ]; then
+  run run aws-temporary --no-setup --no-test --no-run-mz-debug || CI_ANNOTATE_ERRORS_RESULT=1
+elif [ "$BUILDKITE_STEP_KEY" = "terraform-aws-upgrade" ]; then
+  run run aws-upgrade --no-setup --no-test --no-run-mz-debug || CI_ANNOTATE_ERRORS_RESULT=1
+elif [ "$BUILDKITE_STEP_KEY" = "terraform-gcp" ]; then
+  run run gcp-temporary --no-setup --no-test --no-run-mz-debug || CI_ANNOTATE_ERRORS_RESULT=1
+elif [ "$BUILDKITE_STEP_KEY" = "terraform-azure" ]; then
+  run run azure-temporary --no-setup --no-test --no-run-mz-debug || CI_ANNOTATE_ERRORS_RESULT=1
 fi
 rm -rf ~/.kube # Remove potential state from E2E Terraform tests
 

misc/python/materialize/mzcompose/services/testdrive.py

@@ -64,6 +64,7 @@ def __init__(
         stop_grace_period: str = "120s",
         cluster_replica_size: dict[str, dict[str, Any]] | None = None,
         network_mode: str | None = None,
+        set_persist_urls: bool = True,
     ) -> None:
         depends_graph: dict[str, ServiceDependency] = {}
 
@@ -161,27 +162,30 @@ def __init__(
                 f"--fivetran-destination-files-path={fivetran_destination_files_path}"
             )
 
-        if external_blob_store:
-            blob_store = "azurite" if blob_store_is_azure else "minio"
-            address = blob_store if external_blob_store == True else external_blob_store
-            persist_blob_url = (
-                azure_blob_uri(address)
-                if blob_store_is_azure
-                else minio_blob_uri(address)
-            )
-            entrypoint.append(f"--persist-blob-url={persist_blob_url}")
-        else:
-            entrypoint.append("--persist-blob-url=file:///mzdata/persist/blob")
-
-        if external_metadata_store:
-            depends_graph[metadata_store] = {"condition": "service_healthy"}
-            entrypoint.append(
-                "--persist-consensus-url=postgres://root@cockroach:26257?options=--search_path=consensus"
-            )
-        else:
-            entrypoint.append(
-                f"--persist-consensus-url=postgres://root@{mz_service}:26257?options=--search_path=consensus"
-            )
+        if set_persist_urls:
+            if external_blob_store:
+                blob_store = "azurite" if blob_store_is_azure else "minio"
+                address = (
+                    blob_store if external_blob_store == True else external_blob_store
+                )
+                persist_blob_url = (
+                    azure_blob_uri(address)
+                    if blob_store_is_azure
+                    else minio_blob_uri(address)
+                )
+                entrypoint.append(f"--persist-blob-url={persist_blob_url}")
+            else:
+                entrypoint.append("--persist-blob-url=file:///mzdata/persist/blob")
+
+            if external_metadata_store:
+                depends_graph[metadata_store] = {"condition": "service_healthy"}
+                entrypoint.append(
+                    "--persist-consensus-url=postgres://root@cockroach:26257?options=--search_path=consensus"
+                )
+            else:
+                entrypoint.append(
+                    f"--persist-consensus-url=postgres://root@{mz_service}:26257?options=--search_path=consensus"
+                )
 
         entrypoint.extend(entrypoint_extra)
 

src/testdrive/src/action.rs

@@ -776,7 +776,7 @@ impl Run for PosCommand {
                         consistency::skip_consistency_checks(builtin, state)
                     }
                     "check-shard-tombstone" => {
-                        consistency::run_check_shard_tombstoned(builtin, state).await
+                        consistency::run_check_shard_tombstone(builtin, state).await
                     }
                     "fivetran-destination" => {
                         fivetran::run_destination_command(builtin, state).await

src/testdrive/src/action/consistency.rs

@@ -94,12 +94,12 @@ pub async fn run_consistency_checks(state: &State) -> Result<ControlFlow, anyhow
 /// Checks if a shard in Persist has been tombstoned.
 ///
 /// TODO(parkmycar): Run this as part of the consistency checks, instead of as a specific command.
-pub async fn run_check_shard_tombstoned(
+pub async fn run_check_shard_tombstone(
     mut cmd: BuiltinCommand,
     state: &State,
 ) -> Result<ControlFlow, anyhow::Error> {
     let shard_id = cmd.args.string("shard-id")?;
-    check_shard_tombstoned(state, &shard_id).await?;
+    check_shard_tombstone(state, &shard_id).await?;
     Ok(ControlFlow::Continue)
 }
 
@@ -237,8 +237,8 @@ async fn check_catalog_state(state: &State) -> Result<(), anyhow::Error> {
 }
 
 /// Checks if the provided `shard_id` is a tombstone, returning an error if it's not.
-async fn check_shard_tombstoned(state: &State, shard_id: &str) -> Result<(), anyhow::Error> {
-    println!("$ check-shard-tombstoned {shard_id}");
+async fn check_shard_tombstone(state: &State, shard_id: &str) -> Result<(), anyhow::Error> {
+    println!("$ check-shard-tombstone {shard_id}");
 
     let (Some(consensus_uri), Some(blob_uri)) =
         (&state.persist_consensus_url, &state.persist_blob_url)