improves batch build generation (#4138)

aws · Dec 19, 2024 · d0066f9 · d0066f9
1 parent 5303a93
commit d0066f9
Show file tree

Hide file tree

Showing 41 changed files with 323 additions and 783 deletions.
diff --git a/Common.mk b/Common.mk
@@ -84,6 +84,10 @@ BUILDSPEC_VARS_KEYS?=
 BUILDSPEC_VARS_VALUES?=
 BUILDSPEC_PLATFORM?=ARM_CONTAINER
 BUILDSPEC_COMPUTE_TYPE?=BUILD_GENERAL1_SMALL
+BUILDSPECS_FOR_COMBINE_IMAGES=buildspec.yml buildspecs/combine-images.yml
+BUILDSPEC_1_VARS_KEYS?=$(if $(findstring $(BUILDSPECS_FOR_COMBINE_IMAGES),$(BUILDSPECS)),IMAGE_PLATFORMS,)
+BUILDSPEC_1_VARS_VALUES?=$(if $(findstring $(BUILDSPECS_FOR_COMBINE_IMAGES),$(BUILDSPECS)),IMAGE_PLATFORMS,)
+BUILDSPEC_2_DEPENDS_ON_OVERRIDE?=$(if $(filter buildspecs/combine-images.yml,$(word 2,$(BUILDSPECS))),BUILDSPEC_1,)
 ####################################################
 
 #################### GIT ###########################
@@ -907,7 +911,7 @@ release: $(or $(RELEASE_TARGETS_OVERRIDE),$(RELEASE_TARGETS))
 	@set -e; \
 	for version in $(SUPPORTED_K8S_VERSIONS) ; do \
 	    if ! [[ "$(SKIPPED_K8S_VERSIONS)" =~ $$version  ]]; then \
-			$(MAKE) $* $(if $(filter true,$(BINARIES_ARE_RELEASE_BRANCHED)),clean-output,) RELEASE_BRANCH=$$version; \
+			$(MAKE) $* $(if $(filter true,$(BINARIES_ARE_RELEASE_BRANCHED)),clean-output,) RELEASE_BRANCH=$$version IMAGE_PLATFORMS=$(call IF_OVERRIDE_VARIABLE,IMAGE_PLATFORMS,); \
 		fi \
 	done;
 

diff --git a/Makefile b/Makefile
@@ -199,7 +199,7 @@ generate-project-list: | ensure-locale
 generate-staging-buildspec: export BINARY_PLATFORMS=linux/amd64 linux/arm64
 generate-staging-buildspec: export IMAGE_PLATFORMS=linux/amd64 linux/arm64
 generate-staging-buildspec: | ensure-locale
-	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml" "$(BASE_DIRECTORY)/buildspec.yml"
+	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/staging-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" false EXCLUDE_FROM_STAGING_BUILDSPEC BUILDSPECS false
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "$(ALL_PROJECTS)" "$(BASE_DIRECTORY)/release/checksums-build.yml" "$(BASE_DIRECTORY)/buildspecs/checksums-buildspec.yml" true EXCLUDE_FROM_CHECKSUMS_BUILDSPEC CHECKSUMS_BUILDSPECS false buildspecs/checksums-pr-buildspec.yml
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "aws_bottlerocket-bootstrap" "$(BASE_DIRECTORY)/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
 	build/lib/generate_staging_buildspec.sh $(BASE_DIRECTORY) "kubernetes_cloud-provider-vsphere" "$(BASE_DIRECTORY)/projects/kubernetes/cloud-provider-vsphere/buildspecs/batch-build.yml" "$(BASE_DIRECTORY)/buildspec.yml" true
@@ -240,7 +240,7 @@ check-project-path-exists:
 
 .PHONY: validate-release-buildspecs
 validate-release-buildspecs:
-	build/lib/validate_release_buildspecs.sh
+	build/lib/validate_release_buildspecs.sh "$(BASE_DIRECTORY)/release/checksums-build.yml" "$(BASE_DIRECTORY)/release/staging-build.yml" "$(BASE_DIRECTORY)/tools/version-tracker/buildspecs/upgrade.yml"
 
 .PHONY: validate-eksd-releases
 validate-eksd-releases:

diff --git a/build/lib/generate_staging_buildspec.sh b/build/lib/generate_staging_buildspec.sh
@@ -76,6 +76,7 @@ for project in "${PROJECTS[@]}"; do
 
     BUILDSPECS=$(make_var $PROJECT_PATH $BUILDSPECS_VAR)
     SPECS=(${BUILDSPECS// / })
+    PREVIOUS_SPEC_IDENTIFIERS=""
     for (( i=0; i < ${#SPECS[@]}; i++ )); do
         IDENTIFIER="${org//-/_}_${repo//-/_}"
 
@@ -94,17 +95,28 @@ for project in "${PROJECTS[@]}"; do
             if [[ "none" = "$BUILDSPEC_DEPENDS_ON" ]]; then
                 PROJECT_DEPENDENCIES=""
                 break
+            elif [[ -n "$BUILDSPEC_DEPENDS_ON" ]] && [[ $BUILDSPEC_DEPENDS_ON  == BUILDSPEC_* ]]; then
+                # TODO: In the makefiles we will use `BUILDSPEC_1` as if you can control the specific buildspec
+                # to depend on. This code assumes its always the previous
+                HARDCODED_DEP="true"
+                PROJECT_DEPENDENCIES=$PREVIOUS_SPEC_IDENTIFIERS
+                break
             elif [[ -n "$BUILDSPEC_DEPENDS_ON" ]]; then
                 HARDCODED_DEP="true"
                 PROJECT_DEPENDENCIES=$BUILDSPEC_DEPENDS_ON
                 break
             fi
         done
-
+
+        PREVIOUS_SPEC_IDENTIFIERS=""
+
+        BUILDSPEC_NAME=$(basename $buildspec .yml)
         BUILDSPEC_IDENTIFIER_OVERRIDE="$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_IDENTIFIER_OVERRIDE)"
         if [[ -n "$BUILDSPEC_IDENTIFIER_OVERRIDE" ]]; then
             IDENTIFIER="$BUILDSPEC_IDENTIFIER_OVERRIDE"
-        fi   
+        elif [[ "${BUILDSPEC_NAME}" != *buildspec* ]]; then
+            IDENTIFIER+="_${BUILDSPEC_NAME//-/_}"
+        fi
 
         echo "Adding: $IDENTIFIER"
 
@@ -134,10 +146,20 @@ for project in "${PROJECTS[@]}"; do
                 fi
 
                 DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}
+
                 if [ -n "${DEP_RELEASE_BRANCH}" ]; then
                     DEP_IDENTIFIER=${DEP_ORG//-/_}_${DEP_REPO//-/_}_${DEP_RELEASE_BRANCH//[-\/]/_}
                 fi
-                DEPEND_ON+="\"${DEP_IDENTIFIER}\","
+                # if dep is split by binary platform, append platform/arch
+                DEP_BUILDSPEC_VARS="$(make_var $MAKE_ROOT/projects/$DEP_ORG/$DEP_REPO BUILDSPEC_VARS_KEYS)"
+                DEP_BUILDSPECS="$(make_var $MAKE_ROOT/projects/$DEP_ORG/$DEP_REPO BUILDSPECS)"
+                if [[ "$DEP_BUILDSPEC_VARS" == "BINARY_PLATFORMS" ]]; then
+                    DEPEND_ON+="\"${DEP_IDENTIFIER}_linux_amd64\",\"${DEP_IDENTIFIER}_linux_arm64\","
+                elif [[ $DEP_BUILDSPECS == *combine-images.yml* ]]; then
+                    DEPEND_ON+="\"${DEP_IDENTIFIER}_combine_images\","
+                else
+                    DEPEND_ON+="\"${DEP_IDENTIFIER}\","
+                fi
             done
         fi
 
@@ -160,21 +182,42 @@ for project in "${PROJECTS[@]}"; do
         fi
 
         ARCH_TYPE="\"type\":\"$BUILDSPEC_PLATFORM\",\"compute-type\":\"$BUILDSPEC_COMPUTE_TYPE\","
-
-        if [[ "$BUILDSPECS_VAR" == "CHECKSUMS_BUILDSPECS" ]] && [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" || ("$BUILDSPEC_VARS_KEYS" == "RELEASE_BRANCH" && "false" == "$(make_var $PROJECT_PATH BINARIES_ARE_RELEASE_BRANCHED)") ]]; then
-            BUILDSPEC_VARS_KEYS=""
+
+        BUILDSPEC_VARS_VALUES=""
+        if [[ "$BUILDSPECS_VAR" == "CHECKSUMS_BUILDSPECS" ]]; then
+            if [[ "true" == "$(make_var $PROJECT_PATH HAS_RELEASE_BRANCHES)" ]]; then
+                if [[ "false" == "$(make_var $PROJECT_PATH BINARIES_ARE_RELEASE_BRANCHED)" ]]; then
+                    BUILDSPEC_VARS_KEYS=""
+                else
+                    BUILDSPEC_VARS_KEYS="RELEASE_BRANCH"
+                    BUILDSPEC_VARS_VALUES="SUPPORTED_K8S_VERSIONS"
+                fi
+            elif [[ "${BUILDSPEC_VARS_KEYS}" = "IMAGE_PLATFORMS" ]]; then            
+                BUILDSPEC_VARS_KEYS=""
+            fi
         fi
 
-        if [[ "$BUILDSPECS_VAR" == "UPGRADE_BUILDSPECS" ]] && [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then
-            BUILDSPEC_VARS_KEYS=""
+        if [[ "$BUILDSPECS_VAR" == "UPGRADE_BUILDSPECS" ]]; then
+            if [[ "${IDENTIFIER}" = "kubernetes_sigs_image_builder" ]]; then            
+                BUILDSPEC_VARS_KEYS=""
+            elif [[ "true" == "$(make_var $PROJECT_PATH HAS_RELEASE_BRANCHES)" ]]; then
+                if [[ "false" == "$(make_var $PROJECT_PATH BINARIES_ARE_RELEASE_BRANCHED)" ]]; then
+                    BUILDSPEC_VARS_KEYS=""
+                else
+                    BUILDSPEC_VARS_KEYS="RELEASE_BRANCH"
+                    BUILDSPEC_VARS_VALUES="SUPPORTED_K8S_VERSIONS"
+                fi
+            fi
         fi
 
         if [[ -n "$BUILDSPEC_VARS_KEYS" ]]; then
             KEYS=(${BUILDSPEC_VARS_KEYS// / })
 
-            BUILDSPEC_VARS_VALUES=$(make_var $PROJECT_PATH BUILDSPEC_VARS_VALUES)
             if [[ -z "$BUILDSPEC_VARS_VALUES" ]]; then
-                BUILDSPEC_VARS_VALUES=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_VARS_VALUES)
+                BUILDSPEC_VARS_VALUES=$(make_var $PROJECT_PATH BUILDSPEC_VARS_VALUES)
+                if [[ -z "$BUILDSPEC_VARS_VALUES" ]]; then
+                    BUILDSPEC_VARS_VALUES=$(make_var $PROJECT_PATH BUILDSPEC_$((( $i + 1 )))_VARS_VALUES)
+                fi
             fi
             VARS=(${BUILDSPEC_VARS_VALUES// / })
 
@@ -184,8 +227,7 @@ for project in "${PROJECTS[@]}"; do
                 ARR_1=(${VALUES_1// / })
 
                 for val1 in "${ARR_1[@]}"; do                
-                    BUILDSPEC_NAME=$(basename $buildspec .yml)
-                    IDENTIFIER=${org//-/_}_${repo//-/_}_${val1//[-\/]/_}
+                    IDENTIFIER_WITH_VAL=${IDENTIFIER}_${val1//[-\/]/_}
 
                     # If building on one binary platform assume we want to run on a specific arch instance
                     ARCH_TYPE="\"type\":\"$BUILDSPEC_PLATFORM\",\"compute-type\":\"$BUILDSPEC_COMPUTE_TYPE\","
@@ -206,9 +248,10 @@ for project in "${PROJECTS[@]}"; do
                         fi
                     fi
 
-                    ALL_PROJECT_IDS+="\"$IDENTIFIER\","
+                    ALL_PROJECT_IDS+="\"$IDENTIFIER_WITH_VAL\","
+                    PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER_WITH_VAL "
                     yq eval -i -P \
-                        ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\"$EXTRA_VARS}}}]" \
+                        ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER_WITH_VAL\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\"$EXTRA_VARS}}}]" \
                         $STAGING_BUILDSPEC_FILE
 
                 done
@@ -220,8 +263,7 @@ for project in "${PROJECTS[@]}"; do
                 ARR_2=(${VALUES_2// / })
                 for val1 in "${ARR_1[@]}"; do
                     for val2 in "${ARR_2[@]}"; do
-                        BUILDSPEC_NAME=$(basename $buildspec .yml)
-                        IDENTIFIER=${org//-/_}_${repo//-/_}_${val1//[-\/]/_}_${val2//[-\/]/_}_${BUILDSPEC_NAME//-/_}
+                        IDENTIFIER_WITH_VAL=${IDENTIFIER}_${val1//[-\/]/_}_${val2//[-\/]/_}
                         # TODO: revisit this to make it more dynamic if other projects need it in the future
                         EXTRA_VARS=""
                         if [[ "$IDENTIFIER" =~ "kubernetes_sigs_image_builder_bottlerocket" ]]; then
@@ -239,15 +281,17 @@ for project in "${PROJECTS[@]}"; do
                                 ARCH_TYPE="\"type\":\"ARM_CONTAINER\",\"compute-type\":\"$BUILDSPEC_COMPUTE_TYPE\","
                             fi
                         fi
-                        ALL_PROJECT_IDS+="\"$IDENTIFIER\","
+                        ALL_PROJECT_IDS+="\"$IDENTIFIER_WITH_VAL\","
+                        PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER_WITH_VAL "
                         yq eval -i -P \
-                            ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"$EXTRA_VARS}}}]" \
+                            ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER_WITH_VAL\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL,\"${KEYS[0]}\":\"$val1\",\"${KEYS[1]}\":\"$val2\"$EXTRA_VARS}}}]" \
                             $STAGING_BUILDSPEC_FILE 
                     done
                 done
             fi
         else
             ALL_PROJECT_IDS+="\"$IDENTIFIER\","
+            PREVIOUS_SPEC_IDENTIFIERS+="$IDENTIFIER "
             yq eval -i -P \
                 ".batch.build-graph += [{\"identifier\":\"$IDENTIFIER\",$buildspec_field$DEPEND_ON\"env\":{$ARCH_TYPE\"variables\":{\"PROJECT_PATH\": \"projects/$org/$repo\"$CLONE_URL}}}]" \
                 $STAGING_BUILDSPEC_FILE 
@@ -277,3 +321,5 @@ if [[ "${#PROJECTS[@]}" = "1" ]]; then
     yq -i 'del(.. | select(tag == "!!map" and length == 0))' $STAGING_BUILDSPEC_FILE
     yq -i 'del(.. | select(tag == "!!map" and length == 0))' $STAGING_BUILDSPEC_FILE
 fi
+
+$SCRIPT_ROOT/validate_release_buildspecs.sh "$STAGING_BUILDSPEC_FILE"
diff --git a/build/lib/setup.sh b/build/lib/setup.sh
@@ -10,7 +10,7 @@ source "${SCRIPT_ROOT}/common.sh"
 source /docker.sh 
 
 CODEBUILD_CI="${CODEBUILD_CI:-false}"
-QEMU_INSTALLER_IMAGE="public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v7.0.0"
+# QEMU_INSTALLER_IMAGE="public.ecr.aws/eks-distro-build-tooling/binfmt-misc:qemu-v7.0.0"
 GIT_CONFIG_SCOPE="--global"
 if [[ "$CODEBUILD_CI" = "true" ]] && [[ "$CODEBUILD_BUILD_ID" =~ "aws-staging-bundle-build" ]]; then
     GIT_CONFIG_SCOPE="--system"
@@ -36,12 +36,12 @@ update-alternatives --set ip6tables /usr/sbin/ip6tables-legacy
 start::dockerd
 wait::for::dockerd
 
-build::docker::retry_pull $QEMU_INSTALLER_IMAGE
+# build::docker::retry_pull $QEMU_INSTALLER_IMAGE
 
-CURRENT_ARCH="$(uname -m)"
-if [[ "$CURRENT_ARCH" == "amd64" || "$CURRENT_ARCH" == "x86_64" ]]; then
-    EMULATOR_ARCH="aarch64"
-elif [[ "$CURRENT_ARCH" == "arm64" || "$CURRENT_ARCH" == "aarch64" ]]; then
-    EMULATOR_ARCH="amd64"
-fi
-docker run --privileged --rm $QEMU_INSTALLER_IMAGE --install $EMULATOR_ARCH
+# CURRENT_ARCH="$(uname -m)"
+# if [[ "$CURRENT_ARCH" == "amd64" || "$CURRENT_ARCH" == "x86_64" ]]; then
+#     EMULATOR_ARCH="aarch64"
+# elif [[ "$CURRENT_ARCH" == "arm64" || "$CURRENT_ARCH" == "aarch64" ]]; then
+#     EMULATOR_ARCH="amd64"
+# fi
+# docker run --privileged --rm $QEMU_INSTALLER_IMAGE --install $EMULATOR_ARCH
diff --git a/build/lib/validate_release_buildspecs.sh b/build/lib/validate_release_buildspecs.sh
@@ -19,13 +19,12 @@ set -o nounset
 set -o pipefail
 
 REPO_ROOT="$(git rev-parse --show-toplevel)"
-RELEASE_BUILDSPECS=("$REPO_ROOT/release/checksums-build.yml" "$REPO_ROOT/release/staging-build.yml" "$REPO_ROOT/tools/version-tracker/buildspecs/upgrade.yml")
 
 VALIDATIONS_FAILED=0
-for buildspec in "${RELEASE_BUILDSPECS[@]}"; do
+for buildspec in "$@"; do
     echo "Validating builds count in build graph for buildspec - $buildspec"
     num_builds_in_batch=$(yq ".batch.build-graph | length" $buildspec)
-    if [[ $num_builds_in_batch -ge 100 ]]; then
+    if [[ $num_builds_in_batch -gt 100 ]]; then
         echo "Maximum allowed builds in batch is 100, current number of builds: $num_builds_in_batch"
         VALIDATIONS_FAILED=1
         INVALID_BUILDSPEC="true"
@@ -43,7 +42,8 @@ for buildspec in "${RELEASE_BUILDSPECS[@]}"; do
         VALIDATIONS_FAILED=1
         INVALID_BUILDSPEC="true"
     fi
-
+
+    invalid_dependencies=()
     if [ "${#depends_on_list[@]}" -gt 0 ]; then
         echo "Validating identifiers in depend-on list are valid identifiers in build graph in the buildspec - $buildspec"
         invalid_dependencies=($(for dependency in ${depends_on_list[@]}; do

diff --git a/projects/aws/bottlerocket-bootstrap/Makefile b/projects/aws/bottlerocket-bootstrap/Makefile
@@ -6,6 +6,15 @@ REPO_OWNER=aws
 
 BASE_IMAGE_NAME?=eks-distro-base
 
+# we need to set IMAGE_BUILD_ARGS here even though its the same as the default. 
+# it is set in Common.mk on the images target (https://github.com/aws/eks-anywhere-build-tooling/blob/8b6d6d66974e881b22e3c9c8ea29adc26f7df5fd/Common.mk#L799)
+# and the combine-images target (https://github.com/aws/eks-anywhere-build-tooling/blob/8b6d6d66974e881b22e3c9c8ea29adc26f7df5fd/Common.mk#L846)
+# since combine-images has images as prereq target, the ?= does not really behavior as one might expect.
+# the images target being the actual action, its version of the set takes prioirty and resets to empty
+# setting it explicitly to empty here takes allows the combine-images override to take proirty
+IMAGE_BUILD_ARGS=
+DOCKERFILE_FOLDER=$(if $(filter bottlerocket-bootstrap-snow,$(IMAGE_NAME)),./docker/linux/snow,./docker/linux)
+
 IMAGE_NAMES=bottlerocket-bootstrap bottlerocket-bootstrap-snow
 
 BOTTLEROCKET_BOOTSTRAP_IMAGE_COMPONENT=bottlerocket-bootstrap
@@ -19,8 +28,9 @@ GIT_TAG=v$(RELEASE_BRANCH)-$(shell yq e ".releases[] | select(.branch==\"${RELEA
 
 HAS_RELEASE_BRANCHES=true
 
-BUILDSPEC_VARS_KEYS=RELEASE_BRANCH
-BUILDSPEC_VARS_VALUES=SUPPORTED_K8S_VERSIONS
+BUILDSPECS=$(BUILDSPECS_FOR_COMBINE_IMAGES)
+BUILDSPEC_1_COMPUTE_TYPE=BUILD_GENERAL1_LARGE
+
 EXCLUDE_FROM_UPGRADE_BUILDSPEC=true
 
 # Go 1.18 and newer automatically embeds vcs information if the go module being built
@@ -81,8 +91,6 @@ $(TOOLS_BIN_DIR):
 $(MOCKGEN): $(TOOLS_BIN_DIR)
 	GOBIN=$(TOOLS_BIN_DIR) $(GO) install github.com/golang/mock/[email protected]
 
-bottlerocket-bootstrap-snow/images/%: DOCKERFILE_FOLDER=./docker/linux/snow
-
 GIT_TAG:
 	echo 0.0.0 > $@
 

diff --git a/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml b/projects/aws/bottlerocket-bootstrap/buildspecs/batch-build.yml
@@ -17,36 +17,30 @@
 batch:
   fast-fail: true
   build-graph:
-    - identifier: aws_bottlerocket_bootstrap_1_27
+    - identifier: aws_bottlerocket_bootstrap_linux_amd64
       env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          RELEASE_BRANCH: 1-27
-    - identifier: aws_bottlerocket_bootstrap_1_28
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
+        type: LINUX_CONTAINER
+        compute-type: BUILD_GENERAL1_LARGE
         variables:
-          RELEASE_BRANCH: 1-28
-    - identifier: aws_bottlerocket_bootstrap_1_29
+          IMAGE_PLATFORMS: linux/amd64
+          BINARY_PLATFORMS: linux/amd64
+          IMAGE_TAG_SUFFIX: -amd64
+    - identifier: aws_bottlerocket_bootstrap_linux_arm64
       env:
         type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          RELEASE_BRANCH: 1-29
-    - identifier: aws_bottlerocket_bootstrap_1_30
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
+        compute-type: BUILD_GENERAL1_LARGE
         variables:
-          RELEASE_BRANCH: 1-30
-    - identifier: aws_bottlerocket_bootstrap_1_31
+          IMAGE_PLATFORMS: linux/arm64
+          BINARY_PLATFORMS: linux/arm64
+          IMAGE_TAG_SUFFIX: -arm64
+    - identifier: aws_bottlerocket_bootstrap_combine_images
+      buildspec: buildspecs/combine-images.yml
+      depend-on:
+        - aws_bottlerocket_bootstrap_linux_amd64
+        - aws_bottlerocket_bootstrap_linux_arm64
       env:
         type: ARM_CONTAINER
         compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          RELEASE_BRANCH: 1-31
 version: 0.2
 phases:
   pre_build:

diff --git a/projects/aws/eks-anywhere-build-tooling/Makefile b/projects/aws/eks-anywhere-build-tooling/Makefile
@@ -25,6 +25,9 @@ REPO_NO_CLONE=true
 
 EXCLUDE_FROM_CHECKSUMS_BUILDSPEC=true
 EXCLUDE_FROM_UPGRADE_BUILDSPEC=true
+# By default the kind combine-images build will be added to the dep list
+# to avoid waiting for that to finish, just wait on the binaries
+BUILDSPEC_DEPENDS_ON_OVERRIDE=fluxcd_flux2 kubernetes_sigs_cluster_api kubernetes_sigs_kind_binaries replicatedhq_troubleshoot vmware_govmomi helm_helm apache_cloudstack_cloudmonkey
 
 include $(BASE_DIRECTORY)/Common.mk
 

diff --git a/projects/aws/upgrader/Makefile b/projects/aws/upgrader/Makefile
@@ -8,10 +8,6 @@ GIT_TAG=v$(RELEASE_BRANCH)-$(shell yq e ".releases[] | select(.branch==\"${RELEA
 
 HAS_RELEASE_BRANCHES=true
 
-EXCLUDE_FROM_UPGRADE_BUILDSPEC=true
-# for the staging buildspec generation
-BUILDSPEC_DEPENDS_ON_OVERRIDE=containerd_containerd_linux_amd64 containerd_containerd_linux_arm64 kubernetes_sigs_cri_tools
-
 # Go 1.18 and newer automatically embeds vcs information if the go module being built
 # belongs (upwards) to a git repo. In this case, that repo is the buildtooling repo, 
 # which is problematic. Given these binaries are built from different banches,
@@ -33,14 +29,9 @@ GIT_CHECKOUT_TARGET=main.go
 FAKE_GIT_REPO_TARGET=.git
 REPO_NO_CLONE=true
 
-
-BUILDSPECS=buildspec.yml buildspecs/combine-images.yml
+BUILDSPECS=$(BUILDSPECS_FOR_COMBINE_IMAGES)
 BUILDSPEC_1_COMPUTE_TYPE=BUILD_GENERAL1_LARGE
-BUILDSPEC_1_VARS_KEYS=IMAGE_PLATFORMS
-BUILDSPEC_1_VARS_VALUES=IMAGE_PLATFORMS
-BUILDSPEC_1_ARCH_TYPES=LINUX_CONTAINER ARM_CONTAINER
-BUILDSPEC_2_DEPENDS_ON_OVERRIDE=aws_upgrader_linux_amd64 aws_upgrader_linux_arm64
-
+EXCLUDE_FROM_UPGRADE_BUILDSPEC=true
 
 BASE_IMAGE_NAME=eks-distro-minimal-base-nsenter
 IMAGE_NAMES=upgrader