Remove unnecessary builds from batch buildspecs (#4136)

Co-authored-by: Abhay Krishna Arunachalam <[email protected]>

Common.mk

@@ -98,7 +98,7 @@ REPO_SPARSE_CHECKOUT?=
 #################### RELEASE BRANCHES ##############
 HAS_RELEASE_BRANCHES?=false
 RELEASE_BRANCH?=
-SUPPORTED_K8S_VERSIONS=$(shell cat $(BASE_DIRECTORY)/release/SUPPORTED_RELEASE_BRANCHES)
+SUPPORTED_K8S_VERSIONS?=$(shell cat $(BASE_DIRECTORY)/release/SUPPORTED_RELEASE_BRANCHES)
 # Comma-separated list of Kubernetes versions to skip building artifacts for
 SKIPPED_K8S_VERSIONS?=
 BINARIES_ARE_RELEASE_BRANCHED?=true

build/lib/validate_release_buildspecs.sh

@@ -19,15 +19,23 @@ set -o nounset
 set -o pipefail
 
 REPO_ROOT="$(git rev-parse --show-toplevel)"
-RELEASE_BUILDSPECS=("$REPO_ROOT/release/checksums-build.yml" "$REPO_ROOT/release/staging-build.yml")
+RELEASE_BUILDSPECS=("$REPO_ROOT/release/checksums-build.yml" "$REPO_ROOT/release/staging-build.yml" "$REPO_ROOT/tools/version-tracker/buildspecs/upgrade.yml")
 
 VALIDATIONS_FAILED=0
 for buildspec in "${RELEASE_BUILDSPECS[@]}"; do
+    echo "Validating builds count in build graph for buildspec - $buildspec"
+    num_builds_in_batch=$(yq ".batch.build-graph | length" $buildspec)
+    if [[ $num_builds_in_batch -ge 100 ]]; then
+        echo "Maximum allowed builds in batch is 100, current number of builds: $num_builds_in_batch"
+        VALIDATIONS_FAILED=1
+        INVALID_BUILDSPEC="true"
+    fi
+
     depends_on_list=($(yq "[.batch.build-graph[].depend-on[] | select(. != \"null\")] | unique | .[]" $buildspec))
     identifier_list=($(yq ".batch.build-graph[].identifier" $buildspec))
 
     INVALID_BUILDSPEC="false"
-    echo "Validating identifier unqiueness in the buildspec - $buildspec"
+    echo "Validating identifier uniqueness in the buildspec - $buildspec"
     duplicate_ids=($(printf '%s\n' "${identifier_list[@]}"|awk '!($0 in seen){seen[$0];next} 1' | uniq))
     if [ "${#duplicate_ids[@]}" -gt 0 ]; then
         printf -v duplicate_id_csv '%s,' "${duplicate_ids[@]}"
@@ -36,10 +44,12 @@ for buildspec in "${RELEASE_BUILDSPECS[@]}"; do
         INVALID_BUILDSPEC="true"
     fi
 
-    echo "Validating identifiers in depend-on list are valid identifiers in build graph in the buildspec - $buildspec"
-    invalid_dependencies=($(for dependency in ${depends_on_list[@]}; do
-        [[ ${identifier_list[*]} =~ (^|[[:space:]])"$dependency"($|[[:space:]]) ]] || echo "$dependency"
-    done))
+    if [ "${#depends_on_list[@]}" -gt 0 ]; then
+        echo "Validating identifiers in depend-on list are valid identifiers in build graph in the buildspec - $buildspec"
+        invalid_dependencies=($(for dependency in ${depends_on_list[@]}; do
+            [[ ${identifier_list[*]} =~ (^|[[:space:]])"$dependency"($|[[:space:]]) ]] || echo "$dependency"
+        done))
+    fi
 
     if [ "${#invalid_dependencies[@]}" -gt 0 ]; then
         printf -v invalid_deps_csv '%s,' "${invalid_dependencies[@]}"

projects/kubernetes-sigs/image-builder/Makefile

@@ -205,7 +205,7 @@ endif
 
 #####################################################################
 
-BUILDSPECS=projects/$(COMPONENT)/buildspecs/ova.yml projects/$(COMPONENT)/buildspecs/raw.yml projects/$(COMPONENT)/buildspecs/ami.yml
+BUILDSPECS=projects/$(COMPONENT)/buildspecs/ova.yml projects/$(COMPONENT)/buildspecs/raw.yml
 BUILDSPEC_DEPENDS_ON_OVERRIDE=none
 BUILDSPEC_VARS_KEYS=IMAGE_OS RELEASE_BRANCH
 BUILDSPEC_VARS_VALUES=SUPPORTED_OS_DISTROS SUPPORTED_K8S_VERSIONS

projects/kubernetes/cloud-provider-aws/Makefile

@@ -12,6 +12,7 @@ EXTRA_GO_LDFLAGS=-X k8s.io/component-base/version.gitVersion=$(shell cat ./$(REL
 HAS_S3_ARTIFACTS=true
 HAS_RELEASE_BRANCHES=true
 BUILDSPEC_VARS_KEYS=RELEASE_BRANCH
+SUPPORTED_K8S_VERSIONS=1-27
 BUILDSPEC_VARS_VALUES=SUPPORTED_K8S_VERSIONS
 
 IMAGE_NAMES=

release/checksums-build.yml

@@ -268,38 +268,6 @@ batch:
           PROJECT_PATH: projects/kubernetes/cloud-provider-aws
           CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
           RELEASE_BRANCH: 1-27
-    - identifier: kubernetes_cloud_provider_aws_1_28
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-28
-    - identifier: kubernetes_cloud_provider_aws_1_29
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-29
-    - identifier: kubernetes_cloud_provider_aws_1_30
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-30
-    - identifier: kubernetes_cloud_provider_aws_1_31
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-31
     - identifier: kubernetes_cloud_provider_vsphere_1_27
       env:
         type: ARM_CONTAINER
@@ -589,10 +557,6 @@ batch:
         - kubernetes_autoscaler_1_30
         - kubernetes_autoscaler_1_31
         - kubernetes_cloud_provider_aws_1_27
-        - kubernetes_cloud_provider_aws_1_28
-        - kubernetes_cloud_provider_aws_1_29
-        - kubernetes_cloud_provider_aws_1_30
-        - kubernetes_cloud_provider_aws_1_31
         - kubernetes_cloud_provider_vsphere_1_27
         - kubernetes_cloud_provider_vsphere_1_28
         - kubernetes_cloud_provider_vsphere_1_29

release/staging-build.yml

@@ -291,38 +291,6 @@ batch:
           PROJECT_PATH: projects/kubernetes/cloud-provider-aws
           CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
           RELEASE_BRANCH: 1-27
-    - identifier: kubernetes_cloud_provider_aws_1_28
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-28
-    - identifier: kubernetes_cloud_provider_aws_1_29
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-29
-    - identifier: kubernetes_cloud_provider_aws_1_30
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-30
-    - identifier: kubernetes_cloud_provider_aws_1_31
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-31
     - identifier: kubernetes_cloud_provider_vsphere_1_27
       env:
         type: ARM_CONTAINER
@@ -508,61 +476,6 @@ batch:
           IMAGE_OS: bottlerocket
           RELEASE_BRANCH: 1-31
           IMAGE_OS_VERSION: "1"
-    - identifier: kubernetes_sigs_image_builder_bottlerocket_1_27_ami
-      buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes-sigs/image-builder
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder
-          IMAGE_OS: bottlerocket
-          RELEASE_BRANCH: 1-27
-          IMAGE_OS_VERSION: "1"
-    - identifier: kubernetes_sigs_image_builder_bottlerocket_1_28_ami
-      buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes-sigs/image-builder
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder
-          IMAGE_OS: bottlerocket
-          RELEASE_BRANCH: 1-28
-          IMAGE_OS_VERSION: "1"
-    - identifier: kubernetes_sigs_image_builder_bottlerocket_1_29_ami
-      buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes-sigs/image-builder
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder
-          IMAGE_OS: bottlerocket
-          RELEASE_BRANCH: 1-29
-          IMAGE_OS_VERSION: "1"
-    - identifier: kubernetes_sigs_image_builder_bottlerocket_1_30_ami
-      buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes-sigs/image-builder
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder
-          IMAGE_OS: bottlerocket
-          RELEASE_BRANCH: 1-30
-          IMAGE_OS_VERSION: "1"
-    - identifier: kubernetes_sigs_image_builder_bottlerocket_1_31_ami
-      buildspec: projects/kubernetes-sigs/image-builder/buildspecs/ami.yml
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes-sigs/image-builder
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes-sigs.image-builder
-          IMAGE_OS: bottlerocket
-          RELEASE_BRANCH: 1-31
-          IMAGE_OS_VERSION: "1"
     - identifier: kubernetes_sigs_kind
       buildspec: buildspecs/binaries.yml
       env:

tools/version-tracker/buildspecs/upgrade.yml

@@ -244,38 +244,6 @@ batch:
           PROJECT_PATH: projects/kubernetes/cloud-provider-aws
           CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
           RELEASE_BRANCH: 1-27
-    - identifier: kubernetes_cloud_provider_aws_1_28
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-28
-    - identifier: kubernetes_cloud_provider_aws_1_29
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-29
-    - identifier: kubernetes_cloud_provider_aws_1_30
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-30
-    - identifier: kubernetes_cloud_provider_aws_1_31
-      env:
-        type: ARM_CONTAINER
-        compute-type: BUILD_GENERAL1_SMALL
-        variables:
-          PROJECT_PATH: projects/kubernetes/cloud-provider-aws
-          CLONE_URL: https://git-codecommit.us-west-2.amazonaws.com/v1/repos/kubernetes.cloud-provider-aws
-          RELEASE_BRANCH: 1-31
     - identifier: kubernetes_cloud_provider_vsphere_1_27
       env:
         type: ARM_CONTAINER