Merge branch 'GoogleCloudPlatform:main' into tflint_provider

terraform-linters · May 25, 2024 · 89bb69f · 89bb69f
2 parents 36c355b + bf9c43f
commit 89bb69f
Show file tree

Hide file tree

Showing 673 changed files with 22,224 additions and 2,827 deletions.
diff --git a/.ci/containers/build-environment/Dockerfile b/.ci/containers/build-environment/Dockerfile
@@ -23,7 +23,7 @@ RUN mkdir -p "$GOPATH/src" "$GOPATH/bin" && chmod -R 1777 "$GOPATH"
 WORKDIR $GOPATH
 
 # terraform binary used by tfv/tgc
-COPY --from=hashicorp/terraform:1.4.2 /bin/terraform /bin/terraform
+COPY --from=hashicorp/terraform:1.8.3 /bin/terraform /bin/terraform
 
 SHELL ["/bin/bash", "-c"]
 

diff --git a/.ci/containers/go-plus/Dockerfile b/.ci/containers/go-plus/Dockerfile
@@ -29,7 +29,7 @@ RUN apt-get update && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/*
 
-RUN wget https://releases.hashicorp.com/terraform/1.2.5/terraform_1.2.5_linux_amd64.zip \
-    && unzip terraform_1.2.5_linux_amd64.zip \
-    && rm terraform_1.2.5_linux_amd64.zip \
+RUN wget https://releases.hashicorp.com/terraform/1.8.3/terraform_1.8.3_linux_amd64.zip \
+    && unzip terraform_1.8.3_linux_amd64.zip \
+    && rm terraform_1.8.3_linux_amd64.zip \
     && mv ./terraform /bin/terraform
diff --git a/.ci/gcb-push-downstream.yml b/.ci/gcb-push-downstream.yml
@@ -30,20 +30,55 @@ steps:
       waitFor: ["checkout"]
 
     # TPG
-    - name: 'gcr.io/graphite-docker-images/bash-plus'
-      entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       id: tpg-sync
-      waitFor: ["checkout"]
+      waitFor: ["build-magician-binary"]
       args:
+          - wait-for-commit
           - 'tpg-sync'
           - $BRANCH_NAME
           - $COMMIT_SHA
 
+    # TPGB
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      id: tpgb-sync
+      waitFor: ["build-magician-binary"]
+      args:
+          - wait-for-commit
+          - 'tpgb-sync'
+          - $BRANCH_NAME
+          - $COMMIT_SHA
+
+    # TGC
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      id: tgc-sync
+      waitFor: ["build-magician-binary"]
+      args:
+          - wait-for-commit
+          - 'tgc-sync'
+          - $BRANCH_NAME
+          - $COMMIT_SHA
+
+    # TF-OICS
+    - name: 'gcr.io/graphite-docker-images/go-plus'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
+      id: tf-oics-sync
+      waitFor: ["build-magician-binary"]
+      args:
+          - wait-for-commit
+          - 'tf-oics-sync'
+          - $BRANCH_NAME
+          - $COMMIT_SHA
+
+    # TPG
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tpg-push
-      waitFor: ["tpg-sync", "build-magician-binary"]
+      waitFor: ["tpg-sync", "tpgb-sync", "tgc-sync", "tf-oics-sync"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
@@ -67,20 +102,11 @@ steps:
             fi
 
     # TPGB
-    - name: 'gcr.io/graphite-docker-images/bash-plus'
-      entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
-      id: tpgb-sync
-      waitFor: ["checkout"]
-      args:
-          - 'tpgb-sync'
-          - $BRANCH_NAME
-          - $COMMIT_SHA
-
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tpgb-push
-      waitFor: ["tpgb-sync", "build-magician-binary"]
+      waitFor: ["tpg-sync", "tpgb-sync", "tgc-sync", "tf-oics-sync"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
@@ -104,20 +130,11 @@ steps:
             fi
 
     # TGC
-    - name: 'gcr.io/graphite-docker-images/bash-plus'
-      entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
-      id: tgc-sync
-      waitFor: ["checkout"]
-      args:
-          - 'tgc-sync'
-          - $BRANCH_NAME
-          - $COMMIT_SHA
-
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tgc-push
-      waitFor: ["tgc-sync", "tpgb-push"]
+      waitFor: ["tpgb-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
@@ -141,20 +158,11 @@ steps:
             fi
 
     # TF-OICS
-    - name: 'gcr.io/graphite-docker-images/bash-plus'
-      entrypoint: '/workspace/.ci/scripts/bash-plus/downstream-waiter/wait_for_commit.sh'
-      id: tf-oics-sync
-      waitFor: ["checkout"]
-      args:
-          - 'tf-oics-sync'
-          - $BRANCH_NAME
-          - $COMMIT_SHA
-
     - name: 'gcr.io/graphite-docker-images/build-environment'
       entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC"]
       id: tf-oics-push
-      waitFor: ["tf-oics-sync", "build-magician-binary"]
+      waitFor: ["tpg-sync", "tpgb-sync", "tgc-sync", "tf-oics-sync"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
@@ -178,14 +186,15 @@ steps:
             fi
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
-      entrypoint: '/workspace/.ci/scripts/go-plus/vcr-cassette-merger/vcr_merge.sh'
+      entrypoint: '/workspace/.ci/scripts/go-plus/magician/exec.sh'
       secretEnv: ["GITHUB_TOKEN_CLASSIC", "GOOGLE_PROJECT"]
       id: vcr-merge
       waitFor: ["tpg-push", "tpgb-push", "tgc-push", "tf-oics-push"]
       env:
         - BASE_BRANCH=$BRANCH_NAME
       args:
-          - $COMMIT_SHA
+        - "vcr-merge"
+        - $COMMIT_SHA
 
     - name: 'gcr.io/graphite-docker-images/go-plus'
       id: magician-check-vcr-cassettes

diff --git a/.ci/infra/terraform/main.tf b/.ci/infra/terraform/main.tf
@@ -209,11 +209,9 @@ module "project-services" {
     "cloudasset.googleapis.com",
     "cloudbilling.googleapis.com",
     "cloudbuild.googleapis.com",
-    "clouddebugger.googleapis.com",
     "clouddeploy.googleapis.com",
     "cloudfunctions.googleapis.com",
     "cloudidentity.googleapis.com",
-    "cloudiot.googleapis.com",
     "cloudkms.googleapis.com",
     "cloudquotas.googleapis.com",
     "cloudresourcemanager.googleapis.com",
@@ -314,6 +312,7 @@ module "project-services" {
     "securesourcemanager.googleapis.com",
     "securetoken.googleapis.com",
     "securitycenter.googleapis.com",
+    "securitycentermanagement.googleapis.com",
     "securityposture.googleapis.com",
     "serviceconsumermanagement.googleapis.com",
     "servicecontrol.googleapis.com",

diff --git a/.ci/magician/cmd/DIFF_COMMENT.md b/.ci/magician/cmd/DIFF_COMMENT.md
@@ -18,7 +18,7 @@ Your PR generated some diffs in downstreams - here they are.
 The following breaking change(s) were detected within your pull request.
 
 {{- range .BreakingChanges}}
-- {{.}}{{end}}
+- {{.Message}} - [reference]({{.DocumentationReference}}){{end}}
 
 If you believe this detection to be incorrect please raise the concern with your reviewer.
 If you intend to make this change you will need to wait for a [major release](https://www.terraform.io/plugin/sdkv2/best-practices/versioning#example-major-number-increments) window.
@@ -33,4 +33,4 @@ An `override-breaking-change` label can be added to allow merging.
 {{- range .Errors}}
 - {{.}}{{end}}
 {{end}}
-{{- end -}}
+{{- end -}}
diff --git a/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl b/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_CONTRIBUTOR.md.tmpl
@@ -0,0 +1,14 @@
+{{ if lt .SinceDays 30 -}}
+@{{.PullRequest.User.Login}}, this PR is waiting for action from you. Please address any comments or change requests, or [re-request review](https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/proposing-changes-to-your-work-with-pull-requests/requesting-a-pull-request-review) from a core reviewer if no action is required.
+
+![Image showing the re-request review button](https://docs.github.com/assets/cb-28785/mw-1440/images/help/pull_requests/request-re-review.webp)
+
+If no action is taken, this PR will be closed in 
+{{- if eq .SinceDays 10 }} 28 days.{{end}}
+{{- if eq .SinceDays 20 }} 14 days.{{end}}
+{{- if eq .SinceDays 28 }} 2 weekdays.{{end}}
+
+This notification can be disabled with the `disable-automatic-closure` label.
+{{ else -}}
+@{{.PullRequest.User.Login}}, this PR is being closed due to inactivity.
+{{ end -}}
diff --git a/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl b/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_MERGE.md.tmpl
@@ -0,0 +1 @@
+This PR is approved and has been waiting for merge for {{if eq .SinceDays 5}}1 week{{else}}{{weekdaysToWeeks .SinceDays}} weeks{{end}}. Is it ready to merge? Use the label `disable-review-reminders` to disable these notifications.
diff --git a/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl b/.ci/magician/cmd/SCHEDULED_PR_WAITING_FOR_REVIEW.md.tmpl
@@ -0,0 +1 @@
+{{if ge .SinceDays 5}}@GoogleCloudPlatform/terraform-team {{end}}This PR has been waiting for review for {{if lt .SinceDays 5}}{{.SinceDays}} weekdays{{else if eq .SinceDays 5}}1 week{{else}}{{weekdaysToWeeks .SinceDays}} weeks{{end}}. Please take a look! Use the label `disable-review-reminders` to disable these notifications.
diff --git a/.ci/magician/cmd/generate_comment.go b/.ci/magician/cmd/generate_comment.go
@@ -50,6 +50,11 @@ type Diff struct {
 	ShortStat string
 }
 
+type BreakingChange struct {
+	Message                string
+	DocumentationReference string
+}
+
 type Errors struct {
 	Title  string
 	Errors []string
@@ -58,7 +63,7 @@ type Errors struct {
 type diffCommentData struct {
 	PrNumber        int
 	Diffs           []Diff
-	BreakingChanges []string
+	BreakingChanges []BreakingChange
 	MissingTests    string
 	Errors          []Errors
 }
@@ -231,7 +236,7 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 
 	// The breaking changes are unique across both provider versions
 	uniqueAffectedResources := map[string]struct{}{}
-	uniqueBreakingChanges := map[string]struct{}{}
+	uniqueBreakingChanges := map[string]BreakingChange{}
 	diffProcessorPath := filepath.Join(mmLocalPath, "tools", "diff-processor")
 	diffProcessorEnv := map[string]string{
 		"OLD_REF": oldBranch,
@@ -263,7 +268,7 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 			errors[repo.Title] = append(errors[repo.Title], "The diff processor crashed while computing breaking changes. This is usually due to the downstream provider failing to compile.")
 		}
 		for _, breakingChange := range breakingChanges {
-			uniqueBreakingChanges[breakingChange] = struct{}{}
+			uniqueBreakingChanges[breakingChange.Message] = breakingChange
 		}
 
 		if repo.Name == "terraform-provider-google-beta" {
@@ -285,8 +290,10 @@ func execGenerateComment(prNumber int, ghTokenMagicModules, buildId, buildStep,
 			uniqueAffectedResources[resource] = struct{}{}
 		}
 	}
-	breakingChangesSlice := maps.Keys(uniqueBreakingChanges)
-	sort.Strings(breakingChangesSlice)
+	breakingChangesSlice := maps.Values(uniqueBreakingChanges)
+	sort.Slice(breakingChangesSlice, func(i, j int) bool {
+		return breakingChangesSlice[i].Message < breakingChangesSlice[j].Message
+	})
 	data.BreakingChanges = breakingChangesSlice
 
 	// Compute affected resources based on changed files
@@ -412,7 +419,7 @@ func buildDiffProcessor(diffProcessorPath, providerLocalPath string, env map[str
 	return rnr.PopDir()
 }
 
-func computeBreakingChanges(diffProcessorPath string, rnr ExecRunner) ([]string, error) {
+func computeBreakingChanges(diffProcessorPath string, rnr ExecRunner) ([]BreakingChange, error) {
 	if err := rnr.PushDir(diffProcessorPath); err != nil {
 		return nil, err
 	}
@@ -425,7 +432,11 @@ func computeBreakingChanges(diffProcessorPath string, rnr ExecRunner) ([]string,
 		return nil, nil
 	}
 
-	return strings.Split(strings.TrimSuffix(output, "\n"), "\n"), rnr.PopDir()
+	var changes []BreakingChange
+	if err = json.Unmarshal([]byte(output), &changes); err != nil {
+		return nil, err
+	}
+	return changes, rnr.PopDir()
 }
 
 func changedSchemaResources(diffProcessorPath string, rnr ExecRunner) ([]string, error) {

diff --git a/.ci/magician/cmd/generate_comment_test.go b/.ci/magician/cmd/generate_comment_test.go
@@ -188,17 +188,23 @@ func TestFormatDiffComment(t *testing.T) {
 		},
 		"breaking changes are displayed": {
 			data: diffCommentData{
-				BreakingChanges: []string{
-					"Breaking change 1",
-					"Breaking change 2",
+				BreakingChanges: []BreakingChange{
+					{
+						Message:                "Breaking change 1",
+						DocumentationReference: "doc1",
+					},
+					{
+						Message:                "Breaking change 2",
+						DocumentationReference: "doc2",
+					},
 				},
 			},
 			expectedStrings: []string{
 				"## Diff report",
 				"## Breaking Change(s) Detected",
 				"major release",
 				"`override-breaking-change`",
-				"- Breaking change 1\n- Breaking change 2\n",
+				"- Breaking change 1 - [reference](doc1)\n- Breaking change 2 - [reference](doc2)\n",
 			},
 			notExpectedStrings: []string{
 				"generated some diffs",
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		This PR is approved and has been waiting for merge for {{if eq .SinceDays 5}}1 week{{else}}{{weekdaysToWeeks .SinceDays}} weeks{{end}}. Is it ready to merge? Use the label `disable-review-reminders` to disable these notifications.
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		{{if ge .SinceDays 5}}@GoogleCloudPlatform/terraform-team {{end}}This PR has been waiting for review for {{if lt .SinceDays 5}}{{.SinceDays}} weekdays{{else if eq .SinceDays 5}}1 week{{else}}{{weekdaysToWeeks .SinceDays}} weeks{{end}}. Please take a look! Use the label `disable-review-reminders` to disable these notifications.