feat: new version of projects-data-source based on AssetInventory ds

Author: averbuks

modules/projects-data-source/README.md

@@ -1,9 +1,14 @@
 # Projects Data Source Module 
 
-This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects and folders under a specific `parent` recursively. 
+This module extends functionality of [google_projects](https://registry.terraform.io/providers/hashicorp/google/latest/docs/data-sources/projects) data source by retrieving all the projects under a specific `parent` recursively with only one API call against [Cloud Asset Inventory](https://cloud.google.com/asset-inventory) service. 
 
 A good usage pattern would be when we want all the projects under a specific folder (including nested subfolders) to be included into [VPC Service Controls](../vpc-sc/). Instead of manually maintaining the list of project numbers as an input to the `vpc-sc` module we can use that module to retrieve all the project numbers dynamically.
 
+### IAM Permissions required
+
+- `roles/cloudasset.viewer` on the `parent` level or above
+
+
 ## Examples
 
 ### All projects in my org
@@ -15,11 +20,7 @@ module "my-org" {
 }
 
 output "projects" {
-  value = module.my-org.projects
-}
-
-output "folders" {
-  value = module.my-org.folders
+  value = module.my-org.projects_numbers
 }
 
 # tftest skip (uses data sources)
@@ -31,34 +32,65 @@ output "folders" {
 module "my-dev" {
   source = "./fabric/modules/projects-data-source"
   parent = "folders/123456789"
-  filter = "labels.env:DEV lifecycleState:ACTIVE"   
+  query = "labels.env:DEV state:ACTIVE"
 }
 
 output "dev-projects" {
   value = module.my-dev.projects
 }
 
-output "dev-folders" {
-  value = module.my-dev.folders
+# tftest skip (uses data sources)
+```
+
+### Projects under org with folder/project exclusions
+```hcl
+module "my-filtered" {
+  source = "./fabric/modules/projects-data-source"
+  parent     = "organizations/123456789"
+  ignore_projects = [
+    "sandbox-*",       # wildcard ignore
+    "project-full-id", # specific project id
+    "0123456789"       # specific project number
+  ]
+
+  include_projects = [
+    "sandbox-114",  # include specific project which was excluded by wildcard
+    "415216609246"  # include specific project which was excluded by wildcard (by project number)
+  ]
+
+  ignore_folders = [  # subfolders are ingoner as well
+    "343991594985", 
+    "437102807785",
+    "345245235245"
+  ]
+  query = "state:ACTIVE"
+}
+
+output "filtered-projects" {
+  value = module.my-filtered.projects
 }
 
 # tftest skip (uses data sources)
+
 ```
+
 <!-- BEGIN TFDOC -->
 
 ## Variables
 
 | name | description | type | required | default |
 |---|---|:---:|:---:|:---:|
-| [parent](variables.tf#L23) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ |  |
-| [filter](variables.tf#L17) | A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters). | <code>string</code> |  | <code>&#34;lifecycleState:ACTIVE&#34;</code> |
+| [parent](variables.tf#L17) | Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format. | <code>string</code> | ✓ |  |
+| [ignore_folders](variables.tf#L58) | A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
+| [ignore_projects](variables.tf#L32) | A list of project IDs, numbers or prefixes to exclude matching projects from the module output. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
+| [include_projects](variables.tf#L44) | A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries. | <code>list&#40;string&#41;</code> |  | <code>&#91;&#93;</code> |
+| [query](variables.tf#L26) | A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax). | <code>string</code> |  | <code>&#34;state:ACTIVE&#34;</code> |
 
 ## Outputs
 
 | name | description | sensitive |
 |---|---|:---:|
-| [folders](outputs.tf#L17) | Map of folders attributes keyed by folder id. |  |
-| [project_numbers](outputs.tf#L22) | List of project numbers. |  |
-| [projects](outputs.tf#L27) | Map of projects attributes keyed by projects id. |  |
+| [project_numbers](outputs.tf#L17) | List of project numbers. |  |
+| [projects](outputs.tf#L22) | List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format. |  |
 
 <!-- END TFDOC -->

modules/projects-data-source/main.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -15,129 +15,27 @@
  */
 
 locals {
-  folders_l1_map = { for item in data.google_folders.folders_l1.folders : item.name => item }
-
-  folders_l2_map = merge([
-    for _, v in data.google_folders.folders_l2 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l3_map = merge([
-    for _, v in data.google_folders.folders_l3 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l4_map = merge([
-    for _, v in data.google_folders.folders_l4 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l5_map = merge([
-    for _, v in data.google_folders.folders_l5 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l6_map = merge([
-    for _, v in data.google_folders.folders_l6 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l7_map = merge([
-    for _, v in data.google_folders.folders_l7 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l8_map = merge([
-    for _, v in data.google_folders.folders_l8 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l9_map = merge([
-    for _, v in data.google_folders.folders_l9 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  folders_l10_map = merge([
-    for _, v in data.google_folders.folders_l10 :
-    { for item in v.folders : item.name => item }
-  ]...)
-
-  all_folders = merge(
-    local.folders_l1_map,
-    local.folders_l2_map,
-    local.folders_l3_map,
-    local.folders_l4_map,
-    local.folders_l5_map,
-    local.folders_l6_map,
-    local.folders_l7_map,
-    local.folders_l8_map,
-    local.folders_l9_map,
-    local.folders_l10_map
+  _ignore_folder_numbers = [for folder_id in var.ignore_folders: trimprefix(folder_id, "folders/")]
+  _ignore_folders_query = join(" AND NOT folders:", concat([""], local._ignore_folder_numbers))
+  query = var.query != "" ? (
+    format("%s%s", var.query, local._ignore_folders_query)
+  ) : (
+    format("%s%s", var.query, trimprefix(local._ignore_folders_query, " AND "))
   )
 
-  parent_ids = toset(concat(
-    [split("/", var.parent)[1]],
-    [for k, _ in local.all_folders : split("/", k)[1]]
-  ))
-
-  projects = merge([
-    for _, v in data.google_projects.projects :
-    { for item in v.projects : item.project_id => item }
-  ]...)
-}
-
-# 10 datasources are used to cover 10 possible nested layers in GCP organization hirerarcy. 
-data "google_folders" "folders_l1" {
-  parent_id = var.parent
-}
-
-data "google_folders" "folders_l2" {
-  for_each  = local.folders_l1_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l3" {
-  for_each  = local.folders_l2_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l4" {
-  for_each  = local.folders_l3_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l5" {
-  for_each  = local.folders_l4_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l6" {
-  for_each  = local.folders_l5_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l7" {
-  for_each  = local.folders_l6_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l8" {
-  for_each  = local.folders_l7_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l9" {
-  for_each  = local.folders_l8_map
-  parent_id = each.value.name
-}
-
-data "google_folders" "folders_l10" {
-  for_each  = local.folders_l9_map
-  parent_id = each.value.name
-}
-
-# Getting all projects parented by any of the folders in the tree including root prg/folder provided by `parent` variable.
-data "google_projects" "projects" {
-  for_each = local.parent_ids
-  filter   = "parent.id:${each.value} ${var.filter}"
+  ignore_patterns = [for item in var.ignore_projects: "^${replace(item, "*", ".*")}$"]
+  ignore_regexp   = length(local.ignore_patterns) > 0 ? join("|", local.ignore_patterns) : "^NO_PROJECTS_TO_IGNORE$"
+  projects_after_ignore = [ for item in data.google_cloud_asset_resources_search_all.projects.results : item if (
+      length(concat(try(regexall(local.ignore_regexp, trimprefix(item.project, "projects/")), []), try(regexall(local.ignore_regexp, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")), []))) == 0
+    ) || contains(var.include_projects, trimprefix(item.name, "//cloudresourcemanager.googleapis.com/projects/")) || contains(var.include_projects, trimprefix(item.project, "projects/"))
+  ]
+}
+
+data google_cloud_asset_resources_search_all projects {
+  provider = google-beta
+  scope = var.parent
+  asset_types = [
+    "cloudresourcemanager.googleapis.com/Project"
+  ]
+  query = local.query
 }

modules/projects-data-source/outputs.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,17 +14,12 @@
  * limitations under the License.
  */
 
-output "folders" {
-  description = "Map of folders attributes keyed by folder id."
-  value       = local.all_folders
-}
-
 output "project_numbers" {
   description = "List of project numbers."
-  value       = [for _, v in local.projects : v.number]
+  value       = [for item in local.projects_after_ignore : trimprefix(item.project, "projects/")]
 }
 
 output "projects" {
-  description = "Map of projects attributes keyed by projects id."
-  value       = local.projects
+  description = "List of projects in [StandardResourceMetadata](https://cloud.google.com/asset-inventory/docs/reference/rest/v1p1beta1/resources/searchAll#StandardResourceMetadata) format."
+  value       = local.projects_after_ignore
 }

modules/projects-data-source/variables.tf

@@ -1,5 +1,5 @@
 /**
- * Copyright 2022 Google LLC
+ * Copyright 2023 Google LLC
  *
  * Licensed under the Apache License, Version 2.0 (the "License");
  * you may not use this file except in compliance with the License.
@@ -14,12 +14,6 @@
  * limitations under the License.
  */
 
-variable "filter" {
-  description = "A string filter as defined in the [REST API](https://cloud.google.com/resource-manager/reference/rest/v1/projects/list#query-parameters)."
-  type        = string
-  default     = "lifecycleState:ACTIVE"
-}
-
 variable "parent" {
   description = "Parent folder or organization in 'folders/folder_id' or 'organizations/org_id' format."
   type        = string
@@ -28,3 +22,47 @@ variable "parent" {
     error_message = "Parent must be of the form folders/folder_id or organizations/organization_id."
   }
 }
+
+variable "query" {
+  description = "A string query as defined in the [Query Syntax](https://cloud.google.com/asset-inventory/docs/query-syntax)."
+  type        = string
+  default     = "state:ACTIVE"
+}
+
+variable "ignore_projects" {
+  description = "A list of project IDs, numbers or prefixes to exclude matching projects from the module output."
+  type        = list(string)
+  default     = []
+  # example
+  #ignore_projects = [
+  #  "dev-proj-1",
+  #  "uat-proj-2",
+  #  "0123456789",
+  #  "prd-proj-*"
+  #]
+}
+variable "include_projects" {
+  description = "A list of project IDs/numbers to include to the output if some of them are excluded by `ignore_projects` wilcard entries."
+  type        = list(string)
+  default     = []
+  # example excluding all the projects starting with "prf-" except "prd-123457"
+  #ignore_projects = [
+  #  "prd-*"
+  #]
+  #include_projects = [
+  #  "prd-123457",
+  #  "0123456789"
+  #]
+}
+
+variable "ignore_folders" {
+  description = "A list of folder IDs or numbers to be excluded from the output, all the subfolders and projects are exluded from the output regardless of the include_projects variable."
+  type        = list(string)
+  default     = []
+  # example exlusing a folder
+  # ignore_folders = [
+  #   "folders/0123456789",
+  #   "2345678901"
+  # ]
+}
+

modules/projects-data-source/versions.tf

@@ -1,4 +1,4 @@
-# Copyright 2022 Google LLC
+# Copyright 2023 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.