Merge branch 'develop' into wx_927_quieter

centaur/src/main/resources/standardTestCases/docker_hash/docker_hash_dockerhub_private_google_secret_manager.wdl

@@ -0,0 +1,13 @@
+task dockerhub {
+    command {
+        echo "hello"
+    }
+    runtime {
+        docker: "broadinstitute/cloud-cromwell:2024-08-31"
+        backend: "GCPBATCHGoogleSecretManager"
+    }
+}
+
+workflow docker_hash_dockerhub_private {
+    call dockerhub
+}

centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private_google_secret_manager.test

@@ -0,0 +1,18 @@
+name: docker_hash_dockerhub_private_google_secret_manager
+testFormat: workflowsuccess
+# see https://github.com/broadinstitute/cromwell/pull/7515
+backends: [GCPBATCH]
+
+files {
+  workflow: docker_hash/docker_hash_dockerhub_private_google_secret_manager.wdl
+  # Updated the options to read_from_cache: false for
+  # https://github.com/broadinstitute/cromwell/issues/3998
+  options-dir: "Error: BA-6546 The environment variable CROMWELL_BUILD_RESOURCES_DIRECTORY must be set/export pointing to a valid path such as '${YOUR_CROMWELL_DIR}/target/ci/resources'"
+  options-dir: ${?CROMWELL_BUILD_RESOURCES_DIRECTORY}
+  options: ${files.options-dir}/private_docker_papi_v2_usa.options
+}
+
+metadata {
+  "calls.docker_hash_dockerhub_private.dockerhub.runtimeAttributes.docker": "broadinstitute/cloud-cromwell:2024-08-31",
+  "calls.docker_hash_dockerhub_private.dockerhub.dockerImageUsed": "broadinstitute/cloud-cromwell@sha256:49ba8c8c2821c5b894a5bcff0a7873d6c58df7a03c45ee7c3cda159795f6e80f"
+}

centaur/src/main/resources/standardTestCases/docker_hash_dockerhub_private_wf_options.test

@@ -1,7 +1,7 @@
 name: docker_hash_dockerhub_private_wf_options
 testFormat: workflowsuccess
 # see https://github.com/broadinstitute/cromwell/pull/7515
-backends: [Papiv2NoDockerHubConfig, GCPBATCH_FAIL]
+backends: [Papiv2NoDockerHubConfig, GCPBATCH_USES_SECRET_MANAGER_NOT_KMS]
 
 files {
   workflow: docker_hash/docker_hash_dockerhub_private_no_dockerhub_config.wdl

docs/backends/GCPBatch.md

@@ -137,6 +137,33 @@ backend {
 
 `token` is the standard base64-encoded username:password for the appropriate Docker Hub account.
 
+GCP Batch also supports the use of Google Secret Manager for storing private Docker Hub credentials as described in
+Google Batch documentation
+[here](https://cloud.google.com/batch/docs/create-run-job-secret-manager#use-secrets-for-docker-registry). In the
+Cromwell GCP Batch backend, the usage of this feature is very similar to the regular
+base64-encoded `username:password` token, except that it is the GSM paths of username and password that are separated by
+a colon and base64
+encoded:
+
+```
+backend {
+  default = GCPBATCH
+  providers {
+    GCPBATCH {
+      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      config {
+        dockerhub {
+          token = "base64-encoded-GSM-path-to-docker-hub-username:GSM-path-to-docker-hub-password"
+        }
+      }
+    }
+  }
+}
+```
+
+Note that as per the Google Secret Manager docs, the compute service account for the project in which the GCP Batch
+jobs will run will need to be assigned the `Secret Manager Secret Accessor` IAM role.
+
 **Monitoring**
 
 In order to monitor metrics (CPU, Memory, Disk usage...) about the VM during Call Runtime, a workflow option can be used to specify the path to a script that will run in the background and write its output to a log file.

src/ci/resources/dockerhub_provider_config_v2_google_secret_manager_usa.inc.conf.ctmpl

@@ -0,0 +1,11 @@
+{{with $cromwellDockerhub := secret (printf "secret/dsde/cromwell/common/cromwell-dockerhub")}}
+dockerhub {
+  # GCP Batch supports both the usual base64-encoded <username>:<password> Docker token format,
+  # but also supports the use of Google Secret Manager (GSM). When using GSM the token is still base64
+  # encoded but the format becomes
+  # <Path to GSM username secret>:<Path to GSM password secret>
+  # This test is exercising the GCP Batch Google Secret Manager support.
+  token = "{{$cromwellDockerhub.Data.google_secret_manager_token}}"
+  auth = "user_service_account"
+}
+{{end}}

src/ci/resources/gcp_batch_application.conf

@@ -65,5 +65,11 @@ backend {
         include "gcp_batch_provider_config.inc.conf"
       }
     }
+    GCPBATCHGoogleSecretManager {
+      actor-factory = "cromwell.backend.google.batch.GcpBatchBackendLifecycleActorFactory"
+      config {
+        include "gcp_batch_provider_config.inc.conf"
+      }
+    }
   }
 }

src/ci/resources/gcp_batch_shared_application.inc.conf

@@ -173,5 +173,15 @@ backend {
         filesystems.http {}
       }
     }
+    GCPBATCHGoogleSecretManager {
+      actor-factory = "REPLACEME!"
+      config {
+        include "dockerhub_provider_config_v2_google_secret_manager_usa.inc.conf"
+        # This SA does not have permission to bill this project when accessing RP buckets.
+        # This is on purpose so that we can assert the failure (see requester_pays_localization_negative)
+        genomics.compute-service-account = "[email protected]"
+        filesystems.http {}
+      }
+    }
   }
 }