Use Cloud Build to create all resources

broker/cloud_run/lsst/classify_snn/cloudbuild.yaml

@@ -1,29 +1,56 @@
-# https://cloud.google.com/build/docs/deploying-builds/deploy-cloud-run
-# containerize the module and deploy it to Cloud Run
+# --------------- References ------------------ #
+# Cloud Build Overview: https://cloud.google.com/build/docs/overview
+# Deploying to Cloud Run: https://cloud.google.com/build/docs/deploying-builds/deploy-cloud-run
+# Schema for this file: https://cloud.google.com/build/docs/build-config-file-schema
+#
+# --------------- Substitutions --------------- #
+substitutions:
+  _IMAGE_NAME: 'gcr.io/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_NAME}'
+  _IMAGE_PATH: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_IMAGE_NAME}'
+  # Different GCP services use different names for the same env variable:
+  # PROJECT_ID, GOOGLE_CLOUD_PROJECT, and GCP_PROJECT.
+  # We will use GCP_PROJECT as the env variable of our deployed Cloud Run service
+  # for consistency with Cloud Functions, which sets this variable automatically.
+  _MODULE_ENV: 'GCP_PROJECT=${PROJECT_ID},SURVEY=${_SURVEY},TESTID=${_TESTID}'
+#
+# --------------- Steps ----------------------- #
 steps:
-# Build the image
+# Ancillaries: Create ancillary resources.
+- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+  id: Ancillaries
+  waitFor: ['-']
+  entrypoint: bash
+  args:
+    - '-c'
+    # Here we are just copying the needed files from the local machine.
+    # For automatic deployments from GitHub, clone the repo instead.
+    - |
+      cp create-ancillaries.sh construct-name.sh /workspace/
+      chmod +x /workspace/create-ancillaries.sh /workspace/construct-name.sh
+      /workspace/create-ancillaries.sh
+# Build: Build the image.
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['build', '-t', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}', '.']
-# Push the image to Artifact Registry
+  id: Build
+  waitFor: ['-']
+  args: ['build', '-t', '${_IMAGE_PATH}', '.']
+# Push: Push the image to the repository.
 - name: 'gcr.io/cloud-builders/docker'
-  args: ['push', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}']
-# Deploy image to Cloud Run
+  id: Push
+  waitFor: ['Build']
+  args: ['push', '${_IMAGE_PATH}']
+# Deploy: Deploy the Cloud Run service.
 - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk'
+  id: Deploy
+  waitFor: ['Push']  # [CHECKME] Does this also need to wait for Ancillaries?
   entrypoint: gcloud
-  args: ['run', 'deploy', '${_MODULE_NAME}', '--image', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}', '--region', '${_REGION}', '--set-env-vars', '${_ENV_VARS}']
+  args: ['run', 'deploy', '${_MODULE_NAME}', '--image', '${_IMAGE_PATH}', '--region', '${LOCATION}', '--set-env-vars', '${_MODULE_ENV}']
+#
+# --------------- Other ----------------------- #
 images:
-- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}'
-substitutions:
-    _SURVEY: 'lsst'
-    _TESTID: 'testid'
-    _MODULE_NAME: '${_SURVEY}-classify_with_SuperNNova-${_TESTID}'
-    _MODULE_IMAGE_NAME: 'gcr.io/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_NAME}'
-    _REPOSITORY: 'cloud-run-services'
-    # cloud functions automatically sets the projectid env var using the name "GCP_PROJECT"
-    # use the same name here for consistency
-    # [TODO] PROJECT_ID is set in setup.sh. this is confusing and we should revisit the decision.
-    # i (Raen) think i didn't make it a substitution because i didn't want to set a default for it.
-    _ENV_VARS: 'GCP_PROJECT=${PROJECT_ID},SURVEY=${_SURVEY},TESTID=${_TESTID}'
-    _REGION: 'us-central1'
+- '${_IMAGE_PATH}'
 options:
-    dynamic_substitutions: true
+  # Include all built-in and custom substitutions as env variables for all build steps.
+  automapSubstitutions: true
+  # Within user-defined substitutions, allow referencing of other variables and bash parameter expansion.
+  # https://cloud.google.com/build/docs/configuring-builds/use-bash-and-bindings-in-substitutions#bash_parameter_expansions
+  dynamic_substitutions: true

broker/cloud_run/lsst/classify_snn/construct-name.sh

@@ -0,0 +1,80 @@
+#!/bin/bash
+
+_info() {
+    echo "Use '$(basename "$0") --help' for more information."
+}
+
+_help() {
+    echo "Construct the GCP resource name using the supplied options and the env vars SURVEY and TESTID."
+    echo
+    echo "Usage: $0 [-s|--stem] <stem> [[-g|--gcp-service] <gcp_service>]"
+    echo
+    echo "Options:"
+    echo "  -s, --stem <stem>       Name stem for the resource. SURVEY will be prepended and TESTID "
+    echo "                          appened (if not false)."
+    echo "  -g, --gcp-service <gcp_service>"
+    echo "                          Determines the separator. If the value is 'bigquery', the"
+    echo "                          separator will be '_'. Otherwise it is '-'."
+    echo
+    echo "Environment Variables:"
+    echo "  SURVEY                  Required. Prepend to resource name."
+    echo "  TESTID                  Required. Append to resource name if not 'False'."
+}
+
+# Ensure that all required environment variables are set.
+check_env_vars() {
+  local vars=("$@")
+  for var in "${vars[@]}"; do
+    if [ -z "${!var}" ]; then
+      echo "Error: ${var} environment variable is not set."
+      exit 1
+    fi
+  done
+}
+check_env_vars SURVEY TESTID
+
+stem=""
+gcp_service=""
+
+while [[ $# -gt 0 ]]; do
+  key="$1"
+  case $key in
+    -s|--stem)
+      stem="$2"
+      shift
+      shift
+      ;;
+    -g|--gcp-service)
+      gcp_service="$(echo "$2" | tr '[:upper:]' '[:lower:]')"
+      shift
+      shift
+      ;;
+    -h|--help)
+      _help
+      exit 0
+      ;;
+    *)
+      echo "Invalid option: $1"
+      _info
+      exit 1
+      ;;
+  esac
+done
+
+if [ -z "$stem" ]; then
+    echo "Missing required option 'stem'."
+    _info
+    exit 1
+fi
+
+_sep="-"
+if [ "$gcp_service" = "bigquery" ]; then
+    _sep="_"
+fi
+
+_testid="${_sep}${TESTID}"
+if [ "$TESTID" = "False" ] || [ "$TESTID" = "false" ]; then
+    _testid=""
+fi
+
+echo "${SURVEY}${_sep}${stem}${_testid}"

broker/cloud_run/lsst/classify_snn/create-ancillaries.sh

@@ -0,0 +1,28 @@
+#! /bin/bash
+# Create ancillary resources that are needed by the Cloud Run service.
+# This script is intended to be run by Cloud Build.
+
+# Define resource names.
+# BigQuery
+bq_dataset=$(construct-name.sh --stem "$_SURVEY" --gcp-resource bigquery)
+bq_table_supernnova="supernnova"
+# Pub/Sub
+ps_topic_out=$(construct-name.sh --stem "supernnova")
+ps_topic_bqimport=$(construct-name.sh --stem "bigquery-import-supernnova")
+ps_topic_bqimport_deadletter=$(construct-name.sh --stem "bigquery-import-supernnova-deadletter")
+ps_subscrip_bqimport="$ps_topic_bqimport"
+ps_subscrip_bqimport_deadletter="$ps_topic_bqimport_deadletter"
+
+# Create the resources.
+gcloud pubsub topics create "${ps_topic_out}"
+gcloud pubsub topics create "${ps_topic_bqimport}"
+gcloud pubsub topics create "${ps_topic_bqimport_deadletter}"
+gcloud pubsub subscriptions create "${ps_subscrip_bqimport_deadletter}" --topic="${ps_topic_bqimport_deadletter}"
+# [FIXME] This assumes that the BigQuery dataset and table already exist.
+gcloud pubsub subscriptions create "${ps_subscrip_bqimport}" \
+    --topic="${ps_topic_bqimport}" \
+    --bigquery-table="${PROJECT_ID}:${bq_dataset}.${bq_table_supernnova}" \
+    --use-table-schema \
+    --dead-letter-topic="${ps_topic_bqimport_deadletter}" \
+    --max-delivery-attempts=5 \
+    --dead-letter-topic-project="${PROJECT_ID}"

broker/cloud_run/lsst/classify_snn/deploy.sh

@@ -1,122 +1,79 @@
 #! /bin/bash
-# Deploys or deletes broker Cloud Run service
-# This script will not delete a Cloud Run service that is in production
+# Build the image, create ancillary resources, and deploy the module as a Cloud Run service.
+#
+# --------- Example usage -----------------------
+#
+# First, double check the values in env.yaml. Then:
+#
+# $ gcloud auth ...
+# $ export PROJECT_ID=... (Is this set automatically by gcloud auth?)
+# $ bash deploy.sh  # That's it. All variables retrieved from env.yaml.
+#
+# -----------------------------------------------
 
-# "False" uses production resources
-# any other string will be appended to the names of all resources
-testid="${1:-test}"
-# "True" tearsdown/deletes resources, else setup
-teardown="${2:-False}"
-# name of the survey this broker instance will ingest
-survey="${3:-lsst}"
-region="${4:-us-central1}"
-# get environment variables
-PROJECT_ID=$GOOGLE_CLOUD_PROJECT
-PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)")
+# --------- Set environment variables -----------
+# Load env.yaml and set the key/value pairs as environment variables.
+# [FIXME] This depends on yq. We need to provide instructions for installing it
+#         or else just have the user export these manually.
+while IFS='=' read -r key value; do
+  export "$key=$value"
+done < <(yq -r 'to_entries | .[] | .key + "=" + .value' env.yaml)
 
-MODULE_NAME="supernnova"  # lower case required by cloud run
-ROUTE_RUN="/"  # url route that will trigger main.run()
-
-# function used to define GCP resources; appends testid if needed
-define_GCP_resources() {
-    local base_name="$1"
-    local testid_suffix=""
-
-    if [ "$testid" != "False" ]; then
-        if [ "$base_name" = "$survey" ]; then
-            testid_suffix="_${testid}"  # complies with BigQuery naming conventions
-        else
-            testid_suffix="-${testid}"
-        fi
+# Ensure that all required environment variables are set.
+check_env_vars() {
+  local vars=("$@")
+  for var in "${vars[@]}"; do
+    if [ -z "${!var}" ]; then
+      echo "Error: ${var} environment variable is not set."
+      exit 1
     fi
-
-    echo "${base_name}${testid_suffix}"
+    export "_${var}="
+  done
 }
+check_env_vars PROJECT_ID _SURVEY _TESTID MODULE_NAME_STEM MODULE_ROUTE REGION REPOSITORY_STEM TRIGGER_TOPIC_STEM
 
-#--- GCP resources used in this script
-artifact_registry_repo=$(define_GCP_resources "${survey}-cloud-run-services")
-deadletter_topic_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova-deadletter")
-deadletter_subscription_bigquery_import="${deadletter_topic_bigquery_import}"
-ps_input_subscrip=$(define_GCP_resources "${survey}-alerts") # pub/sub subscription used to trigger cloud run module
-ps_output_topic=$(define_GCP_resources "${survey}-SuperNNova")
-subscription_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova") # BigQuery subscription
-topic_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova")
-trigger_topic=$(define_GCP_resources "${survey}-alerts")
+# Construct and export additional environment variables for cloudbuild.yaml.
+# Environment variables that will be used by cloudbuild.yaml must start with "_", per GCP's requirements.
+_MODULE_NAME=$(construct-name.sh --stem "$MODULE_NAME_STEM")
+export _MODULE_NAME="$_MODULE_NAME"
+_REPOSITORY=$(construct-name.sh --stem "$REPOSITORY_STEM")
+export _REPOSITORY="$_REPOSITORY"
+_TRIGGER_TOPIC=$(construct-name.sh --stem "$TRIGGER_TOPIC_STEM")
+export _TRIGGER_TOPIC="$_TRIGGER_TOPIC"
+# -----------------------------------------------
 
-# additional GCP resources & variables used in this script
-bq_dataset=$(define_GCP_resources "${survey}")
-supernnova_classifications_table="SuperNNova"
-cr_module_name=$(define_GCP_resources "${survey}-${MODULE_NAME}")  # lower case required by Cloud Run
+# --------- Project setup -----------------------
+# [FIXME] This is a project setup task, so should be moved to a script dedicated to that.
+# Ensure the Cloud Run service has the necessary permissions.
 runinvoker_svcact="cloud-run-invoker@${PROJECT_ID}.iam.gserviceaccount.com"
+gcloud run services add-iam-policy-binding "${_MODULE_NAME}" \
+    --member="serviceAccount:${runinvoker_svcact}" \
+    --role="roles/run.invoker"
+# -----------------------------------------------
 
+# --------- Build -------------------------------
+# Execute the build steps.
+echo "Executing cloudbuild.yaml..."
+moduledir=$(dirname "$(readlink -f "$0")")  # Absolute path to the parent directory of this script.
+url=$(gcloud builds submit \
+    --config="${moduledir}/cloudbuild.yaml" \
+    --region="${REGION}" \
+    "${moduledir}" | sed -n 's/^Step #2: Service URL: \(.*\)$/\1/p'
+)
+# -----------------------------------------------
 
-if [ "${teardown}" = "True" ]; then
-    # ensure that we do not teardown production resources
-    if [ "${testid}" != "False" ]; then
-        gcloud pubsub topics delete "${ps_output_topic}"
-        gcloud pubsub topics delete "${topic_bigquery_import}"
-        gcloud pubsub topics delete "${deadletter_topic_bigquery_import}"
-        gcloud pubsub subscriptions delete "${ps_input_subscrip}"
-        gcloud pubsub subscriptions delete "${subscription_bigquery_import}"
-        gcloud pubsub subscriptions delete "${deadletter_subscription_bigquery_import}"
-        gcloud run services delete "${cr_module_name}" --region "${region}"
-    fi
-
-else # Deploy the Cloud Run service
-
-#--- Deploy Cloud Run service
-    echo "Configuring Pub/Sub resources for classify_snn Cloud Run service..."
-    gcloud pubsub topics create "${ps_output_topic}"
-    gcloud pubsub topics create "${topic_bigquery_import}"
-    gcloud pubsub topics create "${deadletter_topic_bigquery_import}"
-    gcloud pubsub subscriptions create "${deadletter_subscription_bigquery_import}" --topic="${deadletter_topic_bigquery_import}"
-    # in order to create BigQuery subscriptions, ensure that the following service account:
-    # service-<project number>@gcp-sa-pubsub.iam.gserviceaccount.com" has the
-    # bigquery.dataEditor role for each table
-    PUBSUB_SERVICE_ACCOUNT="service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com"
-    roleid="roles/bigquery.dataEditor"
-    bq add-iam-policy-binding \
-        --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}" \
-        --role="${roleid}" \
-        --table=true "${PROJECT_ID}:${bq_dataset}.${supernnova_classifications_table}"
-    gcloud pubsub subscriptions create "${subscription_bigquery_import}" \
-        --topic="${topic_bigquery_import}" \
-        --bigquery-table="${PROJECT_ID}:${bq_dataset}.${supernnova_classifications_table}" \
-        --use-table-schema \
-        --dead-letter-topic="${deadletter_topic_bigquery_import}" \
-        --max-delivery-attempts=5 \
-        --dead-letter-topic-project="${PROJECT_ID}"
-
-    # this allows dead-lettered messages to be forwarded from the BigQuery subscription to the dead letter topic
-    # and it allows dead-lettered messages to be published to the dead letter topic.
-    gcloud pubsub topics add-iam-policy-binding "${deadletter_topic_bigquery_import}" \
-        --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
-        --role="roles/pubsub.publisher"
-    gcloud pubsub subscriptions add-iam-policy-binding "${subscription_bigquery_import}" \
-        --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\
-        --role="roles/pubsub.subscriber"
-
-    echo "Creating container image and deploying to Cloud Run..."
-    moduledir="."  # deploys what's in our current directory
-    config="${moduledir}/cloudbuild.yaml"
-    url=$(gcloud builds submit --config="${config}" \
-        --substitutions="_SURVEY=${survey},_TESTID=${testid},_MODULE_NAME=${cr_module_name},_REPOSITORY=${artifact_registry_repo}" \
-        --region="${region}" \
-        "${moduledir}" | sed -n 's/^Step #2: Service URL: \(.*\)$/\1/p')
-
-    # ensure the Cloud Run service has the necessary permisions
-    role="roles/run.invoker"
-    gcloud run services add-iam-policy-binding "${cr_module_name}" \
-        --member="serviceAccount:${runinvoker_svcact}" \
-        --role="${role}"
-
-    echo "Creating trigger subscription for Cloud Run..."
-    # WARNING:  This is set to retry failed deliveries. If there is a bug in main.py this will
-    # retry indefinitely, until the message is delete manually.
-    gcloud pubsub subscriptions create "${ps_input_subscrip}" \
-        --topic "${trigger_topic}" \
-        --topic-project "${PROJECT_ID}" \
-        --ack-deadline=600 \
-        --push-endpoint="${url}${ROUTE_RUN}" \
-        --push-auth-service-account="${runinvoker_svcact}"
-fi
+# --------- Finish build ------------------------
+# [FIXME] Figure out how to include this in cloudbuild.yaml. It is here because we need the value of $url.
+# Create the subscription that will trigger the Cloud Run service.
+echo "Creating trigger subscription for Cloud Run..."
+# [FIXME] Handle these retries better.
+echo "WARNING: This is set to retry failed deliveries. If there is a bug in main.py this will"
+echo "         retry indefinitely, until the message is delete manually."
+trigger_subscrip="$_TRIGGER_TOPIC"
+gcloud pubsub subscriptions create "${trigger_subscrip}" \
+    --topic "${_TRIGGER_TOPIC}" \
+    --topic-project "${PROJECT_ID}" \
+    --ack-deadline=600 \
+    --push-endpoint="${url}${MODULE_ROUTE}" \
+    --push-auth-service-account="${runinvoker_svcact}"
+# -----------------------------------------------

broker/cloud_run/lsst/classify_snn/env.yaml

@@ -0,0 +1,8 @@
+# Environment variables that will be used by cloudbuild.yaml must start with "_", per GCP's requirements.
+_TESTID: 'testid'
+_SURVEY: 'lsst'
+MODULE_NAME_STEM: 'supernnova'
+MODULE_ROUTE: '/'  # url route that will trigger main.run()
+REGION: 'us-central1'
+REPOSITORY_STEM: 'cloud-run-services'
+TRIGGER_TOPIC_STEM: 'alerts'