diff --git a/broker/cloud_run/lsst/classify_snn/cloudbuild.yaml b/broker/cloud_run/lsst/classify_snn/cloudbuild.yaml index d06b7760..6293221f 100644 --- a/broker/cloud_run/lsst/classify_snn/cloudbuild.yaml +++ b/broker/cloud_run/lsst/classify_snn/cloudbuild.yaml @@ -1,29 +1,56 @@ -# https://cloud.google.com/build/docs/deploying-builds/deploy-cloud-run -# containerize the module and deploy it to Cloud Run +# --------------- References ------------------ # +# Cloud Build Overview: https://cloud.google.com/build/docs/overview +# Deploying to Cloud Run: https://cloud.google.com/build/docs/deploying-builds/deploy-cloud-run +# Schema for this file: https://cloud.google.com/build/docs/build-config-file-schema +# +# --------------- Substitutions --------------- # +substitutions: + _IMAGE_NAME: 'gcr.io/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_NAME}' + _IMAGE_PATH: '${LOCATION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_IMAGE_NAME}' + # Different GCP services use different names for the same env variable: + # PROJECT_ID, GOOGLE_CLOUD_PROJECT, and GCP_PROJECT. + # We will use GCP_PROJECT as the env variable of our deployed Cloud Run service + # for consistency with Cloud Functions, which sets this variable automatically. + _MODULE_ENV: 'GCP_PROJECT=${PROJECT_ID},SURVEY=${_SURVEY},TESTID=${_TESTID}' +# +# --------------- Steps ----------------------- # steps: -# Build the image +# Ancillaries: Create ancillary resources. +- name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: Ancillaries + waitFor: ['-'] + entrypoint: bash + args: + - '-c' + # Here we are just copying the needed files from the local machine. + # For automatic deployments from GitHub, clone the repo instead. + - | + cp create-ancillaries.sh construct-name.sh /workspace/ + chmod +x /workspace/create-ancillaries.sh /workspace/construct-name.sh + /workspace/create-ancillaries.sh +# Build: Build the image. - name: 'gcr.io/cloud-builders/docker' - args: ['build', '-t', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}', '.'] -# Push the image to Artifact Registry + id: Build + waitFor: ['-'] + args: ['build', '-t', '${_IMAGE_PATH}', '.'] +# Push: Push the image to the repository. - name: 'gcr.io/cloud-builders/docker' - args: ['push', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}'] -# Deploy image to Cloud Run + id: Push + waitFor: ['Build'] + args: ['push', '${_IMAGE_PATH}'] +# Deploy: Deploy the Cloud Run service. - name: 'gcr.io/google.com/cloudsdktool/cloud-sdk' + id: Deploy + waitFor: ['Push'] # [CHECKME] Does this also need to wait for Ancillaries? entrypoint: gcloud - args: ['run', 'deploy', '${_MODULE_NAME}', '--image', '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}', '--region', '${_REGION}', '--set-env-vars', '${_ENV_VARS}'] + args: ['run', 'deploy', '${_MODULE_NAME}', '--image', '${_IMAGE_PATH}', '--region', '${LOCATION}', '--set-env-vars', '${_MODULE_ENV}'] +# +# --------------- Other ----------------------- # images: -- '${_REGION}-docker.pkg.dev/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_IMAGE_NAME}' -substitutions: - _SURVEY: 'lsst' - _TESTID: 'testid' - _MODULE_NAME: '${_SURVEY}-classify_with_SuperNNova-${_TESTID}' - _MODULE_IMAGE_NAME: 'gcr.io/${PROJECT_ID}/${_REPOSITORY}/${_MODULE_NAME}' - _REPOSITORY: 'cloud-run-services' - # cloud functions automatically sets the projectid env var using the name "GCP_PROJECT" - # use the same name here for consistency - # [TODO] PROJECT_ID is set in setup.sh. this is confusing and we should revisit the decision. - # i (Raen) think i didn't make it a substitution because i didn't want to set a default for it. - _ENV_VARS: 'GCP_PROJECT=${PROJECT_ID},SURVEY=${_SURVEY},TESTID=${_TESTID}' - _REGION: 'us-central1' +- '${_IMAGE_PATH}' options: - dynamic_substitutions: true + # Include all built-in and custom substitutions as env variables for all build steps. + automapSubstitutions: true + # Within user-defined substitutions, allow referencing of other variables and bash parameter expansion. + # https://cloud.google.com/build/docs/configuring-builds/use-bash-and-bindings-in-substitutions#bash_parameter_expansions + dynamic_substitutions: true diff --git a/broker/cloud_run/lsst/classify_snn/construct-name.sh b/broker/cloud_run/lsst/classify_snn/construct-name.sh new file mode 100644 index 00000000..81d130c2 --- /dev/null +++ b/broker/cloud_run/lsst/classify_snn/construct-name.sh @@ -0,0 +1,80 @@ +#!/bin/bash + +_info() { + echo "Use '$(basename "$0") --help' for more information." +} + +_help() { + echo "Construct the GCP resource name using the supplied options and the env vars SURVEY and TESTID." + echo + echo "Usage: $0 [-s|--stem] [[-g|--gcp-service] ]" + echo + echo "Options:" + echo " -s, --stem Name stem for the resource. SURVEY will be prepended and TESTID " + echo " appened (if not false)." + echo " -g, --gcp-service " + echo " Determines the separator. If the value is 'bigquery', the" + echo " separator will be '_'. Otherwise it is '-'." + echo + echo "Environment Variables:" + echo " SURVEY Required. Prepend to resource name." + echo " TESTID Required. Append to resource name if not 'False'." +} + +# Ensure that all required environment variables are set. +check_env_vars() { + local vars=("$@") + for var in "${vars[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: ${var} environment variable is not set." + exit 1 + fi + done +} +check_env_vars SURVEY TESTID + +stem="" +gcp_service="" + +while [[ $# -gt 0 ]]; do + key="$1" + case $key in + -s|--stem) + stem="$2" + shift + shift + ;; + -g|--gcp-service) + gcp_service="$(echo "$2" | tr '[:upper:]' '[:lower:]')" + shift + shift + ;; + -h|--help) + _help + exit 0 + ;; + *) + echo "Invalid option: $1" + _info + exit 1 + ;; + esac +done + +if [ -z "$stem" ]; then + echo "Missing required option 'stem'." + _info + exit 1 +fi + +_sep="-" +if [ "$gcp_service" = "bigquery" ]; then + _sep="_" +fi + +_testid="${_sep}${TESTID}" +if [ "$TESTID" = "False" ] || [ "$TESTID" = "false" ]; then + _testid="" +fi + +echo "${SURVEY}${_sep}${stem}${_testid}" diff --git a/broker/cloud_run/lsst/classify_snn/create-ancillaries.sh b/broker/cloud_run/lsst/classify_snn/create-ancillaries.sh new file mode 100644 index 00000000..7edc8caa --- /dev/null +++ b/broker/cloud_run/lsst/classify_snn/create-ancillaries.sh @@ -0,0 +1,28 @@ +#! /bin/bash +# Create ancillary resources that are needed by the Cloud Run service. +# This script is intended to be run by Cloud Build. + +# Define resource names. +# BigQuery +bq_dataset=$(construct-name.sh --stem "$_SURVEY" --gcp-resource bigquery) +bq_table_supernnova="supernnova" +# Pub/Sub +ps_topic_out=$(construct-name.sh --stem "supernnova") +ps_topic_bqimport=$(construct-name.sh --stem "bigquery-import-supernnova") +ps_topic_bqimport_deadletter=$(construct-name.sh --stem "bigquery-import-supernnova-deadletter") +ps_subscrip_bqimport="$ps_topic_bqimport" +ps_subscrip_bqimport_deadletter="$ps_topic_bqimport_deadletter" + +# Create the resources. +gcloud pubsub topics create "${ps_topic_out}" +gcloud pubsub topics create "${ps_topic_bqimport}" +gcloud pubsub topics create "${ps_topic_bqimport_deadletter}" +gcloud pubsub subscriptions create "${ps_subscrip_bqimport_deadletter}" --topic="${ps_topic_bqimport_deadletter}" +# [FIXME] This assumes that the BigQuery dataset and table already exist. +gcloud pubsub subscriptions create "${ps_subscrip_bqimport}" \ + --topic="${ps_topic_bqimport}" \ + --bigquery-table="${PROJECT_ID}:${bq_dataset}.${bq_table_supernnova}" \ + --use-table-schema \ + --dead-letter-topic="${ps_topic_bqimport_deadletter}" \ + --max-delivery-attempts=5 \ + --dead-letter-topic-project="${PROJECT_ID}" diff --git a/broker/cloud_run/lsst/classify_snn/deploy.sh b/broker/cloud_run/lsst/classify_snn/deploy.sh index a1cc8b82..6795f571 100755 --- a/broker/cloud_run/lsst/classify_snn/deploy.sh +++ b/broker/cloud_run/lsst/classify_snn/deploy.sh @@ -1,122 +1,79 @@ #! /bin/bash -# Deploys or deletes broker Cloud Run service -# This script will not delete a Cloud Run service that is in production +# Build the image, create ancillary resources, and deploy the module as a Cloud Run service. +# +# --------- Example usage ----------------------- +# +# First, double check the values in env.yaml. Then: +# +# $ gcloud auth ... +# $ export PROJECT_ID=... (Is this set automatically by gcloud auth?) +# $ bash deploy.sh # That's it. All variables retrieved from env.yaml. +# +# ----------------------------------------------- -# "False" uses production resources -# any other string will be appended to the names of all resources -testid="${1:-test}" -# "True" tearsdown/deletes resources, else setup -teardown="${2:-False}" -# name of the survey this broker instance will ingest -survey="${3:-lsst}" -region="${4:-us-central1}" -# get environment variables -PROJECT_ID=$GOOGLE_CLOUD_PROJECT -PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)") +# --------- Set environment variables ----------- +# Load env.yaml and set the key/value pairs as environment variables. +# [FIXME] This depends on yq. We need to provide instructions for installing it +# or else just have the user export these manually. +while IFS='=' read -r key value; do + export "$key=$value" +done < <(yq -r 'to_entries | .[] | .key + "=" + .value' env.yaml) -MODULE_NAME="supernnova" # lower case required by cloud run -ROUTE_RUN="/" # url route that will trigger main.run() - -# function used to define GCP resources; appends testid if needed -define_GCP_resources() { - local base_name="$1" - local testid_suffix="" - - if [ "$testid" != "False" ]; then - if [ "$base_name" = "$survey" ]; then - testid_suffix="_${testid}" # complies with BigQuery naming conventions - else - testid_suffix="-${testid}" - fi +# Ensure that all required environment variables are set. +check_env_vars() { + local vars=("$@") + for var in "${vars[@]}"; do + if [ -z "${!var}" ]; then + echo "Error: ${var} environment variable is not set." + exit 1 fi - - echo "${base_name}${testid_suffix}" + export "_${var}=" + done } +check_env_vars PROJECT_ID _SURVEY _TESTID MODULE_NAME_STEM MODULE_ROUTE REGION REPOSITORY_STEM TRIGGER_TOPIC_STEM -#--- GCP resources used in this script -artifact_registry_repo=$(define_GCP_resources "${survey}-cloud-run-services") -deadletter_topic_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova-deadletter") -deadletter_subscription_bigquery_import="${deadletter_topic_bigquery_import}" -ps_input_subscrip=$(define_GCP_resources "${survey}-alerts") # pub/sub subscription used to trigger cloud run module -ps_output_topic=$(define_GCP_resources "${survey}-SuperNNova") -subscription_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova") # BigQuery subscription -topic_bigquery_import=$(define_GCP_resources "${survey}-bigquery-import-SuperNNova") -trigger_topic=$(define_GCP_resources "${survey}-alerts") +# Construct and export additional environment variables for cloudbuild.yaml. +# Environment variables that will be used by cloudbuild.yaml must start with "_", per GCP's requirements. +_MODULE_NAME=$(construct-name.sh --stem "$MODULE_NAME_STEM") +export _MODULE_NAME="$_MODULE_NAME" +_REPOSITORY=$(construct-name.sh --stem "$REPOSITORY_STEM") +export _REPOSITORY="$_REPOSITORY" +_TRIGGER_TOPIC=$(construct-name.sh --stem "$TRIGGER_TOPIC_STEM") +export _TRIGGER_TOPIC="$_TRIGGER_TOPIC" +# ----------------------------------------------- -# additional GCP resources & variables used in this script -bq_dataset=$(define_GCP_resources "${survey}") -supernnova_classifications_table="SuperNNova" -cr_module_name=$(define_GCP_resources "${survey}-${MODULE_NAME}") # lower case required by Cloud Run +# --------- Project setup ----------------------- +# [FIXME] This is a project setup task, so should be moved to a script dedicated to that. +# Ensure the Cloud Run service has the necessary permissions. runinvoker_svcact="cloud-run-invoker@${PROJECT_ID}.iam.gserviceaccount.com" +gcloud run services add-iam-policy-binding "${_MODULE_NAME}" \ + --member="serviceAccount:${runinvoker_svcact}" \ + --role="roles/run.invoker" +# ----------------------------------------------- +# --------- Build ------------------------------- +# Execute the build steps. +echo "Executing cloudbuild.yaml..." +moduledir=$(dirname "$(readlink -f "$0")") # Absolute path to the parent directory of this script. +url=$(gcloud builds submit \ + --config="${moduledir}/cloudbuild.yaml" \ + --region="${REGION}" \ + "${moduledir}" | sed -n 's/^Step #2: Service URL: \(.*\)$/\1/p' +) +# ----------------------------------------------- -if [ "${teardown}" = "True" ]; then - # ensure that we do not teardown production resources - if [ "${testid}" != "False" ]; then - gcloud pubsub topics delete "${ps_output_topic}" - gcloud pubsub topics delete "${topic_bigquery_import}" - gcloud pubsub topics delete "${deadletter_topic_bigquery_import}" - gcloud pubsub subscriptions delete "${ps_input_subscrip}" - gcloud pubsub subscriptions delete "${subscription_bigquery_import}" - gcloud pubsub subscriptions delete "${deadletter_subscription_bigquery_import}" - gcloud run services delete "${cr_module_name}" --region "${region}" - fi - -else # Deploy the Cloud Run service - -#--- Deploy Cloud Run service - echo "Configuring Pub/Sub resources for classify_snn Cloud Run service..." - gcloud pubsub topics create "${ps_output_topic}" - gcloud pubsub topics create "${topic_bigquery_import}" - gcloud pubsub topics create "${deadletter_topic_bigquery_import}" - gcloud pubsub subscriptions create "${deadletter_subscription_bigquery_import}" --topic="${deadletter_topic_bigquery_import}" - # in order to create BigQuery subscriptions, ensure that the following service account: - # service-@gcp-sa-pubsub.iam.gserviceaccount.com" has the - # bigquery.dataEditor role for each table - PUBSUB_SERVICE_ACCOUNT="service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com" - roleid="roles/bigquery.dataEditor" - bq add-iam-policy-binding \ - --member="serviceAccount:${PUBSUB_SERVICE_ACCOUNT}" \ - --role="${roleid}" \ - --table=true "${PROJECT_ID}:${bq_dataset}.${supernnova_classifications_table}" - gcloud pubsub subscriptions create "${subscription_bigquery_import}" \ - --topic="${topic_bigquery_import}" \ - --bigquery-table="${PROJECT_ID}:${bq_dataset}.${supernnova_classifications_table}" \ - --use-table-schema \ - --dead-letter-topic="${deadletter_topic_bigquery_import}" \ - --max-delivery-attempts=5 \ - --dead-letter-topic-project="${PROJECT_ID}" - - # this allows dead-lettered messages to be forwarded from the BigQuery subscription to the dead letter topic - # and it allows dead-lettered messages to be published to the dead letter topic. - gcloud pubsub topics add-iam-policy-binding "${deadletter_topic_bigquery_import}" \ - --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\ - --role="roles/pubsub.publisher" - gcloud pubsub subscriptions add-iam-policy-binding "${subscription_bigquery_import}" \ - --member="serviceAccount:$PUBSUB_SERVICE_ACCOUNT"\ - --role="roles/pubsub.subscriber" - - echo "Creating container image and deploying to Cloud Run..." - moduledir="." # deploys what's in our current directory - config="${moduledir}/cloudbuild.yaml" - url=$(gcloud builds submit --config="${config}" \ - --substitutions="_SURVEY=${survey},_TESTID=${testid},_MODULE_NAME=${cr_module_name},_REPOSITORY=${artifact_registry_repo}" \ - --region="${region}" \ - "${moduledir}" | sed -n 's/^Step #2: Service URL: \(.*\)$/\1/p') - - # ensure the Cloud Run service has the necessary permisions - role="roles/run.invoker" - gcloud run services add-iam-policy-binding "${cr_module_name}" \ - --member="serviceAccount:${runinvoker_svcact}" \ - --role="${role}" - - echo "Creating trigger subscription for Cloud Run..." - # WARNING: This is set to retry failed deliveries. If there is a bug in main.py this will - # retry indefinitely, until the message is delete manually. - gcloud pubsub subscriptions create "${ps_input_subscrip}" \ - --topic "${trigger_topic}" \ - --topic-project "${PROJECT_ID}" \ - --ack-deadline=600 \ - --push-endpoint="${url}${ROUTE_RUN}" \ - --push-auth-service-account="${runinvoker_svcact}" -fi +# --------- Finish build ------------------------ +# [FIXME] Figure out how to include this in cloudbuild.yaml. It is here because we need the value of $url. +# Create the subscription that will trigger the Cloud Run service. +echo "Creating trigger subscription for Cloud Run..." +# [FIXME] Handle these retries better. +echo "WARNING: This is set to retry failed deliveries. If there is a bug in main.py this will" +echo " retry indefinitely, until the message is delete manually." +trigger_subscrip="$_TRIGGER_TOPIC" +gcloud pubsub subscriptions create "${trigger_subscrip}" \ + --topic "${_TRIGGER_TOPIC}" \ + --topic-project "${PROJECT_ID}" \ + --ack-deadline=600 \ + --push-endpoint="${url}${MODULE_ROUTE}" \ + --push-auth-service-account="${runinvoker_svcact}" +# ----------------------------------------------- diff --git a/broker/cloud_run/lsst/classify_snn/env.yaml b/broker/cloud_run/lsst/classify_snn/env.yaml new file mode 100644 index 00000000..740bb213 --- /dev/null +++ b/broker/cloud_run/lsst/classify_snn/env.yaml @@ -0,0 +1,8 @@ +# Environment variables that will be used by cloudbuild.yaml must start with "_", per GCP's requirements. +_TESTID: 'testid' +_SURVEY: 'lsst' +MODULE_NAME_STEM: 'supernnova' +MODULE_ROUTE: '/' # url route that will trigger main.run() +REGION: 'us-central1' +REPOSITORY_STEM: 'cloud-run-services' +TRIGGER_TOPIC_STEM: 'alerts'