make and upload files to setup_broker/lvk

broker/consumer/lvk/create_vms.sh → broker/setup_broker/lvk/create_vms.sh

@@ -15,7 +15,6 @@ zone="${6:-us-central1-a}"
 
 #--- GCP resources used in this script
 consumerVM="${survey}-consumer"
-consumerVMsched="${consumerVM}-schedule"
 # use test resources, if requested
 if [ "$testid" != "False" ]; then
     consumerVM="${consumerVM}-${testid}"
@@ -42,18 +41,11 @@ else
     startupscript="startup-script-url=gs://${broker_bucket}/consumer/${survey}/vm_install.sh"
     shutdownscript="shutdown-script-url=gs://${broker_bucket}/consumer/${survey}/vm_shutdown.sh"
     gcloud compute instances create "$consumerVM" \
-        --resource-policies="${consumerVMsched}" \
         --zone="$zone" \
         --address="$consumerIP" \
         --machine-type="$machinetype" \
         --scopes=cloud-platform \
         --metadata="${googlelogging},${startupscript},${shutdownscript}" \
         --tags=ztfport # for the firewall rule to open the port
 
-#--- Disable the schedules for testing instances
-    if [ "$testid" != "False" ]; then
-        gcloud compute instances remove-resource-policies "${consumerVM}" \
-            --resource-policies="${consumerVMsched}"
-    fi
-
 fi

broker/setup_broker/lvk/setup_broker.sh

@@ -0,0 +1,92 @@
+#! /bin/bash
+# Create and configure GCP resources needed to run the nightly broker.
+
+testid="${1:-test}"
+# "False" uses production resources
+# any other string will be appended to the names of all resources
+teardown="${2:-False}"
+# "True" tearsdown/deletes resources, else setup
+survey="${3:-lvk}"
+# name of the survey this broker instance will ingest
+region="${4:-us-central1}"
+zone="${region}-a"  # just use zone "a" instead of adding another script arg
+
+PROJECT_ID=$GOOGLE_CLOUD_PROJECT # get the environment variable
+
+#--- Make the user confirm the settings
+echo
+echo "setup_broker.sh will run with the following configs: "
+echo
+echo "GOOGLE_CLOUD_PROJECT = ${PROJECT_ID}"
+echo "survey = ${survey}"
+echo "testid = ${testid}"
+echo "teardown = ${teardown}"
+echo
+echo "Continue?  [y/(n)]: "
+
+read continue_with_setup
+continue_with_setup="${continue_with_setup:-n}"
+if [ "$continue_with_setup" != "y" ]; then
+    echo "Exiting setup."
+    echo
+    exit
+fi
+
+#--- GCP resources used directly in this script
+broker_bucket="${PROJECT_ID}-${survey}-broker_files"
+topic_alerts="${survey}-alerts"
+client_id="${survey}-${PROJECT_ID}-client-id"
+client_secret="${survey}-${PROJECT_ID}-client-secret"
+# use test resources, if requested
+if [ "$testid" != "False" ]; then
+    broker_bucket="${broker_bucket}-${testid}"
+    topic_alerts="${topic_alerts}-${testid}"
+fi
+
+#--- Create (or delete) GCS, Pub/Sub resources
+if [ "${teardown}" != "True" ]; then
+    # create broker bucket and upload files
+    echo "Creating broker_bucket and uploading files..."
+    gsutil mb -b on -l "${region}" "gs://${broker_bucket}"
+    ./upload_broker_bucket.sh "${broker_bucket}"
+
+    # create pubsub
+    echo "Configuring Pub/Sub resources..."
+    gcloud pubsub topics create "${topic_alerts}"
+
+    # Set IAM policies on resources
+    user="allUsers"
+    roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
+    gcloud pubsub topics add-iam-policy-binding "${topic_alerts}" --member="${user}" --role="${roleid}"
+    user="[email protected]"
+    roleid="roles/secretmanager.secretAccessor"
+    gcloud secrets add-iam-policy-binding ${client_id} --member="serviceAccount:${user}" --role="${roleid}"
+    gcloud secrets add-iam-policy-binding ${client_secret} --member="serviceAccount:${user}" --role="${roleid}"
+
+else
+    # ensure that we do not teardown production resources
+    if [ "${testid}" != "False" ]; then
+        o="GSUtil:parallel_process_count=1" # disable multiprocessing for Macs
+        gsutil -m -o "${o}" rm -r "gs://${broker_bucket}"
+        gcloud pubsub topics delete "${topic_alerts}"
+    fi
+fi
+
+#--- Create VM instances
+echo
+echo "Configuring VMs..."
+./create_vms.sh "${broker_bucket}" "${testid}" "${teardown}" "${survey}" "${region}" "${zone}"
+
+if [ "$teardown" != "True" ]; then
+
+#--- Create a firewall rule to open the port used by Kafka/ZTF
+# on any instance with the flag --tags=ztfport
+    echo
+    echo "Configuring ZTF/Kafka firewall rule..."
+    gcloud compute firewall-rules create 'ztfport' \
+        --allow=tcp:9094 \
+        --description="Allow incoming traffic on TCP port 9094" \
+        --direction=INGRESS \
+        --enable-logging
+
+fi

broker/setup_broker/lvk/upload_broker_bucket.sh

@@ -0,0 +1,10 @@
+#! /bin/bash
+
+broker_bucket=$1 # name of GCS bucket where broker files should be staged
+
+echo
+echo "Uploading broker files to GCS..."
+o="GSUtil:parallel_process_count=1" # disable multiprocessing for Macs
+gsutil -m -o "${o}" cp -r ../../broker_utils/schema_maps "gs://${broker_bucket}"
+gsutil -m -o "${o}" cp -r ../../consumer "gs://${broker_bucket}"
+gsutil -m -o "${o}" cp -r ../../setup_broker "gs://${broker_bucket}"