Skip to content

Commit f7ab6c7

Browse files
committed
make and upload files to setup_broker/lvk
1 parent 02e11c9 commit f7ab6c7

File tree

3 files changed

+102
-8
lines changed

3 files changed

+102
-8
lines changed

broker/consumer/lvk/create_vms.sh renamed to broker/setup_broker/lvk/create_vms.sh

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -15,7 +15,6 @@ zone="${6:-us-central1-a}"
1515

1616
#--- GCP resources used in this script
1717
consumerVM="${survey}-consumer"
18-
consumerVMsched="${consumerVM}-schedule"
1918
# use test resources, if requested
2019
if [ "$testid" != "False" ]; then
2120
consumerVM="${consumerVM}-${testid}"
@@ -42,18 +41,11 @@ else
4241
startupscript="startup-script-url=gs://${broker_bucket}/consumer/${survey}/vm_install.sh"
4342
shutdownscript="shutdown-script-url=gs://${broker_bucket}/consumer/${survey}/vm_shutdown.sh"
4443
gcloud compute instances create "$consumerVM" \
45-
--resource-policies="${consumerVMsched}" \
4644
--zone="$zone" \
4745
--address="$consumerIP" \
4846
--machine-type="$machinetype" \
4947
--scopes=cloud-platform \
5048
--metadata="${googlelogging},${startupscript},${shutdownscript}" \
5149
--tags=ztfport # for the firewall rule to open the port
5250

53-
#--- Disable the schedules for testing instances
54-
if [ "$testid" != "False" ]; then
55-
gcloud compute instances remove-resource-policies "${consumerVM}" \
56-
--resource-policies="${consumerVMsched}"
57-
fi
58-
5951
fi
Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,92 @@
1+
#! /bin/bash
2+
# Create and configure GCP resources needed to run the nightly broker.
3+
4+
testid="${1:-test}"
5+
# "False" uses production resources
6+
# any other string will be appended to the names of all resources
7+
teardown="${2:-False}"
8+
# "True" tearsdown/deletes resources, else setup
9+
survey="${3:-lvk}"
10+
# name of the survey this broker instance will ingest
11+
region="${4:-us-central1}"
12+
zone="${region}-a" # just use zone "a" instead of adding another script arg
13+
14+
PROJECT_ID=$GOOGLE_CLOUD_PROJECT # get the environment variable
15+
16+
#--- Make the user confirm the settings
17+
echo
18+
echo "setup_broker.sh will run with the following configs: "
19+
echo
20+
echo "GOOGLE_CLOUD_PROJECT = ${PROJECT_ID}"
21+
echo "survey = ${survey}"
22+
echo "testid = ${testid}"
23+
echo "teardown = ${teardown}"
24+
echo
25+
echo "Continue? [y/(n)]: "
26+
27+
read continue_with_setup
28+
continue_with_setup="${continue_with_setup:-n}"
29+
if [ "$continue_with_setup" != "y" ]; then
30+
echo "Exiting setup."
31+
echo
32+
exit
33+
fi
34+
35+
#--- GCP resources used directly in this script
36+
broker_bucket="${PROJECT_ID}-${survey}-broker_files"
37+
topic_alerts="${survey}-alerts"
38+
client_id="${survey}-${PROJECT_ID}-client-id"
39+
client_secret="${survey}-${PROJECT_ID}-client-secret"
40+
# use test resources, if requested
41+
if [ "$testid" != "False" ]; then
42+
broker_bucket="${broker_bucket}-${testid}"
43+
topic_alerts="${topic_alerts}-${testid}"
44+
fi
45+
46+
#--- Create (or delete) GCS, Pub/Sub resources
47+
if [ "${teardown}" != "True" ]; then
48+
# create broker bucket and upload files
49+
echo "Creating broker_bucket and uploading files..."
50+
gsutil mb -b on -l "${region}" "gs://${broker_bucket}"
51+
./upload_broker_bucket.sh "${broker_bucket}"
52+
53+
# create pubsub
54+
echo "Configuring Pub/Sub resources..."
55+
gcloud pubsub topics create "${topic_alerts}"
56+
57+
# Set IAM policies on resources
58+
user="allUsers"
59+
roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
60+
gcloud pubsub topics add-iam-policy-binding "${topic_alerts}" --member="${user}" --role="${roleid}"
61+
62+
roleid="roles/secretmanager.secretAccessor"
63+
gcloud secrets add-iam-policy-binding ${client_id} --member="serviceAccount:${user}" --role="${roleid}"
64+
gcloud secrets add-iam-policy-binding ${client_secret} --member="serviceAccount:${user}" --role="${roleid}"
65+
66+
else
67+
# ensure that we do not teardown production resources
68+
if [ "${testid}" != "False" ]; then
69+
o="GSUtil:parallel_process_count=1" # disable multiprocessing for Macs
70+
gsutil -m -o "${o}" rm -r "gs://${broker_bucket}"
71+
gcloud pubsub topics delete "${topic_alerts}"
72+
fi
73+
fi
74+
75+
#--- Create VM instances
76+
echo
77+
echo "Configuring VMs..."
78+
./create_vms.sh "${broker_bucket}" "${testid}" "${teardown}" "${survey}" "${region}" "${zone}"
79+
80+
if [ "$teardown" != "True" ]; then
81+
82+
#--- Create a firewall rule to open the port used by Kafka/ZTF
83+
# on any instance with the flag --tags=ztfport
84+
echo
85+
echo "Configuring ZTF/Kafka firewall rule..."
86+
gcloud compute firewall-rules create 'ztfport' \
87+
--allow=tcp:9094 \
88+
--description="Allow incoming traffic on TCP port 9094" \
89+
--direction=INGRESS \
90+
--enable-logging
91+
92+
fi
Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
#! /bin/bash
2+
3+
broker_bucket=$1 # name of GCS bucket where broker files should be staged
4+
5+
echo
6+
echo "Uploading broker files to GCS..."
7+
o="GSUtil:parallel_process_count=1" # disable multiprocessing for Macs
8+
gsutil -m -o "${o}" cp -r ../../broker_utils/schema_maps "gs://${broker_bucket}"
9+
gsutil -m -o "${o}" cp -r ../../consumer "gs://${broker_bucket}"
10+
gsutil -m -o "${o}" cp -r ../../setup_broker "gs://${broker_bucket}"

0 commit comments

Comments
 (0)