|
1 | 1 | #! /bin/bash |
2 | 2 | # Create and configure GCP resources needed to run the nightly broker. |
3 | 3 |
|
4 | | -testid="${1:-test}" |
5 | 4 | # "False" uses production resources |
6 | 5 | # any other string will be appended to the names of all resources |
7 | | -teardown="${2:-False}" |
| 6 | +testid="${1:-test}" |
8 | 7 | # "True" tearsdown/deletes resources, else setup |
9 | | -survey="${3:-rubin}" |
| 8 | +teardown="${2:-False}" |
10 | 9 | # name of the survey this broker instance will ingest |
11 | | -region="${4:-us-central1}" |
| 10 | +survey="${3:-lsst}" |
| 11 | +schema_version="${4:-7.1}" |
| 12 | +versiontag=v$(echo "${schema_version}" | tr . _) # 7.1 -> v7_1 |
| 13 | +region="${5:-us-central1}" |
12 | 14 | zone="${region}-a" # just use zone "a" instead of adding another script arg |
13 | 15 |
|
14 | 16 | PROJECT_ID=$GOOGLE_CLOUD_PROJECT # get the environment variable |
|
20 | 22 | echo "GOOGLE_CLOUD_PROJECT = ${PROJECT_ID}" |
21 | 23 | echo "survey = ${survey}" |
22 | 24 | echo "testid = ${testid}" |
| 25 | +echo "schema_version = ${schema_version}" |
23 | 26 | echo "teardown = ${teardown}" |
24 | 27 | echo |
25 | 28 | echo "Continue? [y/(n)]: " |
|
34 | 37 |
|
35 | 38 | #--- GCP resources used directly in this script |
36 | 39 | broker_bucket="${PROJECT_ID}-${survey}-broker_files" |
37 | | -topic_alerts="${survey}-alerts" |
38 | | -pubsub_subscription="${topic_alerts}" |
| 40 | +bq_dataset="${survey}" |
| 41 | +topic_alerts="${survey}-alerts_raw" |
| 42 | +pubsub_subscription="${topic_alerts}" # draft, remove before merging PR |
| 43 | +subscription_storebigquery="${survey}-bigquery" |
| 44 | + |
39 | 45 | # use test resources, if requested |
40 | | -# (there must be a better way to do this) |
41 | 46 | if [ "$testid" != "False" ]; then |
42 | 47 | broker_bucket="${broker_bucket}-${testid}" |
| 48 | + bq_dataset="${bq_dataset}_${testid}" |
43 | 49 | topic_alerts="${topic_alerts}-${testid}" |
44 | | - pubsub_subscription="${pubsub_subscription}-${testid}" |
| 50 | + pubsub_subscription="${pubsub_subscription}-${testid}" # draft, remove before merging PR |
| 51 | + subscription_storebigquery="${subscription_storebigquery}-${testid}" |
45 | 52 | fi |
46 | 53 |
|
| 54 | +alerts_table="alerts_${versiontag}" |
47 | 55 |
|
48 | | -#--- Create (or delete) GCS, Pub/Sub resources |
| 56 | +#--- Create (or delete) BigQuery, GCS, Pub/Sub resources |
| 57 | +echo |
| 58 | +echo "Configuring BigQuery, GCS, Pub/Sub resources..." |
49 | 59 | if [ "${teardown}" != "True" ]; then |
| 60 | + # create bigquery dataset and table |
| 61 | + bq --location="${region}" mk --dataset "${bq_dataset}" |
| 62 | + |
| 63 | + cd templates || exit 5 |
| 64 | + bq mk --table "${PROJECT_ID}:${bq_dataset}.${alerts_table}" "bq_${survey}_${alerts_table}_schema.json" || exit 5 |
| 65 | + bq update --description "Alert data from LSST. This table is an archive of the lsst-alerts Pub/Sub stream. It has the same schema as the original alert bytes, including nested and repeated fields." "${PROJECT_ID}:${bq_dataset}.${alerts_table}" |
| 66 | + cd .. || exit 5 |
| 67 | + |
50 | 68 | # create broker bucket and upload files |
51 | 69 | echo "Creating broker_bucket and uploading files..." |
52 | 70 | gsutil mb -b on -l "${region}" "gs://${broker_bucket}" |
53 | 71 | ./upload_broker_bucket.sh "${broker_bucket}" |
54 | 72 |
|
| 73 | + #--- Create a firewall rule to open the port used by Kafka/Rubin LSST |
| 74 | + # on any instance with the flag --tags=tcpport9094 |
| 75 | + echo |
| 76 | + echo "Configuring Rubin/Kafka firewall rule..." |
| 77 | + firewallrule="tcpport9094" |
| 78 | + gcloud compute firewall-rules create "${firewallrule}" \ |
| 79 | + --allow=tcp:9094 \ |
| 80 | + --description="Allow incoming traffic on TCP port 9094" \ |
| 81 | + --direction=INGRESS \ |
| 82 | + --enable-logging |
| 83 | + |
55 | 84 | # create pubsub |
56 | 85 | echo "Configuring Pub/Sub resources..." |
57 | 86 | gcloud pubsub topics create "${topic_alerts}" |
58 | 87 | gcloud pubsub subscriptions create "${pubsub_subscription}" --topic="${topic_alerts}" |
| 88 | + gcloud pubsub subscriptions create "${subscription_storebigquery}" --topic="${topic_alerts}" --bigquery-table="${PROJECT_ID}:${bq_dataset}.${alerts_table}" --use_table_schema=true |
59 | 89 |
|
60 | 90 | # Set IAM policies on resources |
61 | 91 | user="allUsers" |
|
67 | 97 | if [ "${testid}" != "False" ]; then |
68 | 98 | o="GSUtil:parallel_process_count=1" # disable multiprocessing for Macs |
69 | 99 | gsutil -m -o "${o}" rm -r "gs://${broker_bucket}" |
| 100 | + bq rm -r -f "${PROJECT_ID}:${bq_dataset}" |
70 | 101 | gcloud pubsub topics delete "${topic_alerts}" |
71 | 102 | gcloud pubsub subscriptions delete "${pubsub_subscription}" |
| 103 | + gcloud pubsub subscriptions delete "${subscription_storebigquery}" |
72 | 104 | fi |
73 | 105 | fi |
74 | 106 |
|
75 | | -if [ "$teardown" != "True" ]; then |
76 | | - #--- Create a firewall rule to open the port used by Kafka/Rubin |
77 | | - # on any instance with the flag --tags=tcpport9094 |
78 | | - echo |
79 | | - echo "Configuring Rubin/Kafka firewall rule..." |
80 | | - firewallrule="tcpport9094" |
81 | | - gcloud compute firewall-rules create "${firewallrule}" \ |
82 | | - --allow=tcp:9094 \ |
83 | | - --description="Allow incoming traffic on TCP port 9094" \ |
84 | | - --direction=INGRESS \ |
85 | | - --enable-logging |
86 | | -fi |
87 | | - |
88 | 107 | #--- Create VM instances |
89 | 108 | echo |
90 | 109 | echo "Configuring VMs..." |
|
0 commit comments