Skip to content

Commit b7d590b

Browse files
committed
partition BigQuery table
1 parent 173aa5b commit b7d590b

File tree

1 file changed

+10
-3
lines changed

1 file changed

+10
-3
lines changed

broker/setup_broker/lvk/setup_broker.sh

Lines changed: 10 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -82,7 +82,7 @@ manage_resources() {
8282
else
8383
echo "${bq_dataset} already exists."
8484
fi
85-
(cd templates && bq mk --table "${PROJECT_ID}:${bq_dataset}.${bq_table_alerts}" "bq_${survey}_${bq_table_alerts}_schema.json") || exit 5
85+
(cd templates && bq mk --table --time_partitioning_field=kafkaPublishTimestamp --time_partitioning_type=DAY "${PROJECT_ID}:${bq_dataset}.${bq_table_alerts}" "bq_${survey}_${bq_table_alerts}_schema.json") || exit 5
8686
bq update --description "LIGO/Virgo/KAGRA (LVK) alerts with schema version v${schema_version}. The data and schema are as produced by LVK except that skymaps are excluded. Skymaps can be retrieved from the Cloud Storage bucket ${gcs_alerts_bucket}." "${PROJECT_ID}:${bq_dataset}.${bq_table_alerts}"
8787

8888
#--- Create GCS bucket
@@ -122,12 +122,19 @@ manage_resources() {
122122
--drop-unknown-fields \
123123
--dead-letter-topic="${ps_deadletter_topic}" \
124124
--max-delivery-attempts=5 \
125-
--dead-letter-topic-project="${PROJECT_ID}"
125+
--dead-letter-topic-project="${PROJECT_ID}" \
126+
--message-transforms-file=templates/ps_lvk_add_top_level_fields_smt.yaml
126127

127128
# set IAM policies on resources
128129
user="allUsers"
129-
roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
130+
roleid="roles/pubsub.subscriber"
130131
gcloud pubsub topics add-iam-policy-binding "${ps_topic_alerts}" --member="${user}" --role="${roleid}"
132+
gcloud pubsub topics add-iam-policy-binding "${ps_deadletter_topic}" \
133+
--member="serviceAccount:${service_account}" \
134+
--role="roles/pubsub.publisher"
135+
gcloud pubsub subscriptions add-iam-policy-binding "${ps_bigquery_subscription}" \
136+
--member="serviceAccount:${service_account}" \
137+
--role="${roleid}"
131138

132139
#--- Create Artifact Registry Repository
133140
echo

0 commit comments

Comments
 (0)