Merge branch 'develop' into u/ch/ztf/hostlesstransients

Author: hernandezc1

LICENSE

@@ -0,0 +1,28 @@
+BSD 3-Clause License
+
+Copyright (c) 2025, mwvgroup
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+1. Redistributions of source code must retain the above copyright notice, this
+   list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright notice,
+   this list of conditions and the following disclaimer in the documentation
+   and/or other materials provided with the distribution.
+
+3. Neither the name of the copyright holder nor the names of its
+   contributors may be used to endorse or promote products derived from
+   this software without specific prior written permission.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"
+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

broker/cloud_run/lsst/classify_snn/deploy.sh

@@ -11,7 +11,9 @@ teardown="${2:-False}"
 survey="${3:-lsst}"
 region="${4:-us-central1}"
 # get the environment variable
+BASE_DIR=$(pwd)
 PROJECT_ID=$GOOGLE_CLOUD_PROJECT
+PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)")
 
 MODULE_NAME="supernnova"  # lower case required by cloud run
 ROUTE_RUN="/"  # url route that will trigger main.run()
@@ -30,12 +32,13 @@ define_GCP_resources() {
 #--- GCP resources used in this script
 artifact_registry_repo=$(define_GCP_resources "${survey}-cloud-run-services")
 bq_dataset=$(define_GCP_resources "${survey}" "_")
-bq_table="SuperNNova"
+bq_table="${MODULE_NAME}"
 cr_module_name=$(define_GCP_resources "${survey}-${MODULE_NAME}")  # lower case required by cloud run
-ps_input_subscrip=$(define_GCP_resources "${survey}-SuperNNova") # pub/sub subscription used to trigger cloud run module
-ps_output_topic=$(define_GCP_resources "${survey}-SuperNNova")
+ps_input_subscrip=$(define_GCP_resources "${survey}-${MODULE_NAME}") # pub/sub subscription used to trigger cloud run module
+ps_output_topic=$(define_GCP_resources "${survey}-${MODULE_NAME}")
 ps_trigger_topic=$(define_GCP_resources "${survey}-lite")
 runinvoker_svcact="cloud-run-invoker@${PROJECT_ID}.iam.gserviceaccount.com"
+service_account="service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com"
 # topics and subscriptions involved in writing data to BigQuery
 ps_bigquery_subscription=$(define_GCP_resources "${survey}-${MODULE_NAME}-bigquery-import")
 ps_deadletter_topic=$(define_GCP_resources "${survey}-deadletter")
@@ -51,7 +54,8 @@ if [ "${teardown}" = "True" ]; then
         gcloud run services delete "${cr_module_name}" --region "${region}"
     fi
 else
-    echo "Configuring Pub/Sub resources..."
+    echo
+    echo "Configuring Pub/Sub resources for ${MODULE_NAME} module..."
     gcloud pubsub topics create "${ps_output_topic}"
     gcloud pubsub subscriptions create "${ps_bigquery_subscription}" \
         --topic="${ps_output_topic}" \
@@ -60,12 +64,14 @@ else
         --drop-unknown-fields \
         --dead-letter-topic="${ps_deadletter_topic}" \
         --max-delivery-attempts=5 \
-        --dead-letter-topic-project="${PROJECT_ID}"
+        --dead-letter-topic-project="${PROJECT_ID}" \
+        --message-transforms-file="${BASE_DIR%%/cloud_run/*}/setup_broker/lsst/templates/ps_lsst_flatten_schema_smt.yaml"
     # set IAM policies on public Pub/Sub resources
     if [ "$testid" = "False" ]; then
         user="allUsers"
-        roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
+        roleid="roles/pubsub.subscriber"
         gcloud pubsub topics add-iam-policy-binding "${ps_output_topic}" --member="${user}" --role="${roleid}"
+        gcloud pubsub subscriptions add-iam-policy-binding "${ps_bigquery_subscription}" --member="serviceAccount:${service_account}" --role="${roleid}"
     fi
 
     #--- Deploy Cloud Run service

broker/cloud_run/lsst/classify_snn/main.py

@@ -47,7 +47,7 @@
 # GCP resources used in this module
 # pittgoogle will construct the full resource names from the module name, SURVEY, and TESTID
 TOPIC = pittgoogle.Topic.from_cloud(
-    "SuperNNova", survey=SURVEY, testid=TESTID, projectid=PROJECT_ID
+    "supernnova", survey=SURVEY, testid=TESTID, projectid=PROJECT_ID
 )
 
 

broker/cloud_run/lsst/classify_snn/requirements.txt

@@ -2,7 +2,7 @@ pandas==1.5.1
 numpy==1.23.3
 google-cloud-functions
 google-cloud-logging
-pittgoogle-client>=0.3.15
+pittgoogle-client>=0.3.17
 
 # for Cloud Run
 # https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service

broker/cloud_run/lsst/classify_upsilon/deploy.sh

@@ -11,7 +11,9 @@ teardown="${2:-False}"
 survey="${3:-lsst}"
 region="${4:-us-central1}"
 # get the environment variable
+BASE_DIR=$(pwd)
 PROJECT_ID=$GOOGLE_CLOUD_PROJECT
+PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)")
 
 MODULE_NAME="upsilon"  # lower case required by cloud run
 ROUTE_RUN="/"  # url route that will trigger main.run()
@@ -36,6 +38,7 @@ ps_input_subscrip=$(define_GCP_resources "${survey}-upsilon") # pub/sub subscrip
 ps_output_topic=$(define_GCP_resources "${survey}-upsilon")
 ps_trigger_topic=$(define_GCP_resources "${survey}-lite")
 runinvoker_svcact="cloud-run-invoker@${PROJECT_ID}.iam.gserviceaccount.com"
+service_account="service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com"
 # topics and subscriptions involved in writing data to BigQuery
 ps_bigquery_subscription=$(define_GCP_resources "${survey}-${MODULE_NAME}-bigquery-import")
 ps_deadletter_topic=$(define_GCP_resources "${survey}-deadletter")
@@ -52,7 +55,8 @@ if [ "${teardown}" = "True" ]; then
     fi
 
 else
-    echo "Configuring Pub/Sub resources..."
+    echo
+    echo "Configuring Pub/Sub resources for ${MODULE_NAME} module..."
     gcloud pubsub topics create "${ps_output_topic}"
     gcloud pubsub subscriptions create "${ps_bigquery_subscription}" \
         --topic="${ps_output_topic}" \
@@ -61,12 +65,14 @@ else
         --drop-unknown-fields \
         --dead-letter-topic="${ps_deadletter_topic}" \
         --max-delivery-attempts=5 \
-        --dead-letter-topic-project="${PROJECT_ID}"
+        --dead-letter-topic-project="${PROJECT_ID}" \
+        --message-transforms-file="${BASE_DIR%%/cloud_run/*}/setup_broker/lsst/templates/ps_lsst_flatten_schema_smt.yaml"
     # set IAM policies on public Pub/Sub resources
     if [ "$testid" = "False" ]; then
         user="allUsers"
-        roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
+        roleid="roles/pubsub.subscriber"
         gcloud pubsub topics add-iam-policy-binding "${ps_output_topic}" --member="${user}" --role="${roleid}"
+        gcloud pubsub subscriptions add-iam-policy-binding "${ps_bigquery_subscription}" --member="serviceAccount:${service_account}" --role="${roleid}"
     fi
 
     #--- Deploy Cloud Run

broker/cloud_run/lsst/classify_upsilon/requirements.txt

@@ -4,7 +4,7 @@
 # file (or packaged with the function) in the same directory as `main.py`
 
 google-cloud-logging
-pittgoogle-client>=0.3.15
+pittgoogle-client>=0.3.17
 upsilon
 # required by upsilon
 # https://github.com/dwkim78/upsilon?tab=readme-ov-file#1-dependency

broker/cloud_run/lsst/ps_to_storage/requirements.txt

@@ -5,7 +5,7 @@
 
 google-cloud-logging
 google-cloud-storage
-pittgoogle-client>=0.3.15
+pittgoogle-client>=0.3.17
 
 # for Cloud Run
 # https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service

broker/cloud_run/lsst/variability/deploy.sh

@@ -11,7 +11,9 @@ teardown="${2:-False}"
 survey="${3:-lsst}"
 region="${4:-us-central1}"
 # get the environment variable
+BASE_DIR=$(pwd)
 PROJECT_ID=$GOOGLE_CLOUD_PROJECT
+PROJECT_NUMBER=$(gcloud projects describe "$PROJECT_ID" --format="value(projectNumber)")
 
 MODULE_NAME="variability"  # lower case required by cloud run
 ROUTE_RUN="/"  # url route that will trigger main.run()
@@ -36,6 +38,7 @@ ps_input_subscrip=$(define_GCP_resources "${survey}-${MODULE_NAME}") # pub/sub s
 ps_output_topic=$(define_GCP_resources "${survey}-${MODULE_NAME}")
 ps_trigger_topic=$(define_GCP_resources "${survey}-lite")
 runinvoker_svcact="cloud-run-invoker@${PROJECT_ID}.iam.gserviceaccount.com"
+service_account="service-${PROJECT_NUMBER}@gcp-sa-pubsub.iam.gserviceaccount.com"
 # topics and subscriptions involved in writing data to BigQuery
 ps_bigquery_subscription=$(define_GCP_resources "${survey}-${MODULE_NAME}-bigquery-import")
 ps_deadletter_topic=$(define_GCP_resources "${survey}-deadletter")
@@ -51,7 +54,8 @@ if [ "${teardown}" = "True" ]; then
         gcloud run services delete "${cr_module_name}" --region "${region}"
     fi
 else
-    echo "Configuring Pub/Sub resources..."
+    echo
+    echo "Configuring Pub/Sub resources for ${MODULE_NAME} module..."
     gcloud pubsub topics create "${ps_output_topic}"
     gcloud pubsub subscriptions create "${ps_bigquery_subscription}" \
         --topic="${ps_output_topic}" \
@@ -60,12 +64,14 @@ else
         --drop-unknown-fields \
         --dead-letter-topic="${ps_deadletter_topic}" \
         --max-delivery-attempts=5 \
-        --dead-letter-topic-project="${PROJECT_ID}"
+        --dead-letter-topic-project="${PROJECT_ID}" \
+        --message-transforms-file="${BASE_DIR%%/cloud_run/*}/setup_broker/lsst/templates/ps_lsst_flatten_schema_smt.yaml"
     # set IAM policies on public Pub/Sub resources
     if [ "$testid" = "False" ]; then
         user="allUsers"
-        roleid="projects/${GOOGLE_CLOUD_PROJECT}/roles/userPublic"
+        roleid="roles/pubsub.subscriber"
         gcloud pubsub topics add-iam-policy-binding "${ps_output_topic}" --member="${user}" --role="${roleid}"
+        gcloud pubsub subscriptions add-iam-policy-binding "${ps_bigquery_subscription}" --member="serviceAccount:${service_account}" --role="${roleid}"
     fi
 
     #--- Deploy Cloud Run service

broker/cloud_run/lsst/variability/requirements.txt

@@ -4,7 +4,7 @@
 # file (or packaged with the function) in the same directory as `main.py`
 
 google-cloud-logging
-pittgoogle-client>=0.3.15
+pittgoogle-client>=0.3.17
 
 # for Cloud Run
 # https://cloud.google.com/run/docs/quickstarts/build-and-deploy/deploy-python-service

broker/consumer/lsst/vm_startup.sh

@@ -44,7 +44,7 @@ fi
 ) || exit
 
 #--- Set the topic names to the "FORCE" metadata attributes if exist, else defaults
-KAFKA_TOPIC_DEFAULT="alerts-simulated"
+KAFKA_TOPIC_DEFAULT="lsst-alerts-v9.0"
 KAFKA_TOPIC="${KAFKA_TOPIC_FORCE:-${KAFKA_TOPIC_DEFAULT}}"
 PS_TOPIC="${PS_TOPIC_FORCE:-${PS_TOPIC_DEFAULT}}"
 # set VM metadata, just for clarity and easy viewing