changes to bq schema and consumer group id name

mwvgroup · Oct 7, 2024 · b0dc024 · b0dc024
1 parent 99bd0b7
commit b0dc024
Show file tree

Hide file tree

Showing 3 changed files with 30 additions and 29 deletions.
diff --git a/broker/consumer/lvk/psconnect-worker-authenticated.properties b/broker/consumer/lvk/psconnect-worker-authenticated.properties
@@ -23,7 +23,7 @@ sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginMo
      clientSecret="CLIENT_SECRET";
 
 # settings with `consumer.` prefixes are passed through to the Kafka consumer
-consumer.group.id=pitt-google-broker
+consumer.group.id=GROUP_ID
 consumer.auto.offset.reset=earliest
 consumer.sasl.mechanism=OAUTHBEARER
 consumer.sasl.kerberos.service.name=kafka

diff --git a/broker/consumer/lvk/vm_startup.sh b/broker/consumer/lvk/vm_startup.sh
@@ -60,21 +60,22 @@ client_id="${survey}-${PROJECT_ID}-client-id"
 client_secret="${survey}-${PROJECT_ID}-client-secret"
 CLIENT_ID=$(gcloud secrets versions access latest --secret="${client_id}")
 CLIENT_SECRET=$(gcloud secrets versions access latest --secret="${client_secret}")
+group_id="pittgooglebroker"
+# use test resources, if requested
+if [ "$testid" != "False" ]; then
+    group_id="${group_id}-${testid}"
+fi
 
 cd "${workingdir}" || exit
 
 fconfig=admin.properties
-sed -i "s/CLIENT_ID/${CLIENT_ID}/g" ${fconfig}
-sed -i "s/CLIENT_SECRET/${CLIENT_SECRET}/g" ${fconfig}
+sed -i "s/CLIENT_ID/${CLIENT_ID}/g" ${fconfig} && sed -i "s/CLIENT_SECRET/${CLIENT_SECRET}/g" ${fconfig}
 
 fconfig=psconnect-worker-authenticated.properties
-sed -i "s/CLIENT_ID/${CLIENT_ID}/g" ${fconfig}
-sed -i "s/CLIENT_SECRET/${CLIENT_SECRET}/g" ${fconfig}
+sed -i "s/CLIENT_ID/${CLIENT_ID}/g" ${fconfig} && sed -i "s/CLIENT_SECRET/${CLIENT_SECRET}/g" ${fconfig} && sed -i "s/GROUP_ID/${group_id}/g" ${fconfig}
 
 fconfig=ps-connector.properties
-sed -i "s/PROJECT_ID/${PROJECT_ID}/g" ${fconfig}
-sed -i "s/PS_TOPIC/${PS_TOPIC}/g" ${fconfig}
-sed -i "s/KAFKA_TOPIC/${KAFKA_TOPIC}/g" ${fconfig}
+sed -i "s/PROJECT_ID/${PROJECT_ID}/g" ${fconfig} && sed -i "s/PS_TOPIC/${PS_TOPIC}/g" ${fconfig} && sed -i "s/KAFKA_TOPIC/${KAFKA_TOPIC}/g" ${fconfig}
 
 #--- Check until alerts start streaming into the topic
 alerts_flowing=false

diff --git a/broker/setup_broker/lvk/templates/bq_lvk_alerts_v1_0_schema.json b/broker/setup_broker/lvk/templates/bq_lvk_alerts_v1_0_schema.json
@@ -61,7 +61,7 @@
       },
       {
         "description": "A skymap in a multi-order FITS format as a Base64-encoded string.",
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "skymap",
         "type": "STRING"
       },
@@ -93,31 +93,31 @@
         "description": "The central frequency of the burst. Only present if group=Burst.",
         "mode": "NULLABLE",
         "name": "central_frequency",
-        "type": "INTEGER"
+        "type": "FLOAT"
       },
       {
         "description": "The probabilities that the source has various properties, assuming that the source is astrophysical.",
         "fields": [
           {
             "description": "Probability that at least one of the compact objects was a neutron star",
-            "mode": "REQUIRED",
-            "name": "HasNS",
+            "mode": "NULLABLE",
+            "name": "hasns",
             "type": "FLOAT"
           },
           {
             "description": "Probability that the system ejected a non-zero amount of neutron star matter",
-            "mode": "REQUIRED",
-            "name": "HasRemnant",
+            "mode": "NULLABLE",
+            "name": "hasremnant",
             "type": "FLOAT"
           },
           {
             "description": "Probability that at least one of the compact objects has mass in the range 3-5 solar masses",
-            "mode": "REQUIRED",
-            "name": "HasMassGap",
+            "mode": "NULLABLE",
+            "name": "hasmassgap",
             "type": "FLOAT"
           }
         ],
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "properties",
         "type": "RECORD"
       },
@@ -126,30 +126,30 @@
         "fields": [
           {
             "description": "Probability that the source is a BNS",
-            "mode": "REQUIRED",
-            "name": "BNS",
+            "mode": "NULLABLE",
+            "name": "bns",
             "type": "FLOAT"
           },
           {
             "description": "Probability that the source is a NSBH",
-            "mode": "REQUIRED",
-            "name": "NSBH",
+            "mode": "NULLABLE",
+            "name": "nsbh",
             "type": "FLOAT"
           },
           {
             "description": "Probability that the source is a BBH",
-            "mode": "REQUIRED",
-            "name": "BBH",
+            "mode": "NULLABLE",
+            "name": "bbh",
             "type": "FLOAT"
           },
           {
             "description": "Probability that the source is Terrestrial (i.e, noise)",
-            "mode": "REQUIRED",
-            "name": "Terrestrial",
+            "mode": "NULLABLE",
+            "name": "terrestrial",
             "type": "FLOAT"
           }
         ],
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "classification",
         "type": "RECORD"
       }
@@ -193,19 +193,19 @@
       },
       {
         "description": "The joint time coincidence FAR in units of Hz, if available.",
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "time_coincidence_far",
         "type": "FLOAT"
       },
       {
         "description": "The joint time and sky position coincidence FAR in units of Hz, if available.",
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "time_sky_position_coincidence_far",
         "type": "FLOAT"
       },
       {
         "description": "A skymap in a multi-order FITS format produced by combining the GW skymap and the external coincidence skymap.",
-        "mode": "REQUIRED",
+        "mode": "NULLABLE",
         "name": "combined_skymap",
         "type": "STRING"
       }