includes updates related to schema version and config parameters

Author: hernandezc1

broker/cloud_functions/lvk/store_BigQuery/deploy.sh

@@ -9,6 +9,8 @@ testid="${1:-test}"
 teardown="${2:-False}"
 # name of the survey this broker instance will ingest
 survey="${3:-lvk}"
+# schema version
+versiontag="${4:-v1_0}"
 
 #--- GCP resources used in this script
 store_bq_trigger_topic="${survey}-alerts"
@@ -37,5 +39,5 @@ else # Deploy the Cloud Functions
         --runtime python312 \
         --memory "${memory}" \
         --trigger-topic "${store_bq_trigger_topic}" \
-        --set-env-vars TESTID="${testid}",SURVEY="${survey}",GCP_PROJECT="${GOOGLE_CLOUD_PROJECT}"
+        --set-env-vars TESTID="${testid}",SURVEY="${survey}",GCP_PROJECT="${GOOGLE_CLOUD_PROJECT}",VERSIONTAG="${versiontag}"
 fi

broker/consumer/lvk/ps-connector.properties

@@ -20,7 +20,7 @@ name=CPSSinkConnector
 # Tha Java class for the Pub/Sub sink connector.
 connector.class=com.google.pubsub.kafka.sink.CloudPubSubSinkConnector
 # The maximum number of tasks that should be created for this connector.
-tasks.max=10
+tasks.max=1
 # Set the key converter for the Pub/Sub sink connector.
 key.converter=org.apache.kafka.connect.converters.ByteArrayConverter
 # Set the value converter for the Pub/Sub sink connector.

broker/consumer/lvk/psconnect-worker-authenticated.properties

@@ -4,6 +4,14 @@
 bootstrap.servers=kafka.gcn.nasa.gov:9092
 plugin.path=/usr/local/share/kafka/plugins
 offset.storage.file.filename=/tmp/connect.offsets
+session.timeout.ms=30000
+request.timeout.ms=30000
+max.poll.interval.ms=300000
+heartbeat.interval.ms=3000
+retry.backoff.ms=1000
+reconnect.backoff.max.ms=10000
+fetch.max.wait.ms=1000
+connections.max.idle.ms=5400000
 
 # ByteArrayConverter provides a “pass-through” option that does no conversion.
 key.converter=org.apache.kafka.connect.converters.ByteArrayConverter

broker/setup_broker/lvk/README.md

@@ -99,9 +99,10 @@ Initialize parameters and call the deployment script:
 testid="mytest"
 teardown="False"
 survey="lvk"
+schema_version="1.0"
 region="us-central1"
 
-./setup_broker.sh "${testid}" "${teardown}" "${survey}" "${region}"
+./setup_broker.sh "${testid}" "${teardown}" "${survey}" "${schema_version}" "${region}"
 ```
 
 This will create all of the necessary GCP resources. Allow the consumer VM to finish its installation process. Once
@@ -141,7 +142,8 @@ Similar to [deploy broker instance](#deploy-broker-instance). Initialize paramet
 testid="mytest"
 teardown="True"
 survey="lvk"
+schema_version="1.0"
 region="us-central1"
 
-./setup_broker.sh "${testid}" "${teardown}" "${survey}" "${region}"
+./setup_broker.sh "${testid}" "${teardown}" "${survey}" "${schema_version}" "${region}"
 ```

broker/setup_broker/lvk/setup_broker.sh

@@ -22,6 +22,7 @@ echo
 echo "GOOGLE_CLOUD_PROJECT = ${PROJECT_ID}"
 echo "survey = ${survey}"
 echo "testid = ${testid}"
+echo "schema_version = ${schema_version}"
 echo "teardown = ${teardown}"
 echo
 echo "Continue?  [y/(n)]: "
@@ -98,7 +99,7 @@ echo "Configuring Cloud Functions..."
 cd .. && cd .. && cd cloud_functions && cd lvk || exit 5
 
 #--- BigQuery storage cloud function
-cd store_BigQuery && ./deploy.sh "$testid" "$teardown" "$survey" || exit 5
+cd store_BigQuery && ./deploy.sh "$testid" "$teardown" "$survey" "$versiontag"|| exit 5
 
 #--- return to setup_broker/lvk directory
 cd .. && cd .. && cd .. && cd setup_broker && cd lvk || exit 5

broker/setup_broker/lvk/templates/bq_lvk_alerts_v1_0_schema.json

@@ -19,9 +19,18 @@
   },
   {
     "description": "URLs relevant to the event, if any.",
+    "fields": [
+      {
+        "description": "",
+        "mode": "REQUIRED",
+        "name": "gracedb",
+        "type": "STRING"
+      }
+    ],
     "mode": "REQUIRED",
     "name": "urls",
-    "type": "STRING"
+    "type": "RECORD"
+
   },
   {
     "description": "Event information",