Merge branch 'develop' into u/ch/store_BigQuery

mwvgroup · Oct 7, 2024 · 506b6d4 · 506b6d4
2 parents 4a1c6ed + d1e5a6c
commit 506b6d4
Show file tree

Hide file tree

Showing 10 changed files with 85 additions and 24 deletions.
diff --git a/broker/consumer/admin.properties b/broker/consumer/admin.properties
@@ -1,4 +1,4 @@
-# Authentication information.
+# Kafka Admin client configs
 # This file is part of a workflow that creates an authenticated connection to the Kafka broker.
 # In cases where we can connect without authentication (e.g., ZTF), this file is not used.
 # For config options, see https://kafka.apache.org/documentation/#adminclientconfigs

diff --git a/broker/consumer/lvk/admin.properties b/broker/consumer/lvk/admin.properties
@@ -1,11 +1,13 @@
-# Authentication information.
+# Kafka Admin client configs
 # This file is part of a workflow that creates an authenticated connection to the Kafka broker.
-# For config options, see https://gcn.nasa.gov/docs/client#java
+# In cases where we can connect without authentication (e.g., ZTF), this file is not used.
+# For config options, see https://kafka.apache.org/documentation/#adminclientconfigs
+# For LVK-specific options, see https://gcn.nasa.gov/docs/client#java
 
 security.protocol=SASL_SSL
 sasl.mechanism=OAUTHBEARER
 sasl.login.callback.handler.class=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginCallbackHandler
 sasl.oauthbearer.token.endpoint.url=https://auth.gcn.nasa.gov/oauth2/token
 sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginModule required \
      clientId="CLIENT_ID" \
-     clientSecret="CLIENT_SECRET";
+     clientSecret="CLIENT_SECRET";
diff --git a/broker/consumer/lvk/ps-connector.properties b/broker/consumer/lvk/ps-connector.properties
@@ -1,7 +1,13 @@
-# Adapted from:
+# Kafka Connect sink connector configs
+# For config options, see https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html
+# For additional Pub/Sub-specific options, see https://github.com/googleapis/java-pubsub-group-kafka-connector?tab=readme-ov-file#sink-connector
+#
+# --------------------------------------------------------------------------
+# This file is adapted from:
 # https://github.com/googleapis/java-pubsub-group-kafka-connector/blob/main/config/cps-sink-connector.properties
-
-# Copyright 2024 Google LLC
+# The original copyright and license are reproduced below.
+#
+# Copyright 2022 Google LLC
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -14,6 +20,7 @@
 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 # See the License for the specific language governing permissions and
 # limitations under the License.
+# --------------------------------------------------------------------------
 
 # Unique name for the Pub/Sub sink connector.
 name=CPSSinkConnector

diff --git a/broker/consumer/lvk/psconnect-worker-authenticated.properties b/broker/consumer/lvk/psconnect-worker-authenticated.properties
@@ -1,5 +1,7 @@
-# Kafka connect worker configuration
-# See: https://docs.confluent.io/platform/current/connect/references/allconfigs.html
+# Kafka Connect worker configuration
+# This file is part of a workflow that creates an authenticated connection to the Kafka broker.
+# For config options, see https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
+# See also: https://kafka.apache.org/documentation/#adminclientconfigs
 
 bootstrap.servers=kafka.gcn.nasa.gov:9092
 plugin.path=/usr/local/share/kafka/plugins
@@ -27,7 +29,7 @@ sasl.jaas.config=org.apache.kafka.common.security.oauthbearer.OAuthBearerLoginMo
      clientId="CLIENT_ID" \
      clientSecret="CLIENT_SECRET";
 
-# settings with `consumer` prefixes are passed through to the Kafka consumer
+# settings with `consumer.` prefixes are passed through to the Kafka consumer
 consumer.group.id=pitt-google-broker-test
 consumer.auto.offset.reset=earliest
 consumer.sasl.mechanism=OAUTHBEARER

diff --git a/broker/consumer/ps-connector.properties b/broker/consumer/ps-connector.properties
@@ -1,3 +1,27 @@
+# Kafka Connect sink connector configs
+# For config options, see https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html
+# For additional Pub/Sub-specific options, see https://github.com/googleapis/java-pubsub-group-kafka-connector?tab=readme-ov-file#sink-connector
+#
+# --------------------------------------------------------------------------
+# This file is adapted from:
+# https://github.com/googleapis/java-pubsub-group-kafka-connector/blob/main/config/cps-sink-connector.properties
+# The original copyright and license are reproduced below.
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# --------------------------------------------------------------------------
+
 name=ps-sink-connector
 connector.class=com.google.pubsub.kafka.sink.CloudPubSubSinkConnector
 tasks.max=10

diff --git a/broker/consumer/psconnect-worker-authenticated.properties b/broker/consumer/psconnect-worker-authenticated.properties
@@ -1,7 +1,7 @@
-# Kafka connect worker configuration for the ZTF consumer
+# Kafka Connect worker configuration
 # This file is part of a workflow that creates an authenticated connection to the Kafka broker.
-# reference: https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
-# see also: https://kafka.apache.org/documentation/#adminclientconfigs
+# For config options, see https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
+# See also: https://kafka.apache.org/documentation/#adminclientconfigs
 
 bootstrap.servers=public2.alerts.ztf.uw.edu:9094
 plugin.path=/usr/local/share/kafka/plugins

diff --git a/broker/consumer/psconnect-worker-unauthenticated.properties b/broker/consumer/psconnect-worker-unauthenticated.properties
@@ -1,7 +1,7 @@
-# Kafka connect worker configuration for the ZTF consumer
+# Kafka Connect worker configuration
 # This file is part of a workflow that creates an unauthenticated connection to the Kafka broker.
-# reference: https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
-# see also: https://kafka.apache.org/documentation/#adminclientconfigs
+# For config options, see https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
+# See also: https://kafka.apache.org/documentation/#adminclientconfigs
 
 bootstrap.servers=public.alerts.ztf.uw.edu:9092
 plugin.path=/usr/local/share/kafka/plugins

diff --git a/broker/consumer/rubin/admin.properties b/broker/consumer/rubin/admin.properties
@@ -1,4 +1,7 @@
-# see https://kafka.apache.org/documentation/#adminclientconfigs
+# Kafka Admin client configs
+# This file is part of a workflow that creates an authenticated connection to the Kafka broker.
+# In cases where we can connect without authentication (e.g., ZTF), this file is not used.
+# For config options, see https://kafka.apache.org/documentation/#adminclientconfigs
 
 bootstrap.servers=usdf-alert-stream-dev.lsst.cloud:9094
 sasl.mechanism=SCRAM-SHA-512

diff --git a/broker/consumer/rubin/ps-connector.properties b/broker/consumer/rubin/ps-connector.properties
@@ -1,9 +1,30 @@
-# Adapted from:
-# https://github.com/GoogleCloudPlatform/pubsub/blob/master/kafka-connector/config/cps-sink-connector.properties
+# Kafka Connect sink connector configs
+# For config options, see https://docs.confluent.io/platform/current/installation/configuration/connect/sink-connect-configs.html
+# For additional Pub/Sub-specific options, see https://github.com/googleapis/java-pubsub-group-kafka-connector?tab=readme-ov-file#sink-connector
+#
+# --------------------------------------------------------------------------
+# This file is adapted from:
+# https://github.com/googleapis/java-pubsub-group-kafka-connector/blob/main/config/cps-sink-connector.properties
+# The original copyright and license are reproduced below.
+#
+# Copyright 2022 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# --------------------------------------------------------------------------
 
 name=ps-sink-connector
 connector.class=com.google.pubsub.kafka.sink.CloudPubSubSinkConnector
-# Tasks do the actual data copying and determine parallelism of the consumer. 
+# Tasks do the actual data copying and determine parallelism of the consumer.
 # When I (Troy Raen) used 10 max tasks (as we do with ZTF),
 # the consumer VM (size e2-standard-2) ran out of memory.
 # Using 1 task for now.

diff --git a/broker/consumer/rubin/psconnect-worker.properties b/broker/consumer/rubin/psconnect-worker.properties
@@ -1,5 +1,7 @@
-# Kafka connect worker configuration
-# See: https://docs.confluent.io/platform/current/connect/references/allconfigs.html
+# Kafka Connect worker configuration
+# This file is part of a workflow that creates an authenticated connection to the Kafka broker.
+# For config options, see https://docs.confluent.io/platform/current/connect/references/allconfigs.html#worker-configuration-properties
+# See also: https://kafka.apache.org/documentation/#adminclientconfigs
 
 bootstrap.servers=usdf-alert-stream-dev.lsst.cloud:9094
 plugin.path=/usr/local/share/kafka/plugins
@@ -32,8 +34,8 @@ sasl.jaas.config=org.apache.kafka.common.security.scram.ScramLoginModule require
    username="pittgoogle-idfint"\
    password="KAFKA_PASSWORD";
 
-# settings with `consumer` prefixes are passed through to the Kafka consumer
-# consumer.group.id must begin with our username: pittgoogle-idfint
+# settings with `consumer.` prefixes are passed through to the Kafka consumer
+# for Rubin, consumer.group.id must begin with our username: pittgoogle-idfint
 consumer.group.id=pittgoogle-idfint-kafka-pubsub-connector
 consumer.auto.offset.reset=earliest
 consumer.sasl.mechanism=SCRAM-SHA-512