Release/0.2.4

.github/workflows/ci.yml

@@ -16,6 +16,8 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 11
+      - name: Install sbt
+        uses: sbt/setup-sbt@v1
       - name: Check Scala formatting
         run: sbt scalafmtCheckAll scalafmtSbtCheck
       - name: Run tests
@@ -39,6 +41,8 @@ jobs:
         uses: actions/setup-java@v1
         with:
           java-version: 11
+      - name: Install sbt
+        uses: sbt/setup-sbt@v1
       - name: Docker login
         uses: docker/login-action@v1
         with:

CHANGELOG

@@ -1,3 +1,9 @@
+Version 0.2.4 (2024-10-14)
+--------------------------
+Bump snowflake-ingest-sdk to 2.2.2 (#47)
+Add sbt installation step to CI (#47)
+Upgrade dependencies to fix some vulnerabilities (#49)
+
 Version 0.2.3 (2024-06-12)
 --------------------------
 Bump aws-sdk-2 to 2.25.16 (#40)

README.md

@@ -19,7 +19,7 @@ Basic usage:
 ```bash
 docker run \
   -v /path/to/config.hocon:/var/config.hocon \
-  snowplow/snowflake-loader-kafka:0.2.3 \
+  snowplow/snowflake-loader-kafka:0.2.4 \
   --config /var/config.hocon
 ```
 
@@ -30,7 +30,7 @@ The GCP snowflake loader reads the stream of enriched events from Pubsub.
 ```bash
 docker run \
   -v /path/to/config.hocon:/var/config.hocon \
-  snowplow/snowflake-loader-pubsub:0.2.3 \
+  snowplow/snowflake-loader-pubsub:0.2.4 \
   --config /var/config.hocon
 ```
 
@@ -41,7 +41,7 @@ The AWS snowflake loader reads the stream of enriched events from Kinesis.
 ```bash
 docker run \
   -v /path/to/config.hocon:/var/config.hocon \
-  snowplow/snowflake-loader-kinesis:0.2.3 \
+  snowplow/snowflake-loader-kinesis:0.2.4 \
   --config /var/config.hocon
 ```
 
@@ -71,7 +71,7 @@ Licensed under the [Snowplow Limited Use License Agreement][license]. _(If you a
 [build-image]: https://github.com/snowplow-incubator/snowplow-snowflake-streaming-loader/workflows/CI/badge.svg
 [build]: https://github.com/snowplow-incubator/snowplow-snowflake-streaming-loader/actions/workflows/ci.yml
 
-[release-image]: https://img.shields.io/badge/release-0.2.3-blue.svg?style=flat
+[release-image]: https://img.shields.io/badge/release-0.2.4-blue.svg?style=flat
 [releases]: https://github.com/snowplow-incubator/snowplow-snowflake-streaming-loader/releases
 
 [license]: https://docs.snowplow.io/limited-use-license-1.0

modules/core/src/main/scala/com.snowplowanalytics.snowplow.snowflake/processing/Channel.scala

@@ -204,8 +204,10 @@ object Channel {
     props.setProperty(ParameterProvider.ENABLE_SNOWPIPE_STREAMING_METRICS, "false")
 
     // Disable SDK's background flushing because we manage it ourselves
-    props.setProperty(ParameterProvider.BUFFER_FLUSH_INTERVAL_IN_MILLIS, Long.MaxValue.toString)
     props.setProperty(ParameterProvider.BUFFER_FLUSH_CHECK_INTERVAL_IN_MILLIS, Long.MaxValue.toString)
+    // Max allowed value for MAX_CLIENT_LAG is 10 min. However, background flushing is disabled with above
+    // line. Therefore, it isn't very important what value we use in here.
+    props.setProperty(ParameterProvider.MAX_CLIENT_LAG, "600000")
     props.setProperty(ParameterProvider.INSERT_THROTTLE_INTERVAL_IN_MILLIS, "0")
     props.setProperty(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_PERCENTAGE, "0")
     props.setProperty(ParameterProvider.INSERT_THROTTLE_THRESHOLD_IN_BYTES, "0")

project/Dependencies.scala

@@ -25,13 +25,15 @@ object Dependencies {
     val slf4j     = "2.0.7"
     val azureSdk  = "1.9.1"
     val sentry    = "6.25.2"
-    val snowflake = "2.0.3"
+    val snowflake = "2.2.2"
     val jaxb      = "2.3.1"
     val awsSdk2   = "2.25.16"
     val netty     = "4.1.100.Final" // Version override
     val reactor   = "1.0.39" // Version override
     val snappy    = "1.1.10.4" // Version override
     val nimbusJwt = "9.37.2" // Version override
+    val jackson   = "2.15.0" // Version override
+    val protobuf  = "3.25.5" // Version override
 
     // Snowplow
     val streams = "0.7.0"
@@ -51,16 +53,19 @@ object Dependencies {
   val doobie            = "org.tpolecat"     %% "doobie-core"          % V.doobie
 
   // java
-  val slf4j           = "org.slf4j"               % "slf4j-simple"         % V.slf4j
-  val azureIdentity   = "com.azure"               % "azure-identity"       % V.azureSdk
-  val sentry          = "io.sentry"               % "sentry"               % V.sentry
-  val snowflakeIngest = "net.snowflake"           % "snowflake-ingest-sdk" % V.snowflake
-  val jaxb            = "javax.xml.bind"          % "jaxb-api"             % V.jaxb
-  val stsSdk2         = "software.amazon.awssdk"  % "sts"                  % V.awsSdk2
-  val nettyCodecHttp  = "io.netty"                % "netty-codec-http2"    % V.netty
-  val reactorNetty    = "io.projectreactor.netty" % "reactor-netty-http"   % V.reactor
-  val snappyJava      = "org.xerial.snappy"       % "snappy-java"          % V.snappy
-  val nimbusJoseJwt   = "com.nimbusds"            % "nimbus-jose-jwt"      % V.nimbusJwt
+  val slf4j           = "org.slf4j"                  % "slf4j-simple"         % V.slf4j
+  val azureIdentity   = "com.azure"                  % "azure-identity"       % V.azureSdk
+  val sentry          = "io.sentry"                  % "sentry"               % V.sentry
+  val snowflakeIngest = "net.snowflake"              % "snowflake-ingest-sdk" % V.snowflake
+  val jaxb            = "javax.xml.bind"             % "jaxb-api"             % V.jaxb
+  val stsSdk2         = "software.amazon.awssdk"     % "sts"                  % V.awsSdk2
+  val nettyCodecHttp  = "io.netty"                   % "netty-codec-http2"    % V.netty
+  val reactorNetty    = "io.projectreactor.netty"    % "reactor-netty-http"   % V.reactor
+  val snappyJava      = "org.xerial.snappy"          % "snappy-java"          % V.snappy
+  val nimbusJoseJwt   = "com.nimbusds"               % "nimbus-jose-jwt"      % V.nimbusJwt
+  val jacksonCore     = "com.fasterxml.jackson.core" % "jackson-core"         % V.jackson
+  val protobufJava    = "com.google.protobuf"        % "protobuf-java"        % V.protobuf
+  val protobufUtil    = "com.google.protobuf"        % "protobuf-java-util"   % V.protobuf
 
   val streamsCore = "com.snowplowanalytics" %% "streams-core"   % V.streams
   val kinesis     = "com.snowplowanalytics" %% "kinesis"        % V.streams
@@ -86,6 +91,7 @@ object Dependencies {
     snowflakeIngest,
     doobie,
     circeGenericExtra,
+    jacksonCore,
     specs2,
     catsEffectSpecs2,
     catsEffectTestkit,
@@ -107,6 +113,8 @@ object Dependencies {
 
   val pubsubDependencies = Seq(
     pubsub,
+    protobufJava,
+    protobufUtil,
     jaxb  % Runtime,
     slf4j % Runtime,
     specs2,
@@ -115,6 +123,7 @@ object Dependencies {
 
   val kinesisDependencies = Seq(
     kinesis,
+    protobufJava,
     jaxb    % Runtime,
     slf4j   % Runtime,
     stsSdk2 % Runtime,