Skip to content

Upgrade dependencies #92

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 2 commits into from
Oct 31, 2024
Merged

Upgrade dependencies #92

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
790e67b
Upgrade dependencies
oguzhanunlu Oct 29, 2024
7b55bf4
fix jackson version mismatch
oguzhanunlu Oct 30, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 37 additions & 30 deletions project/Dependencies.scala
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,50 +15,51 @@ object Dependencies {
object Spark {

// A version of Spark which is compatible with the current version of Iceberg and Delta
val forIcebergDelta = "3.5.1"
val forIcebergDelta = "3.5.3"
val forIcebergDeltaMinor = "3.5"

// Hudi can use a different version of Spark because we bundle a separate Docker image
// This version of Spark must be compatible with the current version of Hudi
val forHudi = "3.5.1"
val forHudi = "3.5.3"
val forHudiMinor = "3.5"
}

// Scala
val catsEffect = "3.5.4"
val decline = "2.4.1"
val circe = "0.14.3"
val http4s = "0.23.16"
val circe = "0.14.4"
val http4s = "0.23.29"
val betterMonadicFor = "0.3.1"

// Spark
val delta = "3.2.0"
val hudi = "0.15.0"
val hudiAws = "1.0.0-beta2"
val iceberg = "1.5.2"
val hadoop = "3.4.0"
val gcsConnector = "hadoop3-2.2.25"
val biglakeIceberg = "0.1.0"
val hive = "3.1.3"
val delta = "3.2.1"
val hudi = "0.15.0"
val hudiAws = "1.0.0-beta2"
val iceberg = "1.6.1"
val hadoop = "3.4.1"
val gcsConnector = "hadoop3-2.2.25"
val hive = "3.1.3"

// java
val slf4j = "2.0.13"
val azureSdk = "1.11.4"
val sentry = "6.25.2"
val awsSdk1 = "1.12.646"
val awsSdk2 = "2.25.16" // Match common-streams
val azureSdk = "1.12.2"
val sentry = "7.16.0"
val awsSdk1 = "1.12.777"
val awsSdk2 = "2.29.0" // Match common-streams
val awsRegistry = "1.1.20"

// Snowplow
val streams = "0.8.0-M5"
val igluClient = "3.2.0"
val streams = "0.8.0"
val igluClient = "4.0.0"

// Transitive overrides
val protobuf = "3.25.1"
val protobuf = "3.25.5"
val snappy = "1.1.10.5"
val thrift = "0.18.1"
val netty = "4.1.109.Final"
val pubsubSdk = "1.132.3"
val thrift = "0.21.0"
val netty = "4.1.114.Final"
val pubsubSdk = "1.134.1"
val avro = "1.11.4"
val jackson = "2.17.2"

// tests
val specs2 = "4.20.0"
Expand Down Expand Up @@ -103,13 +104,16 @@ object Dependencies {
val awsRegistry = "software.amazon.glue" % "schema-registry-serde" % V.awsRegistry

// transitive overrides
val protobuf = "com.google.protobuf" % "protobuf-java" % V.protobuf
val snappy = "org.xerial.snappy" % "snappy-java" % V.snappy
val hadoopYarn = "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % V.hadoop
val thrift = "org.apache.thrift" % "libthrift" % V.thrift
val netty = "io.netty" % "netty-all" % V.netty
val awsCore = "com.amazonaws" % "aws-java-sdk-core" % V.awsSdk1
val pubsubSdk = "com.google.cloud" % "google-cloud-pubsub" % V.pubsubSdk
val protobuf = "com.google.protobuf" % "protobuf-java" % V.protobuf
val snappy = "org.xerial.snappy" % "snappy-java" % V.snappy
val hadoopYarn = "org.apache.hadoop" % "hadoop-yarn-server-resourcemanager" % V.hadoop
val thrift = "org.apache.thrift" % "libthrift" % V.thrift
val netty = "io.netty" % "netty-all" % V.netty
val awsCore = "com.amazonaws" % "aws-java-sdk-core" % V.awsSdk1
val pubsubSdk = "com.google.cloud" % "google-cloud-pubsub" % V.pubsubSdk
val avro = "org.apache.avro" % "avro" % V.avro
val jacksonDT = "com.fasterxml.jackson.datatype" % "jackson-datatype-jsr310" % V.jackson
val jacksonMS = "com.fasterxml.jackson.module" %% "jackson-module-scala" % V.jackson

// snowplow
val streamsCore = "com.snowplowanalytics" %% "streams-core" % V.streams
Expand All @@ -129,7 +133,8 @@ object Dependencies {
slf4j % Runtime,
protobuf % Runtime,
netty % Runtime,
snappy % Runtime
snappy % Runtime,
avro % Runtime
)

val icebergDeltaRuntimeDependencies = Seq(
Expand All @@ -147,6 +152,8 @@ object Dependencies {
Spark.coreForIcebergDelta % Provided,
Spark.sqlForIcebergDelta % Provided,
iceberg % Provided,
jacksonDT,
jacksonMS,
igluClientHttp4s,
decline,
sentry,
Expand Down
Loading