diff --git a/modules/core/src/main/resources/reference.conf b/modules/core/src/main/resources/reference.conf index 33d8526..1a91488 100644 --- a/modules/core/src/main/resources/reference.conf +++ b/modules/core/src/main/resources/reference.conf @@ -128,7 +128,7 @@ "spark.memory.storageFraction": "0" "spark.databricks.delta.autoCompact.enabled": "false" "spark.scheduler.mode": "FAIR" - "spark.sql.adaptive.enabled": "false" + "spark.sql.adaptive.enabled": "true" } "gcpUserAgent": ${gcpUserAgent} } diff --git a/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala b/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala index 2dbe4bf..31c437d 100644 --- a/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala +++ b/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala @@ -119,5 +119,6 @@ private[processing] object SparkUtils { * the intermediate dataframes. */ private def chooseWriterParallelism(): Int = - (Runtime.getRuntime.availableProcessors - 1).max(1) + 200 + // (Runtime.getRuntime.availableProcessors - 1).max(1) }