With AQE

snowplow-incubator · Jan 7, 2025 · fbd3218 · fbd3218
1 parent 2f2482f
commit fbd3218
Show file tree

Hide file tree

Showing 2 changed files with 3 additions and 2 deletions.
diff --git a/modules/core/src/main/resources/reference.conf b/modules/core/src/main/resources/reference.conf
@@ -128,7 +128,7 @@
       "spark.memory.storageFraction": "0"
       "spark.databricks.delta.autoCompact.enabled": "false"
       "spark.scheduler.mode": "FAIR"
-      "spark.sql.adaptive.enabled": "false"
+      "spark.sql.adaptive.enabled": "true"
     }
     "gcpUserAgent": ${gcpUserAgent}
   }

diff --git a/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala b/modules/core/src/main/scala/com.snowplowanalytics.snowplow.lakes/processing/SparkUtils.scala
@@ -119,5 +119,6 @@ private[processing] object SparkUtils {
    * the intermediate dataframes.
    */
   private def chooseWriterParallelism(): Int =
-    (Runtime.getRuntime.availableProcessors - 1).max(1)
+    200
+  // (Runtime.getRuntime.availableProcessors - 1).max(1)
 }