Update collector to 3.2.1

Author: Mike Robins

docker-compose.yml

@@ -17,7 +17,7 @@ services:
 
   stream-collector:
     container_name: "snowplow-stream-collector"
-    image: snowplow/scala-stream-collector-kinesis:3.2.0-distroless
+    image: snowplow/scala-stream-collector-kinesis:3.2.1-distroless
     command: ["--config", "/snowplow/collector/config.hocon"]
     depends_on:
       - localstack

loaders/lake_loader_config_iceberg_s3.hocon

@@ -0,0 +1,53 @@
+{
+  "license": {
+    "accept": ${ACCEPT_LICENSE}
+  },
+  "input": {
+    "streamName": "enriched-good"
+    "appName": "lake-loader-iceberg"
+    "customEndpoint": "http://localhost.localstack.cloud:4566"
+    "initialPosition": {
+      "type": "LATEST"
+    }
+    "retrievalMode": {
+      "type": "FanOut"
+    }
+    "workerIdentifier": ${HOSTNAME}
+    "leaseDuration": "10 seconds"
+  }
+
+  "output": {
+
+    "good": {
+      "type": "Iceberg",
+      "database": "atomic"
+      "table": "events"
+      "location": "s3a://snowplow-lake-loader/events" # this can also be S3!
+      "catalog": {
+        "type": "Hadoop"
+      }
+    }
+
+    "bad": {
+      "streamName": "bad"
+    }
+  }
+
+  "spark": {
+
+    # -- How many times a Spark task should be retried in case of failure.
+    "taskRetries": 3
+    # -- Any valid spark configuration key/value.
+    # -- This can be blank in most setups because the loader already sets sensible defaults.
+    # https://github.com/apache/iceberg/commit/fa6403b1888847ce04de50c6b159dd43fdcb4590
+    "conf": {
+      "fs.s3a.endpoint": "http://s3.localhost.localstack.cloud:4566"
+    }
+  }
+
+  "windowing": "15 seconds" # deliberately short period for frequent writes
+  "inMemBatchBytes": 1000
+  "telemetry": {
+    "disable": true
+  }
+}