Upgrade to Spark 3.5.4

Signed-off-by: Jacob Salway <[email protected]>
kubeflow · Jan 9, 2025 · f9fd865 · f9fd865
1 parent 92deff0
commit f9fd865
Show file tree

Hide file tree

Showing 18 changed files with 47 additions and 46 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-ARG SPARK_IMAGE=spark:3.5.3
+ARG SPARK_IMAGE=spark:3.5.4
 
 FROM golang:1.23.1 AS builder
 

diff --git a/README.md b/README.md
@@ -77,6 +77,7 @@ The following table lists the most recent few versions of the operator.
 
 | Operator Version      | API Version | Kubernetes Version | Base Spark Version |
 |-----------------------|-------------|--------------------|--------------------|
+| `v2.1.0`              | `v1beta2`   | 1.16+              | `3.5.3`            |
 | `v2.0.x`              | `v1beta2`   | 1.16+              | `3.5.2`            |
 | `v1beta2-1.6.x-3.5.0` | `v1beta2`   | 1.16+              | `3.5.0`            |
 | `v1beta2-1.5.x-3.5.0` | `v1beta2`   | 1.16+              | `3.5.0`            |

diff --git a/config/samples/v1beta1_sparkapplication.yaml b/config/samples/v1beta1_sparkapplication.yaml
@@ -8,16 +8,16 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     serviceAccount: spark-operator-spark
   executor:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     instances: 1
diff --git a/config/samples/v1beta2_scheduledsparkapplication.yaml b/config/samples/v1beta2_scheduledsparkapplication.yaml
@@ -11,23 +11,23 @@ spec:
   template:
     type: Scala
     mode: cluster
-    image: spark:3.5.3
+    image: spark:3.5.4
     imagePullPolicy: IfNotPresent
     mainClass: org.apache.spark.examples.SparkPi
     mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-    sparkVersion: 3.5.3
+    sparkVersion: 3.5.4
     restartPolicy:
       type: Never
     driver:
       labels:
-        version: 3.5.3
+        version: 3.5.4
       cores: 1
       coreLimit: 1200m
       memory: 512m
       serviceAccount: spark-operator-spark
     executor:
       labels:
-        version: 3.5.3
+        version: 3.5.4
       instances: 1
       cores: 1
       coreLimit: 1200m

diff --git a/config/samples/v1beta2_sparkapplication.yaml b/config/samples/v1beta2_sparkapplication.yaml
@@ -8,16 +8,16 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     serviceAccount: spark-operator-spark
   executor:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     instances: 1
diff --git a/examples/spark-pi-configmap.yaml b/examples/spark-pi-configmap.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   restartPolicy:
     type: Never
   volumes:

diff --git a/examples/spark-pi-custom-resource.yaml b/examples/spark-pi-custom-resource.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   restartPolicy:
     type: Never
   driver:

diff --git a/examples/spark-pi-dynamic-allocation.yaml b/examples/spark-pi-dynamic-allocation.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi-kube-scheduler.yaml b/examples/spark-pi-kube-scheduler.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi-pod-template.yaml b/examples/spark-pi-pod-template.yaml
@@ -57,8 +57,8 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  sparkVersion: 3.5.3
-  image: spark:3.5.3
+  sparkVersion: 3.5.4
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
   mainClass: org.apache.spark.examples.SparkPi
@@ -68,9 +68,9 @@ spec:
     template:
       metadata:
         labels:
-          spark.apache.org/version: 3.5.3
+          spark.apache.org/version: 3.5.4
         annotations:
-          spark.apache.org/version: 3.5.3
+          spark.apache.org/version: 3.5.4
       spec:
         containers:
         - name: spark-kubernetes-driver
@@ -133,9 +133,9 @@ spec:
     template:
       metadata:
         labels:
-          spark.apache.org/version: 3.5.3
+          spark.apache.org/version: 3.5.4
         annotations:
-          spark.apache.org/version: 3.5.3
+          spark.apache.org/version: 3.5.4
       spec:
         containers:
         - name: spark-kubernetes-executor

diff --git a/examples/spark-pi-prometheus.yaml b/examples/spark-pi-prometheus.yaml
@@ -22,27 +22,27 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: {IMAGE_REGISTRY}/{IMAGE_REPOSITORY}/spark:3.5.3-gcs-prometheus
+  image: {IMAGE_REGISTRY}/{IMAGE_REPOSITORY}/spark:3.5.4-gcs-prometheus
   imagePullPolicy: Always
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
   arguments:
   - "100000"
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   restartPolicy:
     type: Never
   driver:
     cores: 1
     memory: 512m
     labels:
-      version: 3.5.3
+      version: 3.5.4
     serviceAccount: spark-operator-spark
   executor:
     cores: 1
     instances: 1
     memory: 512m
     labels:
-      version: 3.5.3
+      version: 3.5.4
   monitoring:
     exposeDriverMetrics: true
     exposeExecutorMetrics: true

diff --git a/examples/spark-pi-python.yaml b/examples/spark-pi-python.yaml
@@ -22,10 +22,10 @@ spec:
   type: Python
   pythonVersion: "3"
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainApplicationFile: local:///opt/spark/examples/src/main/python/pi.py
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi-scheduled.yaml b/examples/spark-pi-scheduled.yaml
@@ -25,11 +25,11 @@ spec:
   template:
     type: Scala
     mode: cluster
-    image: spark:3.5.3
+    image: spark:3.5.4
     imagePullPolicy: IfNotPresent
     mainClass: org.apache.spark.examples.SparkPi
     mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-    sparkVersion: 3.5.3
+    sparkVersion: 3.5.4
     restartPolicy:
       type: Never
     driver:

diff --git a/examples/spark-pi-ttl.yaml b/examples/spark-pi-ttl.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   timeToLiveSeconds: 30
   driver:
     cores: 1

diff --git a/examples/spark-pi-volcano.yaml b/examples/spark-pi-volcano.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi-yunikorn.yaml b/examples/spark-pi-yunikorn.yaml
@@ -21,11 +21,11 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     cores: 1
     memory: 512m

diff --git a/examples/spark-pi.yaml b/examples/spark-pi.yaml
@@ -21,22 +21,22 @@ metadata:
 spec:
   type: Scala
   mode: cluster
-  image: spark:3.5.3
+  image: spark:3.5.4
   imagePullPolicy: IfNotPresent
   mainClass: org.apache.spark.examples.SparkPi
   mainApplicationFile: local:///opt/spark/examples/jars/spark-examples.jar
   arguments:
   - "5000"
-  sparkVersion: 3.5.3
+  sparkVersion: 3.5.4
   driver:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     cores: 1
     memory: 512m
     serviceAccount: spark-operator-spark
   executor:
     labels:
-      version: 3.5.3
+      version: 3.5.4
     instances: 1
     cores: 1
     memory: 512m
diff --git a/spark-docker/Dockerfile b/spark-docker/Dockerfile
@@ -14,7 +14,7 @@
 # limitations under the License.
 #
 
-ARG SPARK_IMAGE=spark:3.5.3
+ARG SPARK_IMAGE=spark:3.5.4
 FROM ${SPARK_IMAGE}
 
 # Switch to user root so we can add additional jars and configuration files.