diff --git a/CHANGELOG.md b/CHANGELOG.md index 90ea254e..4b923daa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,7 @@ All notable changes to this project will be documented in this file. - Use `--file-log-rotation-period` (or `FILE_LOG_ROTATION_PERIOD`) to configure the frequency of rotation. - Use `--console-log-format` (or `CONSOLE_LOG_FORMAT`) to set the format to `plain` (default) or `json`. - Expose history and connect services via listener classes ([#562]). +- Support for Spark 3.5.6 ([#580]). ### Changed @@ -56,6 +57,7 @@ All notable changes to this project will be documented in this file. [#572]: https://github.com/stackabletech/spark-k8s-operator/pull/572 [#573]: https://github.com/stackabletech/spark-k8s-operator/pull/573 [#574]: https://github.com/stackabletech/spark-k8s-operator/pull/574 +[#580]: https://github.com/stackabletech/spark-k8s-operator/pull/580 [#575]: https://github.com/stackabletech/spark-k8s-operator/pull/575 ## [25.3.0] - 2025-03-21 diff --git a/docs/modules/spark-k8s/examples/example-history-app.yaml b/docs/modules/spark-k8s/examples/example-history-app.yaml index d5825f14..19642f76 100644 --- a/docs/modules/spark-k8s/examples/example-history-app.yaml +++ b/docs/modules/spark-k8s/examples/example-history-app.yaml @@ -5,7 +5,7 @@ metadata: name: spark-pi-s3-1 spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 pullPolicy: IfNotPresent mode: cluster mainClass: org.apache.spark.examples.SparkPi diff --git a/docs/modules/spark-k8s/examples/example-history-server.yaml b/docs/modules/spark-k8s/examples/example-history-server.yaml index 3d7365c9..48325019 100644 --- a/docs/modules/spark-k8s/examples/example-history-server.yaml +++ b/docs/modules/spark-k8s/examples/example-history-server.yaml @@ -5,7 +5,7 @@ metadata: name: spark-history spec: image: - productVersion: 3.5.5 + productVersion: 3.5.6 logFileDirectory: # <1> s3: prefix: eventlogs/ # <2> diff --git a/docs/modules/spark-k8s/examples/example-spark-connect.yaml b/docs/modules/spark-k8s/examples/example-spark-connect.yaml index c89cd2e1..3b6d82af 100644 --- a/docs/modules/spark-k8s/examples/example-spark-connect.yaml +++ b/docs/modules/spark-k8s/examples/example-spark-connect.yaml @@ -5,7 +5,7 @@ metadata: name: spark-connect # <1> spec: image: - productVersion: "3.5.5" # <2> + productVersion: "3.5.6" # <2> pullPolicy: IfNotPresent args: - "--package org.apache.iceberg:iceberg-spark-runtime-3.5_2.12:1.8.1" # <3> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml index 09ae9893..3f16e07e 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-configmap.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.1.0.jar # <3> mainClass: tech.stackable.demo.spark.NYTLCReport diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml index 8c60fd38..58a8348f 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml @@ -7,7 +7,7 @@ metadata: spec: image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1> sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py # <2> args: diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml index c1b3f124..1b44a541 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-pvc.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: s3a://stackable-spark-k8s-jars/jobs/ny-tlc-report-1.0-SNAPSHOT.jar # <1> mainClass: org.example.App # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml index 9d54537c..a06ae5e5 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-s3-private.yaml @@ -5,7 +5,7 @@ metadata: name: example-sparkapp-s3-private spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: s3a://my-bucket/spark-examples.jar # <1> mainClass: org.apache.spark.examples.SparkPi # <2> diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml index db399938..fc67a9fb 100644 --- a/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml +++ b/docs/modules/spark-k8s/examples/example-sparkapp-streaming.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/streaming/hdfs_wordcount.py args: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh index c51c3fd8..aad89c62 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh @@ -56,7 +56,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 index 7d9fde04..90d6783f 100755 --- a/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 +++ b/docs/modules/spark-k8s/examples/getting_started/getting_started.sh.j2 @@ -56,7 +56,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster mainApplicationFile: local:///stackable/spark/examples/src/main/python/pi.py driver: diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc index 1735246d..a5bbd918 100644 --- a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc +++ b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc @@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver: [source, Dockerfile] ---- -FROM oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable25.3.0 # <1> +FROM oci.stackable.tech/sdp/spark-k8s:3.5.6-stackable25.3.0 # <1> RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar" ---- @@ -75,8 +75,8 @@ metadata: name: spark-jdbc spec: sparkImage: - custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev" # <1> - productVersion: "3.5.5" # <2> + custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev" # <1> + productVersion: "3.5.6" # <2> pullPolicy: IfNotPresent # <3> ... ---- diff --git a/docs/modules/spark-k8s/partials/supported-versions.adoc b/docs/modules/spark-k8s/partials/supported-versions.adoc index cdabb982..a132ec2d 100644 --- a/docs/modules/spark-k8s/partials/supported-versions.adoc +++ b/docs/modules/spark-k8s/partials/supported-versions.adoc @@ -3,4 +3,5 @@ // Stackable Platform documentation. // Please sort the versions in descending order (newest first) -- 3.5.5 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) +- 3.5.5 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (Deprecated) +- 3.5.6 (Hadoop 3.3.4, Scala 2.12, Python 3.11, Java 17) (LTS) diff --git a/examples/README-examples.md b/examples/README-examples.md index ced40e25..33756eb4 100644 --- a/examples/README-examples.md +++ b/examples/README-examples.md @@ -50,10 +50,10 @@ Several resources are needed in this store. These can be loaded like this: ````text kubectl exec minio-mc-0 -- sh -c 'mc alias set test-minio http://test-minio:9000/' -kubectl cp examples/ny-tlc-report-1.1.0-3.5.5.jar minio-mc-0:/tmp +kubectl cp examples/ny-tlc-report-1.1.0-3.5.6.jar minio-mc-0:/tmp kubectl cp apps/ny_tlc_report.py minio-mc-0:/tmp kubectl cp examples/yellow_tripdata_2021-07.csv minio-mc-0:/tmp -kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.5.jar test-minio/my-bucket +kubectl exec minio-mc-0 -- mc cp /tmp/ny-tlc-report-1.1.0-3.5.6.jar test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/ny_tlc_report.py test-minio/my-bucket kubectl exec minio-mc-0 -- mc cp /tmp/yellow_tripdata_2021-07.csv test-minio/my-bucket ```` diff --git a/examples/ny-tlc-report-external-dependencies.yaml b/examples/ny-tlc-report-external-dependencies.yaml index 3aaa2060..209df660 100644 --- a/examples/ny-tlc-report-external-dependencies.yaml +++ b/examples/ny-tlc-report-external-dependencies.yaml @@ -6,7 +6,7 @@ metadata: namespace: default spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 pullPolicy: IfNotPresent mode: cluster mainApplicationFile: s3a://my-bucket/ny_tlc_report.py diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml index 1578be11..93f49fc0 100644 --- a/examples/ny-tlc-report-image.yaml +++ b/examples/ny-tlc-report-image.yaml @@ -7,7 +7,7 @@ metadata: spec: # everything under /jobs will be copied to /stackable/spark/jobs image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 - sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.5-stackable0.0.0-dev + sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.6-stackable0.0.0-dev sparkImagePullPolicy: IfNotPresent mode: cluster mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py diff --git a/examples/ny-tlc-report.yaml b/examples/ny-tlc-report.yaml index 45e918a6..02afea5a 100644 --- a/examples/ny-tlc-report.yaml +++ b/examples/ny-tlc-report.yaml @@ -13,9 +13,9 @@ metadata: name: spark-ny-cm spec: sparkImage: - productVersion: 3.5.5 + productVersion: 3.5.6 mode: cluster - mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.5.jar + mainApplicationFile: s3a://my-bucket/ny-tlc-report-1.1.0-3.5.6.jar mainClass: tech.stackable.demo.spark.NYTLCReport volumes: - name: cm-job-arguments diff --git a/rust/operator-binary/src/crd/affinity.rs b/rust/operator-binary/src/crd/affinity.rs index ae11e403..ef719dbf 100644 --- a/rust/operator-binary/src/crd/affinity.rs +++ b/rust/operator-binary/src/crd/affinity.rs @@ -47,7 +47,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.5 + productVersion: 3.5.6 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/crd/history.rs b/rust/operator-binary/src/crd/history.rs index 86147901..87cdbcf1 100644 --- a/rust/operator-binary/src/crd/history.rs +++ b/rust/operator-binary/src/crd/history.rs @@ -425,7 +425,7 @@ mod test { name: spark-history spec: image: - productVersion: 3.5.5 + productVersion: 3.5.6 logFileDirectory: s3: prefix: eventlogs/ diff --git a/rust/operator-binary/src/history/config/jvm.rs b/rust/operator-binary/src/history/config/jvm.rs index 1dfb60c7..4022a3ad 100644 --- a/rust/operator-binary/src/history/config/jvm.rs +++ b/rust/operator-binary/src/history/config/jvm.rs @@ -67,7 +67,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.5 + productVersion: 3.5.6 logFileDirectory: s3: prefix: eventlogs/ @@ -100,7 +100,7 @@ mod tests { name: spark-history spec: image: - productVersion: 3.5.5 + productVersion: 3.5.6 logFileDirectory: s3: prefix: eventlogs/ diff --git a/tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2 b/tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2 index 067ea843..676b3a56 100644 --- a/tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2 +++ b/tests/templates/kuttl/spark-connect/20-run-connect-client.yaml.j2 @@ -60,7 +60,7 @@ spec: {% if test_scenario['values']['spark-connect-client'].find(",") > 0 %} image: "{{ test_scenario['values']['spark-connect-client'].split(',')[1] }}" {% else %} - image: oci.stackable.tech/sdp/spark-connect-client:{{ test_scenario['values']['spark-connect-client'] }}-stackable0.0.0-dev + image: oci.stackable.tech/stackable/spark-connect-client:{{ test_scenario['values']['spark-connect-client'] }}-stackable0.0.0-dev {% endif %} imagePullPolicy: IfNotPresent command: diff --git a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.2.jar b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.6.jar similarity index 90% rename from tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.2.jar rename to tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.6.jar index c88b521e..81c37ca1 100644 Binary files a/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.2.jar and b/tests/templates/kuttl/spark-ny-public-s3/ny-tlc-report-1.1.0-3.5.6.jar differ diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml index c96fd92d..8db6043a 100644 --- a/tests/test-definition.yaml +++ b/tests/test-definition.yaml @@ -6,21 +6,22 @@ dimensions: - name: spark values: - 3.5.5 + - 3.5.6 # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version # as in the example below. - # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev + # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-delta-lake values: - - 3.5.5 - # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev + - 3.5.6 + # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-connect values: - - 3.5.5 - # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev + - 3.5.6 + # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: spark-connect-client values: - - 3.5.5 - # - 3.5.5,oci.stackable.tech/sandbox/spark-k8s:3.5.5-stackable0.0.0-dev + - 3.5.6 + # - 3.5.6,oci.stackable.tech/sandbox/spark-k8s:3.5.6-stackable0.0.0-dev - name: hbase values: - 2.6.2