From 5faa70eacc1d769568252cbcd005dbf3315f0674 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:25:22 +0200 Subject: [PATCH 1/4] chore(spark): update hdfs dependencies --- spark-k8s/boil-config.toml | 48 +++++++++++++++++++------------------- 1 file changed, 24 insertions(+), 24 deletions(-) diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index a11731ba3..b78b5a6fc 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -1,43 +1,43 @@ [versions."3.5.6".local-images] -"hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. +"hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" -hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector. +hbase = "2.6.2" vector = "0.49.0" [versions."3.5.6".build-arguments] python-version = "3.11" -aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 -azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 -azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 -jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 -stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop +azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop +azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop +jackson-dataformat-xml-version = "2.15.2" +stax2-api-version = "4.2.1" +woodstox-core-version = "6.5.1" jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" [versions."3.5.7".local-images] -"hadoop/hadoop" = "3.4.1" # Current Stackable LTS version. Source of the AWS and Azure artifacts to Spark's classpath. +"hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" -hbase = "2.6.2" # Current Stackable LTS version. Used to build the HBase connector. +hbase = "2.6.2" vector = "0.49.0" [versions."3.5.7".build-arguments] python-version = "3.11" -aws-java-sdk-bundle-version = "2.24.6" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-aws/3.4.1 -azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.4.1 -azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 -jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.2 -stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop +azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop +azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop +jackson-dataformat-xml-version = "2.15.2" +stax2-api-version = "4.2.1" +woodstox-core-version = "6.5.1" jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" [versions."4.0.1".local-images] -"hadoop/hadoop" = "3.4.1" +"hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" hbase = "2.6.2" @@ -45,12 +45,12 @@ vector = "0.49.0" [versions."4.0.1".build-arguments] python-version = "3.11" -aws-java-sdk-bundle-version = "2.24.6" -azure-storage-version = "7.0.1" # https://mvnrepository.com/artifact/org.apache.hadoop/hadoop-azure/3.3.4 -azure-keyvault-core-version = "1.0.0" # https://mvnrepository.com/artifact/com.microsoft.azure/azure-storage/7.0.1 -jackson-dataformat-xml-version = "2.15.2" # https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/3.5.1 -stax2-api-version = "4.2.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 -woodstox-core-version = "6.5.1" # https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop +azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop +azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop +jackson-dataformat-xml-version = "2.15.2" +stax2-api-version = "4.2.1" +woodstox-core-version = "6.5.1" jmx-exporter-version = "1.3.0" tini-version = "0.19.0" -hbase-connector-version = "1.0.1" # This is not supported in Spark 4 yet. +hbase-connector-version = "1.0.1" # This is not supported in Spark 4 yet. From 63300ead8ee30ee85fe5a188a6803cfc9dc08c2e Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 6 Oct 2025 11:40:26 +0200 Subject: [PATCH 2/4] bump hbase version --- spark-k8s/boil-config.toml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index b78b5a6fc..91a420cb2 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -2,7 +2,7 @@ "hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" -hbase = "2.6.2" +hbase = "2.6.3" vector = "0.49.0" [versions."3.5.6".build-arguments] @@ -21,7 +21,7 @@ hbase-connector-version = "1.0.1" "hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" -hbase = "2.6.2" +hbase = "2.6.3" vector = "0.49.0" [versions."3.5.7".build-arguments] @@ -40,7 +40,7 @@ hbase-connector-version = "1.0.1" "hadoop/hadoop" = "3.4.2" java-base = "17" java-devel = "17" -hbase = "2.6.2" +hbase = "2.6.3" vector = "0.49.0" [versions."4.0.1".build-arguments] From 505f9d5d0f8b6c8beb90c999e887cd16f39c8a98 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 6 Oct 2025 14:18:10 +0200 Subject: [PATCH 3/4] add maven repo links for logging libs --- spark-k8s/boil-config.toml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index 91a420cb2..b6cdf6e57 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -10,9 +10,9 @@ python-version = "3.11" aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" -stax2-api-version = "4.2.1" -woodstox-core-version = "6.5.1" +jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 +stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" @@ -29,9 +29,9 @@ python-version = "3.11" aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" -stax2-api-version = "4.2.1" -woodstox-core-version = "6.5.1" +jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 +stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 +woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" @@ -48,9 +48,9 @@ python-version = "3.11" aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" -stax2-api-version = "4.2.1" -woodstox-core-version = "6.5.1" +jackson-dataformat-xml-version = "2.18.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.13/4.0.1 +stax2-api-version = "4.2.2" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 +woodstox-core-version = "7.0.0" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.18.2 jmx-exporter-version = "1.3.0" tini-version = "0.19.0" hbase-connector-version = "1.0.1" # This is not supported in Spark 4 yet. From 008ccf796f996f833ab640758200961d9d66ac95 Mon Sep 17 00:00:00 2001 From: Razvan-Daniel Mihai <84674+razvan@users.noreply.github.com> Date: Mon, 6 Oct 2025 14:33:09 +0200 Subject: [PATCH 4/4] Update spark-k8s/boil-config.toml Co-authored-by: Sebastian Bernauer --- spark-k8s/boil-config.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/spark-k8s/boil-config.toml b/spark-k8s/boil-config.toml index b6cdf6e57..6dac4f272 100644 --- a/spark-k8s/boil-config.toml +++ b/spark-k8s/boil-config.toml @@ -10,7 +10,7 @@ python-version = "3.11" aws-java-sdk-bundle-version = "2.29.52" # needs to match the version shipped by Hadoop azure-storage-version = "7.0.1" # needs to match the version shipped by Hadoop azure-keyvault-core-version = "1.0.0" # needs to match the version shipped by Hadoop -jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.7 +jackson-dataformat-xml-version = "2.15.2" # needs to match the version shipped by Spark https://mvnrepository.com/artifact/org.apache.spark/spark-core_2.12/3.5.6 stax2-api-version = "4.2.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 woodstox-core-version = "6.5.1" # needs to match the jackson version https://mvnrepository.com/artifact/com.fasterxml.jackson.dataformat/jackson-dataformat-xml/2.15.2 jmx-exporter-version = "1.3.0"