Chore: update demos for release

demos/airflow-scheduled-job/03-enable-and-run-spark-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-pyspark-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/airflow-scheduled-job/04-enable-and-run-date-dag.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: start-date-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           # N.B. it is possible for the scheduler to report that a DAG exists, only for the worker task to fail if a pod is unexpectedly
           # restarted. Additionally, the db-init job takes a few minutes to complete before the cluster is deployed. The wait/watch steps
           # below are not "water-tight" but add a layer of stability by at least ensuring that the db is initialized and ready and that

demos/data-lakehouse-iceberg-trino-spark/create-nifi-ingestion-job.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/instance=kafka -l app.kubernetes.io/name=kafka"]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/LakehouseKafkaIngest.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/create-spark-ingestion-job.yaml

@@ -12,11 +12,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-kafka
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for all kafka brokers to be ready' && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=kafka -l app.kubernetes.io/instance=kafka"]
       containers:
         - name: create-spark-ingestion-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ingestion-job.yaml"]
           volumeMounts:
             - name: manifest

demos/data-lakehouse-iceberg-trino-spark/create-trino-tables.yaml

@@ -9,11 +9,11 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for job load-test-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-test-data"]
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/data-lakehouse-iceberg-trino-spark/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/data-lakehouse-iceberg-trino-spark/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/end-to-end-security/create-spark-report.yaml

@@ -12,7 +12,7 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-trino-tables
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command:
             - bash
             - -euo
@@ -23,7 +23,7 @@ spec:
               kubectl wait --timeout=30m --for=condition=complete job/create-tables-in-trino
       containers:
         - name: create-spark-report
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command:
             - bash
             - -euo

demos/end-to-end-security/create-trino-tables.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-tables-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable23.11.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/hbase-hdfs-load-cycling-data/create-hfile-and-import-to-hbase.yaml

@@ -9,7 +9,7 @@ spec:
     spec:
       containers:
         - name: create-hfile-and-import-to-hbase
-          image: docker.stackable.tech/stackable/hbase:2.4.17-stackable24.3.0
+          image: docker.stackable.tech/stackable/hbase:2.4.18-stackable24.7.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/hbase-hdfs-load-cycling-data/distcp-cycling-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: distcp-cycling-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable24.3.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.7.0
           env:
             - name: HADOOP_USER_NAME
               value: stackable

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/Dockerfile

@@ -1,6 +1,6 @@
 # docker build -t docker.stackable.tech/demos/pyspark-k8s-with-scikit-learn:3.3.0-stackable0.0.0-dev .
 
-FROM docker.stackable.tech/stackable/pyspark-k8s:3.5.0-stackable24.3.0
+FROM docker.stackable.tech/stackable/pyspark-k8s:3.5.0-stackable24.7.0
 
 COPY requirements.txt .
 

demos/jupyterhub-pyspark-hdfs-anomaly-detection-taxi-data/load-test-data.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: load-ny-taxi-data
-          image: docker.stackable.tech/stackable/hadoop:3.3.6-stackable24.3.0
+          image: docker.stackable.tech/stackable/hadoop:3.4.0-stackable24.7.0
           command: ["bash", "-c", "/stackable/hadoop/bin/hdfs dfs -mkdir -p /ny-taxi-data/raw \
           && cd /tmp \
           && for month in 2020-09; do \

demos/nifi-kafka-druid-earthquake-data/create-druid-ingestion-job.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-druid-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor"]
           volumeMounts:
             - name: ingestion-job-spec

demos/nifi-kafka-druid-earthquake-data/create-nifi-ingestion-job.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/IngestEarthquakesToKafka.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/nifi-kafka-druid-earthquake-data/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-earthquake-data/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/nifi-kafka-druid-water-level-data/create-druid-ingestion-job.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-druid-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/stations-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-ingestion-job-spec.json https://druid-coordinator:8281/druid/indexer/v1/supervisor && curl -X POST --insecure -H 'Content-Type: application/json' -d @/tmp/ingestion-job-spec/measurements-compaction-job-spec.json https://druid-coordinator:8281/druid/coordinator/v1/config/compaction"]
           volumeMounts:
             - name: ingestion-job-spec

demos/nifi-kafka-druid-water-level-data/create-nifi-ingestion-job.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/IngestWaterLevelsToKafka.xml && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/nifi-kafka-druid-water-level-data/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/nifi-kafka-druid-water-level-data/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/signal-processing/Dockerfile-nifi

@@ -1,5 +1,5 @@
-# docker build -f ./Dockerfile-nifi -t docker.stackable.tech/demos/nifi:1.25.0-postgresql .
+# docker build -f ./Dockerfile-nifi -t docker.stackable.tech/demos/nifi:1.27.0-postgresql .
 
-FROM docker.stackable.tech/stackable/nifi:1.25.0-stackable24.3.0
+FROM docker.stackable.tech/stackable/nifi:1.27.0-stackable24.7.0
 
 RUN curl --fail -o /stackable/nifi/postgresql-42.6.0.jar "https://repo.stackable.tech/repository/misc/postgresql-timescaledb/postgresql-42.6.0.jar"

demos/signal-processing/create-nifi-ingestion-job.yaml

@@ -9,13 +9,13 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-timescale-job
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for timescaleDB tables to be ready'
                     && kubectl wait --for=condition=complete job/create-timescale-tables-job"
                   ]
       containers:
         - name: create-nifi-ingestion-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "export PGPASSWORD=$(cat /timescale-admin-credentials/password) && \
             curl -O https://raw.githubusercontent.com/stackabletech/demos/main/demos/signal-processing/DownloadAndWriteToDB.xml && \
             sed -i \"s/PLACEHOLDERPGPASSWORD/$PGPASSWORD/g\" DownloadAndWriteToDB.xml && \

demos/signal-processing/create-timescale-tables.yaml

@@ -9,7 +9,7 @@ spec:
       serviceAccountName: demo-serviceaccount
       initContainers:
         - name: wait-for-timescale
-          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for timescaleDB to be ready'
                     && kubectl wait --for=condition=ready --timeout=30m pod -l app.kubernetes.io/name=postgresql-timescaledb"
                   ]

demos/spark-k8s-anomaly-detection-taxi-data/create-spark-anomaly-detection-job.yaml

@@ -8,11 +8,11 @@ spec:
     spec:
       initContainers:
         - name: wait-for-testdata
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Waiting for job load-ny-taxi-data to finish' && kubectl wait --for=condition=complete --timeout=30m job/load-ny-taxi-data"]
       containers:
         - name: create-spark-anomaly-detection-job
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "echo 'Submitting Spark job' && kubectl apply -f /tmp/manifest/spark-ad-job.yaml"]
           volumeMounts:
             - name: manifest
@@ -37,7 +37,7 @@ data:
       name: spark-ad
     spec:
       sparkImage:
-        productVersion: 3.5.0
+        productVersion: 3.5.1
       mode: cluster
       mainApplicationFile: local:///spark-scripts/spark-ad.py
       deps:

demos/spark-k8s-anomaly-detection-taxi-data/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/spark-k8s-anomaly-detection-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/trino-taxi-data/create-table-in-trino.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: create-ny-taxi-data-table-in-trino
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

demos/trino-taxi-data/setup-superset.yaml

@@ -8,7 +8,7 @@ spec:
     spec:
       containers:
         - name: setup-superset
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.3.0
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["bash", "-c", "curl -o superset-assets.zip https://raw.githubusercontent.com/stackabletech/demos/main/demos/trino-taxi-data/superset-assets.zip && python -u /tmp/script/script.py"]
           volumeMounts:
             - name: script

docs/modules/demos/pages/data-lakehouse-iceberg-trino-spark.adoc

@@ -34,7 +34,7 @@ $ stackablectl demo install data-lakehouse-iceberg-trino-spark
 [#system-requirements]
 == System requirements
 
-The demo was developed and tested on a kubernetes cluster with 10 nodes (4 cores (8 threads), 20GiB RAM and 30GB HDD).
+The demo was developed and tested on a kubernetes cluster with about 12 nodes (4 cores with hyperthreading/SMT, 20GiB RAM and 30GB HDD).
 Instance types that loosely correspond to this on the Hyperscalers are:
 
 - *Google*: `e2-standard-8`
@@ -179,7 +179,7 @@ As you can see, the table `house-sales` is partitioned by year. Go ahead and cli
 
 image::data-lakehouse-iceberg-trino-spark/minio_5.png[]
 
-You can see that Trino has placed a single file into the selected folder containing all the house sales of that
+You can see that Trino has data into the selected folder containing all the house sales of that
 particular year.
 
 == NiFi
@@ -491,7 +491,7 @@ Here you can see all the available Trino catalogs.
 * `tpcds`: https://trino.io/docs/current/connector/tpcds.html[TPCDS connector] providing a set of schemas to support the
   http://www.tpc.org/tpcds/[TPC Benchmark™ DS]
 * `tpch`: https://trino.io/docs/current/connector/tpch.html[TPCH connector] providing a set of schemas to support the
-  http://www.tpc.org/tpcds/[TPC Benchmark™ DS]
+  http://www.tpc.org/tpch/[TPC Benchmark™ H]
 * `lakehouse`: The lakehouse area containing the enriched and performant accessible data
 
 == Superset
@@ -541,14 +541,14 @@ image::data-lakehouse-iceberg-trino-spark/superset_7.png[]
 On the left, select the database `Trino lakehouse`, the schema `house_sales`, and set `See table schema` to
 `house_sales`.
 
-image::data-lakehouse-iceberg-trino-spark/superset_8.png[]
-
-[NOTE]
+[IMPORTANT]
 ====
-This older screenshot shows how the table preview would look like. Currently, there is an https://github.com/apache/superset/issues/25307[open issue] 
+The older screenshot below shows how the table preview would look like. Currently, there is an https://github.com/apache/superset/issues/25307[open issue] 
 with previewing trino tables using the Iceberg connector. This doesn't affect the execution the following execution of the SQL statement.
 ====
 
+image::data-lakehouse-iceberg-trino-spark/superset_8.png[]
+
 In the right textbox, you can enter the desired SQL statement. If you want to avoid making one up, use the following:
 
 [source,sql]

docs/modules/demos/pages/logging.adoc

@@ -46,10 +46,10 @@ This demo will
    synchronization, and providing group services. This demo makes its log data observable in OpenSearch Dashboards.
 ** *Vector*: A tool for building observability pipelines. This demo uses Vector as a log agent to gather and transform
    the logs and as an aggregator to forward the collected logs to OpenSearch.
-** *OpenSearch*: A data store and search engine. This demo uses it to store and index the of the log data.
+** *OpenSearch*: A data store and search engine. This demo uses it to store and index the log data.
 ** *OpenSearch Dashboards*: A visualization and user interface. This demo uses it to make the log data easily accessible
    to the user.
-* Create a view in OpenSearch Dashboards for convenient browsing the log data.
+* Create a view in OpenSearch Dashboards to conveniently browse the log data.
 
 You can see the deployed products and their relationship in the following diagram:
 

docs/modules/demos/pages/spark-k8s-anomaly-detection-taxi-data.adoc

@@ -23,7 +23,7 @@ This demo should not be run alongside other demos.
 
 To run this demo, your system needs at least:
 
-* 8 {k8s-cpu}[cpu units] (core/hyperthread)
+* 10 {k8s-cpu}[cpu units] (core/hyperthread)
 * 32GiB memory
 * 35GiB disk storage
 

stacks/_templates/jupyterhub.yaml

@@ -3,7 +3,7 @@ name: jupyterhub
 repo:
   name: jupyterhub
   url: https://jupyterhub.github.io/helm-chart/
-version: 3.2.1
+version: 3.3.7
 options:
   hub:
     config:
@@ -49,7 +49,7 @@ options:
       HADOOP_CONF_DIR: "/home/jovyan/hdfs"
     initContainers:
       - name: download-notebook
-        image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.3.0
+        image: docker.stackable.tech/stackable/tools:1.0.0-stackable24.7.0
         command: ['sh', '-c', 'curl https://raw.githubusercontent.com/stackabletech/demos/main/stacks/jupyterhub-pyspark-hdfs/notebook.ipynb -o /notebook/notebook.ipynb']
         volumeMounts:
           - mountPath: /notebook

stacks/_templates/keycloak.yaml

@@ -48,7 +48,7 @@ spec:
             - name: tls
               mountPath: /tls/
         - name: create-auth-class
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev # We need 0.0.0-dev, so we get kubectl
+          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable24.7.0
           command: ["/bin/bash", "-c"]
           args:
             - |

stacks/_templates/minio-distributed-small.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.14
+version: 5.2.0
 options:
   additionalLabels:
     stackable.tech/vendor: Stackable

stacks/_templates/minio-distributed.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.14
+version: 5.2.0
 options:
   additionalLabels:
     stackable.tech/vendor: Stackable

stacks/_templates/minio.yaml

@@ -3,7 +3,7 @@ name: minio
 repo:
   name: minio
   url: https://charts.min.io/
-version: 5.0.14
+version: 5.2.0
 options:
   additionalLabels:
     stackable.tech/vendor: Stackable