chore: add oci references, merge templating changes (#514)

xeniape · adwk67 · web-flow · commit 8b51fba84be5 · 2025-01-30T12:19:17.000Z
* replace repo urls

* bumped op-rs to 0.85, changelog

* corrected path to nyc image

---------

Co-authored-by: Andrew Kenworthy &lt;andrew.kenworthy@stackable.de&gt;
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -11,8 +11,13 @@ All notable changes to this project will be documented in this file.
 - Run a `containerdebug` process in the background of each Spark container to collect debugging information ([#508]).
 - Aggregate emitted Kubernetes events on the CustomResources ([#515]).
 
+### Changed
+
+- Default to OCI for image metadata and product image selection ([#514]).
+
 [#501]: https://github.com/stackabletech/spark-k8s-operator/pull/501
 [#508]: https://github.com/stackabletech/spark-k8s-operator/pull/508
+[#514]: https://github.com/stackabletech/spark-k8s-operator/pull/514
 [#515]: https://github.com/stackabletech/spark-k8s-operator/pull/515
 
 ## [24.11.1] - 2025-01-10
diff --git a/Cargo.lock b/Cargo.lock
diff --git a/Cargo.nix b/Cargo.nix
diff --git a/Cargo.toml b/Cargo.toml
@@ -22,7 +22,7 @@ serde = { version = "1.0", features = ["derive"] }
 serde_json = "1.0"
 serde_yaml = "0.9"
 snafu = "0.8"
-stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.84.1" }
+stackable-operator = { git = "https://github.com/stackabletech/operator-rs.git", tag = "stackable-operator-0.85.0" }
 strum = { version = "0.26", features = ["derive"] }
 tokio = { version = "1.39", features = ["full"] }
 tracing = "0.1"
diff --git a/crate-hashes.json b/crate-hashes.json
diff --git a/deploy/helm/spark-k8s-operator/crds/crds.yaml b/deploy/helm/spark-k8s-operator/crds/crds.yaml
@@ -917,7 +917,7 @@ spec:
                     Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details.
                   properties:
                     custom:
-                      description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0`
+                      description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0`
                       type: string
                     productVersion:
                       description: Version of the product, e.g. `1.4.1`.
@@ -944,7 +944,7 @@ spec:
                       nullable: true
                       type: array
                     repo:
-                      description: Name of the docker repo, e.g. `docker.stackable.tech/stackable`
+                      description: Name of the docker repo, e.g. `oci.stackable.tech/sdp`
                       nullable: true
                       type: string
                     stackableVersion:
@@ -1047,7 +1047,7 @@ spec:
                     Consult the [Product image selection documentation](https://docs.stackable.tech/home/nightly/concepts/product_image_selection) for details.
                   properties:
                     custom:
-                      description: Overwrite the docker image. Specify the full docker image name, e.g. `docker.stackable.tech/stackable/superset:1.4.1-stackable2.1.0`
+                      description: Overwrite the docker image. Specify the full docker image name, e.g. `oci.stackable.tech/sdp/superset:1.4.1-stackable2.1.0`
                       type: string
                     productVersion:
                       description: Version of the product, e.g. `1.4.1`.
@@ -1074,7 +1074,7 @@ spec:
                       nullable: true
                       type: array
                     repo:
-                      description: Name of the docker repo, e.g. `docker.stackable.tech/stackable`
+                      description: Name of the docker repo, e.g. `oci.stackable.tech/sdp`
                       nullable: true
                       type: string
                     stackableVersion:
diff --git a/deploy/helm/spark-k8s-operator/values.yaml b/deploy/helm/spark-k8s-operator/values.yaml
@@ -1,7 +1,7 @@
 # Default values for spark-k8s-operator.
 ---
 image:
-  repository: docker.stackable.tech/stackable/spark-k8s-operator
+  repository: oci.stackable.tech/sdp/spark-k8s-operator
   pullPolicy: IfNotPresent
   pullSecrets: []
 
diff --git a/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml b/docs/modules/spark-k8s/examples/example-sparkapp-image.yaml
@@ -5,7 +5,7 @@ metadata:
   name: example-sparkapp-image
   namespace: default
 spec:
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
+  image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0 # <1>
   sparkImage:
     productVersion: 3.5.2
   mode: cluster
diff --git a/docs/modules/spark-k8s/pages/reference/environment-variables.adoc b/docs/modules/spark-k8s/pages/reference/environment-variables.adoc
@@ -30,7 +30,7 @@ docker run \
 --env KUBECONFIG=/home/stackable/.kube/config \
 --env KUBERNETES_CLUSTER_DOMAIN=mycluster.local \
 --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \
-docker.stackable.tech/stackable/spark-k8s-operator:latest
+oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev
 ----
 
 == WATCH_NAMESPACE
@@ -59,5 +59,5 @@ docker run \
 --env KUBECONFIG=/home/stackable/.kube/config \
 --env WATCH_NAMESPACE=test \
 --mount type=bind,source="$HOME/.kube/config",target="/home/stackable/.kube/config" \
-docker.stackable.tech/stackable/spark-k8s-operator:latest
+oci.stackable.tech/sdp/spark-k8s-operator:0.0.0-dev
 ----
diff --git a/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc b/docs/modules/spark-k8s/pages/usage-guide/job-dependencies.adoc
@@ -58,7 +58,7 @@ Below is an example of a custom image that includes a JDBC driver:
 
 [source, Dockerfile]
 ----
-FROM docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable24.11.0 # <1>
+FROM oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable24.11.0 # <1>
 
 RUN curl --fail -o /stackable/spark/jars/postgresql-42.6.0.jar "https://jdbc.postgresql.org/download/postgresql-42.6.0.jar"
 ----
@@ -75,7 +75,7 @@ metadata:
   name: spark-jdbc
 spec:
   sparkImage:
-    custom: "docker.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1>
+    custom: "oci.stackable.tech/sandbox/spark-k8s:3.5.2-stackable0.0.0-dev" # <1>
     productVersion: "3.5.2" # <2>
     pullPolicy: IfNotPresent # <3>
 ...
diff --git a/examples/README-examples.md b/examples/README-examples.md
@@ -16,7 +16,7 @@ stackablectl operator install spark-k8s commons secret
 Load the `ny-tlc-report` image to the cluster:
 
 ````text
-kind load docker-image docker.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform
+kind load docker-image oci.stackable.tech/stackable/ny-tlc-report:0.2.0 --name stackable-data-platform
 ````
 
 ## Set up the `PersistentVolumeClaim`
diff --git a/examples/ny-tlc-report-image.yaml b/examples/ny-tlc-report-image.yaml
@@ -6,8 +6,8 @@ metadata:
   namespace: default
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
-  image: docker.stackable.tech/stackable/ny-tlc-report:0.2.0
-  sparkImage: docker.stackable.tech/stackable/spark-k8s:3.5.2-stackable0.0.0-dev
+  image: oci.stackable.tech/stackable/ny-tlc-report:0.2.0
+  sparkImage: oci.stackable.tech/sdp/spark-k8s:3.5.2-stackable0.0.0-dev
   sparkImagePullPolicy: IfNotPresent
   mode: cluster
   mainApplicationFile: local:///stackable/spark/jobs/ny_tlc_report.py
diff --git a/kind/assert-pvc-jars.yaml b/kind/assert-pvc-jars.yaml
@@ -13,7 +13,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: assert-pvc-jars
-          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"
diff --git a/kind/kind-pvc.yaml b/kind/kind-pvc.yaml
@@ -24,7 +24,7 @@ spec:
             claimName: pvc-ksv
       containers:
         - name: aws-deps
-          image: docker.stackable.tech/stackable/tools:0.2.0-stackable0.4.0
+          image: oci.stackable.tech/sdp/tools:0.2.0-stackable0.4.0
           env:
             - name: DEST_DIR
               value: "/dependencies/jars"
diff --git a/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml b/tests/templates/kuttl/custom-log-directory/20-test-logs.yaml
@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",
diff --git a/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/06-deploy-automatic-log-config-spark-app.yaml.j2
@@ -12,7 +12,7 @@ spec:
     productVersion: "{{ test_scenario['values']['spark'] }}"
 {% endif %}
     pullPolicy: IfNotPresent
-  image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
+  image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS
diff --git a/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2 b/tests/templates/kuttl/logging/07-deploy-custom-log-config-spark-app.yaml.j2
@@ -29,7 +29,7 @@ spec:
     productVersion: "{{ test_scenario['values']['spark'] }}"
 {% endif %}
     pullPolicy: IfNotPresent
-  image: docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
+  image: oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}
   vectorAggregatorConfigMapName: spark-vector-aggregator-discovery
   mode: cluster
   mainClass: org.apache.spark.examples.SparkALS
diff --git a/tests/templates/kuttl/logging/10-install-test-runner.yaml b/tests/templates/kuttl/logging/10-install-test-runner.yaml
@@ -17,6 +17,6 @@ spec:
     spec:
       containers:
         - name: test-runner
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           stdin: true
           tty: true
diff --git a/tests/templates/kuttl/overrides/20-test-logs.yaml b/tests/templates/kuttl/overrides/20-test-logs.yaml
@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",
diff --git a/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2 b/tests/templates/kuttl/pyspark-ny-public-s3-image/10-deploy-spark-app.yaml.j2
@@ -5,7 +5,7 @@ metadata:
   name: pyspark-ny-public-s3-image
 spec:
   # everything under /jobs will be copied to /stackable/spark/jobs
-  image: "docker.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}"
+  image: "oci.stackable.tech/stackable/ny-tlc-report:{{ test_scenario['values']['ny-tlc-report'] }}"
 {% if lookup('env', 'VECTOR_AGGREGATOR') %}
   vectorAggregatorConfigMapName: vector-aggregator-discovery
 {% endif %}
diff --git a/tests/templates/kuttl/smoke/60-test-logs.yaml b/tests/templates/kuttl/smoke/60-test-logs.yaml
@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",
diff --git a/tests/templates/kuttl/spark-history-server/20-test-logs.yaml b/tests/templates/kuttl/spark-history-server/20-test-logs.yaml
@@ -10,7 +10,7 @@ spec:
       activeDeadlineSeconds: 100
       containers:
         - name: history-api-check
-          image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev
+          image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev
           command:
             [
               "bash",
diff --git a/tests/test-definition.yaml b/tests/test-definition.yaml
@@ -9,11 +9,11 @@ dimensions:
       - 3.5.2
       # Alternatively, if you want to use a custom image, append a comma and the full image name to the product version
       # as in the example below.
-      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
+      # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: spark-delta-lake
     values:
       - 3.5.2
-      # - 3.5.1,docker.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
+      # - 3.5.1,oci.stackable.tech/sandbox/spark-k8s:3.5.1-stackable0.0.0-dev
   - name: hbase
     values:
       - 2.6.0

Original file line number	Diff line number	Diff line change
`@@ -10,7 +10,7 @@ spec:`
`10`	`10`	`activeDeadlineSeconds: 100`
`11`	`11`	`containers:`
`12`	`12`	`- name: history-api-check`
`13`		`- image: docker.stackable.tech/stackable/testing-tools:0.2.0-stackable0.0.0-dev`
	`13`	`+ image: oci.stackable.tech/sdp/testing-tools:0.2.0-stackable0.0.0-dev`
`14`	`14`	`command:`
`15`	`15`	`[`
`16`	`16`	`"bash",`