review: Update script for including testing images

scripts/airgapped/README.md

@@ -33,8 +33,9 @@ This script makes the following assumptions:
 
 ```bash
 python3 scripts/get-all-images.py \
+    --append-images=tests/airgapped/ckf-1.8-testing-images.txt \
     releases/1.8/stable/kubeflow/bundle.yaml \
-    --airgap-testing > images.txt
+    > images.txt
 ```
 
 ## Pull images to docker cache

scripts/get-all-images.py

@@ -21,13 +21,6 @@
 GH_DEPENDENCY_BRANCH_KEY = "_github_dependency_repo_branch"
 GET_IMAGES_SH = "tools/get-images.sh"
 
-AIRGAP_TESTING_IMAGES = [
-    "charmedkubeflow/pipelines-runner:ckf-1.8",
-    "docker.io/kubeflowkatib/simple-pbt:v0.16.0",
-    "ghcr.io/knative/helloworld-go:latest",
-    "gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0",
-]
-
 
 def is_dependency_app(app: dict) -> bool:
     """
@@ -205,21 +198,42 @@ def get_bundle_images(bundle_path: str) -> list[str]:
     return cleanup_images(images)
 
 
+def get_static_images_from_file(images_file_path: str) -> list[str]:
+    """
+    Return a list of images stored in a text file and separated by \n.
+
+    Args:
+        images_file_path: Path of the file containing images
+
+    Returns:
+        list of strings, containing the images in the file
+    """
+    with open(images_file_path, "r") as file:
+        images = file.readlines()
+
+    cleaned_images = [image.strip() for image in images]
+    [logging.info(image) for image in cleaned_images]
+
+    return cleaned_images
+
+
 def main():
     parser = argparse.ArgumentParser(
         description="Gather all images from a bundle"
     )
     parser.add_argument("bundle")
-    parser.add_argument("--airgap-testing", action="store_true",
-                        help="Include images needed for airgap testing.")
+    parser.add_argument("--append-images",
+                        help="Appends list of images from input file.")
 
     args = parser.parse_args()
 
     images = get_bundle_images(args.bundle)
 
     # append the airgap images
-    if args.airgap_testing:
-        images.extend(AIRGAP_TESTING_IMAGES)
+    if args.append_images:
+        logging.info("Appending images found in file '%s'", args.append_images)
+        extra_images = get_static_images_from_file(args.append_images)
+        images.extend(extra_images)
 
     logging.info(f"Found {len(images)} different images")
 

tests/airgapped/README.md

@@ -29,6 +29,7 @@ You can run the script that will spin up an airgapped microk8s cluster with:
   --node-name airgapped-microk8s \
   --microk8s-channel 1.25-strict/stable \
   --bundle-path releases/1.8/stable/kubeflow/bundle.yaml \
+  --testing-images-path tests/airgapped/ckf-1.8-testing-images.txt \
   --juju-channel 3.1/stable
 ```
 
@@ -60,8 +61,9 @@ they'd like to be loaded during tests.
 
 ```bash
 python3 scripts/get-all-images.py \
+  --testing-images-path tests/airgapped/ckf-1.8-testing-images.txt \
     releases/1.8/stable/kubeflow/bundle.yaml \
-    --airgap-testing > images-all.txt
+    images-all.txt
 ```
 
 This will generate an `images-all.txt`, with all images of CKF 1.8. You can
@@ -130,3 +132,9 @@ To test Charmed Kubeflow components in airgapped, follow the instructions in the
 * [KNative](./knative/README.md)
 * [Pipelines](./pipelines/README.md)
 * [Training Operator](./training/README.md)
+
+You will also need to ensure the airgap setup script is including the file that contains the airgap
+testing images. For a new release, we'll need to create a new dedicated file with the images needed
+for airgap testing the new release.
+
+Such images files are usually located under `tests/airgapped/ckf-X.Y-testing-images.txt`.

tests/airgapped/airgap.sh

@@ -125,6 +125,7 @@ DISTRO="${DISTRO:-"ubuntu:22.04"}"
 MICROK8S_CHANNEL="${MICROK8S_CHANNEL:-}"
 JUJU_CHANNEL="${JUJU_CHANNEL:-"2.9/stable"}"
 BUNDLE_PATH="${BUNDLE_PATH:-"releases/latest/edge/bundle.yaml"}"
+TESTING_IMAGES_PATH="${TESTING_IMAGES_PATH:-"tests/airgapped/ckf-1.8-testing-images.txt"}"
 LIBRARY_MODE=false
 
 
@@ -136,6 +137,7 @@ while true; do
     --microk8s-channel ) MICROK8S_CHANNEL="$2"; shift 2 ;;
     --juju-channel) JUJU_CHANNEL="$2"; shift 2 ;;
     --bundle-path) BUNDLE_PATH="$2"; shift 2 ;;
+    --testing-images-path) TESTING_IMAGES_PATH="$2"; shift 2 ;;
     -h | --help )
       prog=$(basename -s.wrapper "$0")
       echo "Usage: $prog [options...]"
@@ -158,7 +160,7 @@ done
 if [ "$LIBRARY_MODE" == "false" ];
 then
   echo "1/X -- (us) Create images tar.gz"
-  create_images_tar "$BUNDLE_PATH"
+  create_images_tar "$BUNDLE_PATH" "$TESTING_IMAGES_PATH"
   echo "2/X -- (us) Create charms tar.gz"
   create_charms_tar "$BUNDLE_PATH"
   echo "3/X -- (client) Setup K8s cluster (MicroK8s)"

tests/airgapped/ckf-1.8-testing-images.txt

@@ -0,0 +1,4 @@
+  charmedkubeflow/pipelines-runner:ckf-1.8
+  docker.io/kubeflowkatib/simple-pbt:v0.16.0
+  ghcr.io/knative/helloworld-go:latest
+  gcr.io/kubeflow-ci/tf-mnist-with-summaries:1.0

tests/airgapped/ckf.sh

@@ -7,6 +7,7 @@
 
 function create_images_tar() {
   local BUNDLE_PATH=$1
+  local TESTING_IMAGES_PATH=$2
 
   if [ -f "images.tar.gz" ]; then
       echo "images.tar.gz exists. Will not recreate it."
@@ -17,8 +18,9 @@ function create_images_tar() {
 
   echo "Generating list of images of Charmed Kubeflow"
   python3 scripts/get-all-images.py \
+    --append-images "$TESTING_IMAGES_PATH" \
     "$BUNDLE_PATH" \
-    --airgap-testing > images.txt
+    > images.txt
 
   echo "Using produced list to load it into our machine's docker cache"
   python3 scripts/airgapped/save-images-to-cache.py images.txt