Merge pull request #189 from piotrpdev/e2e-aiedge

[RHOAIENG-289] Emit metadata from e2e and gitops pipeline

pipelines/README.md

@@ -212,3 +212,23 @@ oc create -f tekton/gitops-update-pipeline/example-pipelineruns/gitops-update-pi
 After the Pipeline Run(s) finish, check your git repository -- there should be a pull request with an update of the respective
 `acm/odh-edge/apps/*/kustomization.yaml` file with the SHA-256 of the new container image that got built, tested, and pushed to Quay
 in previous steps.
+
+#### View the PipelineRun results
+
+If the PipelineRun completes successfully, you can see the results in the OpenShift Console by going to Pipelines > PipelineRuns > (Select your PipelineRun and scroll down).
+
+You can also click on the "YAML" tab in the PipelineRun and scroll down to the `pipelineResults` section, it will look something like this:
+
+<details><summary>Typical GitOps pipeline results</summary>
+
+```yaml
+  pipelineResults:
+    - name: target-registry-url
+      value: quay.io/pplaczek/tensorflow-housing
+    - name: image-sha
+      value: 'sha256:0cc9a636c2f18b0f15224a234995ecd27a3dc2e5eb7ffefc8ecfd72c099da31f'
+    - name: pr-url
+      value: 'https://github.com/piotrpdev/ai-edge/pull/5'
+```
+
+</details>

pipelines/tekton/aiedge-e2e/README.md

@@ -90,3 +90,64 @@ Update the `s3-bucket-name` parameter value from its default `rhoai-edge-models`
 #### For Git fetch
 Update the `git-model-repo` parameter with the repository url, the `modelRelativePath` parameter to the model files path and the `git-revision` parameter for the version/branch of the repository in [`aiedge-e2e.pipelinerun.yaml`](pipelines/tekton/aiedge-e2e/aiedge-e2e.pipelinerun.yaml).
 
+#### View the PipelineRun results
+
+If the PipelineRun completes successfully, you can see the results in the OpenShift Console by going to Pipelines > PipelineRuns > (Select your PipelineRun and scroll down).
+
+You can also click on the "YAML" tab in the PipelineRun and scroll down to the `pipelineResults` section, it will look something like this:
+
+<details><summary>Typical E2E pipeline results</summary>
+
+```yaml
+  pipelineResults:
+    - name: git-model-fetched-commit
+      value: f89e2bc61e6ff7f539c52fded3a2bdc991a55b7c
+    - name: git-model-fetched-url
+      value: 'https://github.com/piotrpdev/ai-edge.git'
+    - name: git-model-fetched-commit-epoch
+      value: '1702404808'
+    - name: git-containerfile-fetched-commit
+      value: f89e2bc61e6ff7f539c52fded3a2bdc991a55b7c
+    - name: git-containerfile-fetched-url
+      value: 'https://github.com/piotrpdev/ai-edge'
+    - name: git-containerfile-fetched-commit-epoch
+      value: '1702404808'
+    - name: model-files-size
+      value: |
+        2084
+    - name: model-files-list
+      value: |
+        MLmodel
+        README.md
+        conda.yaml
+        convert_csv_to_json.py
+        dataset.csv
+        python_env.yaml
+        requirements.txt
+        tf2model/
+    - name: internal-registry-url
+      value: >
+        image-registry.openshift-image-registry.svc:5000/pplaczek-pipeline-dev/tensorflow-housing:1
+    - name: target-registry-url
+      value: quay.io/pplaczek/tensorflow-housing
+    - name: internal-image-url
+      value: >-
+        image-registry.openshift-image-registry.svc:5000/pplaczek-pipeline-dev/tensorflow-housing@sha256:0cc9a636c2f18b0f15224a234995ecd27a3dc2e5eb7ffefc8ecfd72c099da31f
+    - name: target-image-url
+      value: >-
+        quay.io/pplaczek/tensorflow-housing:1-fa9e93e0-c66c-4075-9333-61769420f102
+    - name: internal-image-size
+      value: '126507187'
+    - name: buildah-sha
+      value: 'sha256:0cc9a636c2f18b0f15224a234995ecd27a3dc2e5eb7ffefc8ecfd72c099da31f'
+    - name: model-name
+      value: tensorflow-housing
+    - name: model-version
+      value: '1'
+    - name: internal-image-created-at
+      value: '2023-12-12T18:39:25Z'
+    - name: internal-image-buildah-version
+      value: 1.24.2
+```
+
+</details>

pipelines/tekton/aiedge-e2e/aiedge-e2e.pipeline.yaml

@@ -55,10 +55,63 @@ spec:
     description: "Action to perform on the k8s deployment created to test the model container image. Valid values in [delete, keep, stop]"
     default: delete
   results:
+  - name: s3-model-fetched-url
+    description: The S3 URL used to download the model
+    value: $(tasks.fetch-model-s3.results.s3-url)
+  - name: git-model-fetched-commit
+    description: The commit hash of the git repo where the model files were fetched from
+    value: $(tasks.fetch-model-git.results.commit)
+  - name: git-model-fetched-url
+    description: The url of the git repo where the model files were fetched from
+    value: $(tasks.fetch-model-git.results.url)
+  - name: git-model-fetched-commit-epoch
+    description: The commit timestamp of the git repo where the model files were fetched from
+    value: $(tasks.fetch-model-git.results.committer-date)
+  - name: git-containerfile-fetched-commit
+    description: The commit hash of the git repo where the containerfile was fetched from
+    value: $(tasks.git-clone-containerfile-repo.results.commit)
+  - name: git-containerfile-fetched-url
+    description: The url of the git repo where the containerfile was fetched from
+    value: $(tasks.git-clone-containerfile-repo.results.url)
+  - name: git-containerfile-fetched-commit-epoch
+    description: The commit timestamp of the git repo where the containerfile was fetched from
+    value: $(tasks.git-clone-containerfile-repo.results.committer-date)
+  - name: model-files-size
+    description: The size of the model files
+    value: $(tasks.check-model-and-containerfile-exists.results.model-files-size)
+  - name: model-files-list
+    description: The list of model files
+    value: $(tasks.check-model-and-containerfile-exists.results.model-files-list)
+  - name: internal-registry-url
+    description: The tag where the model container image was pushed to in the internal registry (e.g. image-reg...svc:5000/rhoai-models/ai-model:1-232)
+    value: $(tasks.build-container.results.IMAGE_URL)
   - name: target-registry-url
-    value: $(tasks.retrieve-image-digest.results.target-registry-url)
-  - name: image-sha
-    value: $(tasks.retrieve-image-digest.results.image-sha)
+    description: The url of the target registry (e.g. quay.io/rhoai-models/ai-model/)
+    value: $(tasks.retrieve-build-image-info.results.target-registry-url)
+  - name: internal-image-url
+    description: The url of the image in the internal registry
+    value: $(tasks.retrieve-build-image-info.results.internal-image-url)
+  - name: target-image-url
+    description: The url of the image in the target registry (e.g. quay.io/rhoai-models/ai-model:1-232)
+    value: $(tasks.retrieve-build-image-info.results.target-image-url)
+  - name: internal-image-size
+    description: The size of the model container image in the internal registry in bytes
+    value: $(tasks.retrieve-build-image-info.results.internal-image-size)
+  - name: buildah-sha
+    description: The SHA digest of the model container image
+    value: $(tasks.build-container.results.IMAGE_DIGEST)
+  - name: model-name
+    description: The model name
+    value: $(tasks.retrieve-build-image-info.results.model-name)
+  - name: model-version
+    description: The model version
+    value: $(tasks.retrieve-build-image-info.results.model-version)
+  - name: internal-image-created-at
+    description: Timestamp of when the model container image was created in UTC format
+    value: $(tasks.retrieve-build-image-info.results.internal-image-created-at)
+  - name: internal-image-buildah-version
+    description: The buildah version used to build the model container image
+    value: $(tasks.retrieve-build-image-info.results.internal-image-buildah-version)
   tasks:
   - name: fetch-model-git
     taskRef:
@@ -280,17 +333,21 @@ spec:
     taskRef:
       kind: ClusterTask
       name: openshift-client
-  - name: retrieve-image-digest
+  - name: retrieve-build-image-info
     taskRef:
       kind: Task
-      name: retrieve-image-digest
+      name: retrieve-build-image-info
     params:
     - name: model-name
       value: $(params.model-name)
     - name: model-version
       value: $(params.model-version)
     - name: namespace
       value: $(params.target-namespace)
+    - name: buildah-sha
+      value: $(tasks.build-container.results.IMAGE_DIGEST)
+    - name: pipeline-run-uid
+      value: $(context.pipelineRun.uid)
     - name: target-registry-url
       value: quay.io/$(params.target-imagerepo)/$(params.model-name)
     runAfter:
@@ -299,15 +356,15 @@ spec:
   - name: skopeo-copy
     params:
     - name: srcImageURL
-      value: docker://image-registry.openshift-image-registry.svc:5000/$(context.pipelineRun.namespace)/$(params.model-name)@$(tasks.retrieve-image-digest.results.image-sha)
+      value: docker://$(tasks.retrieve-build-image-info.results.internal-image-url)
     - name: destImageURL
-      value: docker://$(tasks.retrieve-image-digest.results.target-registry-url):$(params.model-version)-$(context.pipelineRun.uid)
+      value: docker://$(tasks.retrieve-build-image-info.results.target-image-url)
     - name: srcTLSverify
       value: "true"
     - name: destTLSverify
       value: "true"
     runAfter:
-    - retrieve-image-digest
+    - retrieve-build-image-info
     taskRef:
       kind: ClusterTask
       name: skopeo-copy

pipelines/tekton/aiedge-e2e/tasks/check-model-and-containerfile-exists.yaml

@@ -14,6 +14,11 @@ spec:
     type: string
   - name: modelRelativePath
     type: string
+  results:
+  - name: model-files-size
+    description: Total size of the model files
+  - name: model-files-list
+    description: Space separated list of model files
   steps:
   - name: check-model-and-containerfile
     image: registry.access.redhat.com/ubi9/ubi-micro
@@ -22,9 +27,15 @@ spec:
 
       # Check model files exist
       if [ -n "$(params.modelRelativePath)" ]; then
-        ls -l model_dir/$(params.modelRelativePath)/$(params.model-name)/
+        ls -l model_dir/$(params.modelRelativePath)/$(params.model-name)/ ;
+
+        du -s model_dir/$(params.modelRelativePath)/$(params.model-name)/ | cut -f1 | tee $(results.model-files-size.path) ;
+        ls -p model_dir/$(params.modelRelativePath)/$(params.model-name)/ | tee $(results.model-files-list.path) ;
       else
         ls -l model_dir/$(params.model-name)/
+
+        du -s model_dir/$(params.model-name)/ | cut -f1 | tee $(results.model-files-size.path) ;
+        ls -p model_dir/$(params.model-name)/ | tee $(results.model-files-list.path) ;
       fi
 
       # Check containerfile exists

pipelines/tekton/aiedge-e2e/tasks/kserve-download-model.yaml

@@ -12,13 +12,20 @@ spec:
     type: string
   - name: s3-bucket-name
     type: string
+  results:
+  - name: s3-url
+    description: The S3 URL used to download the model
   steps:
   - name: download-model-s3
     image: quay.io/opendatahub/kserve-storage-initializer:v0.11
     script: |
       mkdir -p $(workspaces.workspace.path)/model_dir/
+
+      export S3_URL="s3://$(params.s3-bucket-name)/$(params.model-name)"
+      echo -n $S3_URL | tee $(results.s3-url.path) ;
+
       STORAGE_CONFIG="$(cat $(workspaces.s3-secret.path)/s3-storage-config)" /storage-initializer/scripts/initializer-entrypoint \
-      's3://$(params.s3-bucket-name)/$(params.model-name)' \
+      $S3_URL \
       $(workspaces.workspace.path)/model_dir/$(params.model-name)
   workspaces:
   - description: The workspace for the downloaded model.

pipelines/tekton/aiedge-e2e/tasks/kustomization.yaml

@@ -4,4 +4,4 @@ resources:
 - check-model-and-containerfile-exists.yaml
 - kserve-download-model.yaml
 - test-model-rest-svc-task.yaml
-- retrieve-image-digest.task.yaml
+- retrieve-build-image-info.task.yaml

pipelines/tekton/aiedge-e2e/tasks/retrieve-build-image-info.task.yaml

@@ -0,0 +1,60 @@
+apiVersion: tekton.dev/v1beta1
+kind: Task
+metadata:
+  name: retrieve-build-image-info
+spec:
+  description: Return information about the built image
+  params:
+  - name: namespace
+    type: string
+  - name: model-name
+    type: string
+  - name: model-version
+    type: string
+  - name: buildah-sha
+    type: string
+  - name: pipeline-run-uid
+    type: string
+  - name: target-registry-url
+    type: string
+  steps:
+  - name: get-image-sha
+    image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest
+    script: |
+      echo -n "$(params.model-name)" | tee $(results.model-name.path) ;
+      echo ;
+      echo -n "$(params.model-version)" | tee $(results.model-version.path) ;
+      echo ;
+      echo -n "$(params.target-registry-url)" | tee $(results.target-registry-url.path) ;
+      echo ;
+      export DOCKER_IMAGE_REF=$(oc get -n $(params.namespace) -o jsonpath='{.image.dockerImageReference}' imagestreamtag/$(params.model-name):$(params.model-version)) ;
+      if [[ $DOCKER_IMAGE_REF != *"$(params.buildah-sha)"* ]]; then
+        echo "ImageStreamTag doesn't contain the correct image SHA"
+        exit 1 ;
+      fi
+      echo -n $DOCKER_IMAGE_REF | tee $(results.internal-image-url.path) ;
+      echo ;
+      oc get -n $(params.namespace) -o jsonpath='{.image.dockerImageMetadata.Size}' imagestreamtag/$(params.model-name):$(params.model-version) | tee $(results.internal-image-size.path) ;
+      echo ;
+      oc get -n $(params.namespace) -o jsonpath='{.image.dockerImageMetadata.Created}' imagestreamtag/$(params.model-name):$(params.model-version) | tee $(results.internal-image-created-at.path) ;
+      echo ;
+      oc get -n $(params.namespace) -o jsonpath='{.image.dockerImageMetadata.Config.Labels.io\.buildah\.version}' imagestreamtag/$(params.model-name):$(params.model-version) | tee $(results.internal-image-buildah-version.path) ;
+      echo ;
+      echo -n "$(params.target-registry-url):$(params.model-version)-$(params.pipeline-run-uid)" | tee $(results.target-image-url.path) ;
+  results:
+  - name: model-name
+    description: The name of the model
+  - name: model-version
+    description: The version of the model
+  - name: internal-image-size
+    description: The size of the image
+  - name: internal-image-created-at
+    description: The date and time the image was created in UTC format
+  - name: internal-image-buildah-version
+    description: The version of buildah used to build the image
+  - name: internal-image-url
+    description: The url of the image in the internal registry
+  - name: target-registry-url
+    description: The url of the target registry (e.g. quay.io/rhoai-models/ai-model/)
+  - name: target-image-url
+    description: The url of the image in the target registry (e.g. quay.io/rhoai-models/ai-model:1-232)

pipelines/tekton/gitops-update-pipeline/gitops-update-pipeline.yaml

@@ -27,6 +27,19 @@ spec:
     - name: gitTokenSecretKey
       default: token
       type: string
+    - name: useMergedBuildTestPipeline
+      type: string
+      default: "false"
+  results:
+    - name: target-registry-url
+      description: The url of the target registry (e.g. quay.io/rhoai-models/ai-model/)
+      value: $(tasks.retrieve-image-info.results.target-registry-url)
+    - name: image-sha
+      description: The image digest used to tag the image
+      value: $(tasks.retrieve-image-info.results.image-sha)
+    - name: pr-url
+      description: The URL of the pull request
+      value: $(tasks.open-pr.results.URL)
   tasks:
     - name: retrieve-image-info
       taskRef:
@@ -37,6 +50,8 @@ spec:
         value: $(context.pipelineRun.namespace)
       - name: model-name
         value: $(params.model-name)
+      - name: useMergedBuildTestPipeline
+        value: $(params.useMergedBuildTestPipeline)
     - name: git-clone
       params:
         - name: url

pipelines/tekton/gitops-update-pipeline/test-mlflow-retrieve-image-info-task.yaml

@@ -9,11 +9,19 @@ spec:
     type: string
   - name: model-name
     type: string
+  - name: useMergedBuildTestPipeline
+    type: string
   steps:
   - name: get-image-sha
     image: image-registry.openshift-image-registry.svc:5000/openshift/cli:latest
     script: |
-      oc get -n $(params.namespace) pipelinerun --selector tekton.dev/pipeline=test-mlflow-image --sort-by=.status.completionTime -o jsonpath='{range .items[?(@.spec.params[0].name == "model-name")]}{.spec.params[?(@.name == "model-name")].value} {.status.results[?(@.name == "image-sha")].value} {.status.results[?(@.name == "target-registry-url")].value}{"\n"}{end}' | awk -v model=$(params.model-name) '$1 == model && NF == 3 { print $2, $3 }' | tail -1 | tee /dev/stderr | while read sha registry ; do echo -n "$sha" > $(results.image-sha.path) ; echo -n "$registry" > $(results.target-registry-url.path) ; done ;
+      if [[ "$(params.useMergedBuildTestPipeline)" == "true" ]]; then
+        echo "Using aiedge-e2e pipeline"
+        oc get -n $(params.namespace) pipelinerun --selector tekton.dev/pipeline=aiedge-e2e --sort-by=.status.completionTime -o jsonpath='{range .items[?(@.spec.params[0].name == "model-name")]}{.spec.params[?(@.name == "model-name")].value} {.status.results[?(@.name == "buildah-sha")].value} {.status.results[?(@.name == "target-registry-url")].value}{"\n"}{end}' | awk -v model=$(params.model-name) '$1 == model && NF == 3 { print $2, $3 }' | tail -1 | tee /dev/stderr | while read sha registry ; do echo -n "$sha" > $(results.image-sha.path) ; echo -n "$registry" > $(results.target-registry-url.path) ; done ;
+      else
+        echo "Using test-mlflow-image pipeline"
+        oc get -n $(params.namespace) pipelinerun --selector tekton.dev/pipeline=test-mlflow-image --sort-by=.status.completionTime -o jsonpath='{range .items[?(@.spec.params[0].name == "model-name")]}{.spec.params[?(@.name == "model-name")].value} {.status.results[?(@.name == "image-sha")].value} {.status.results[?(@.name == "target-registry-url")].value}{"\n"}{end}' | awk -v model=$(params.model-name) '$1 == model && NF == 3 { print $2, $3 }' | tail -1 | tee /dev/stderr | while read sha registry ; do echo -n "$sha" > $(results.image-sha.path) ; echo -n "$registry" > $(results.target-registry-url.path) ; done ;
+      fi
       test -s $(results.image-sha.path) && test -s $(results.target-registry-url.path)
   results:
   - name: target-registry-url