Merge pull request opendatahub-io#60 from opendatahub-io/2023b

Sync release 2023b branch with upstream

.github/workflows/auto-add-issue-to-project.yml

@@ -1,14 +1,25 @@
-name: Add Issues to ODH Feature Tracking Project
-on:
+---
+name: Auto Add Issues to Tracking boards
+on:  # yamllint disable-line rule:truthy
   issues:
     types:
       - opened
 jobs:
   add-to-project:
-    name: Add issue to project
+    name: Add issue to projects
     runs-on: ubuntu-latest
     steps:
+      - name: Generate github-app token
+        id: app-token
+        uses: getsentry/action-github-app-token@v2
+        with:
+          app_id: ${{ secrets.DEVOPS_APP_ID }}
+          private_key: ${{ secrets.DEVOPS_APP_PRIVATE_KEY }}
       - uses: actions/add-to-project@v0.5.0
         with:
           project-url: https://github.com/orgs/opendatahub-io/projects/40
-          github-token: ${{ secrets.GH_TOKEN }}
+          github-token: ${{ steps.app-token.outputs.token }}
+      - uses: actions/add-to-project@v0.5.0
+        with:
+          project-url: https://github.com/orgs/opendatahub-io/projects/45
+          github-token: ${{ steps.app-token.outputs.token }}

.github/workflows/code-quality.yaml

@@ -0,0 +1,42 @@
+---
+name: Code static analysis
+on: [pull_request]  # yamllint disable-line rule:truthy
+
+permissions:
+  contents: read
+
+jobs:
+  code-static-analysis:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Validate YAML files (best code practices check included)
+        id: validate-yaml-files
+        uses: ibiqlik/action-yamllint@v3.1.1
+        with:
+          config_file: ./ci/yamllint-config.yaml
+
+      # In some YAML files we use JSON strings, let's check these
+      - name: Validate JSON strings in YAML files (just syntax)
+        id: validate-json-strings-in-yaml-files
+        run: |
+          type json_verify || sudo apt-get install yajl-tools
+          bash ./ci/check-json.sh
+
+      - name: Validate JSON files (just syntax)
+        id: validate-json-files
+        run: |
+          type json_verify || sudo apt-get install yajl-tools
+          shopt -s globstar
+          ret_code=0
+          echo "-- Checking a regular '*.json' files"
+          for f in **/*.json; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
+          echo "-- Checking a 'Pipfile.lock' files"
+          for f in **/Pipfile.lock; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
+          echo "-- Checking a '*.ipynb' Jupyter notebook files"
+          for f in **/*.ipynb; do echo "Checking: '${f}"; echo -n "  > "; cat $f | json_verify || ret_code=1; done
+          if test "${ret_code}" -ne 0; then
+              echo "There were errors in some of the checked files. Please run `json_verify` on such files and fix issues there."
+          fi
+          exit "${ret_code}"

.github/workflows/notebooks-digest-updater-upstream.yaml

@@ -1,19 +1,20 @@
+---
 # The aim of this GitHub workflow is to update the params.env file with the latest digest.
-on:
+name: Update notebook image build commit hashes
+on:  # yamllint disable-line rule:truthy
   workflow_dispatch:
     inputs:
       branch:
         required: true
-        description: "Provide the name of the branch you want to update ex main, vYYYYx etc"
-      release-n:
-        required: true
-        description: "Provide release N version of the notebooks ex 2023a"
-  schedule:
-    - cron: "0 0 * * 5" #Scheduled every Friday
+        description: "Provide the name of the branch you want to update ex main, vYYYYx etc: "
+        # Put the scheduler on comment until automate the full release procedure
+        # schedule:
+        #   - cron: "0 0 * * 5"  #Scheduled every Friday
 env:
   DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }}
   BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }}
-  RELEASE_VERSION: ${{ github.event.inputs.release-n || '2023a' }}
+  RELEASE_VERSION_N: 2023b
+  RELEASE_VERSION_N_1: 2023a
 jobs:
   initialize:
     runs-on: ubuntu-latest
@@ -26,8 +27,8 @@ jobs:
           sudo apt-get -y update
           sudo apt-get -y install skopeo
 
-      # Checkout the release branch
-      - name: Checkout release branch
+      # Checkout the branch
+      - name: Checkout branch
         uses: actions/checkout@v3
         with:
           ref: ${{ env.BRANCH_NAME }}
@@ -40,7 +41,7 @@ jobs:
          git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }}
 
   update-n-version:
-    needs: [ initialize ]
+    needs: [initialize]
     runs-on: ubuntu-latest
     permissions:
       contents: write
@@ -50,17 +51,18 @@ jobs:
          git config --global user.email "github-actions[bot]@users.noreply.github.com"
          git config --global user.name "GitHub Actions"
 
-      # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023a
+      # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023b
       - name: Checkout upstream notebooks repo
         uses: actions/checkout@v3
         with:
-         repository: opendatahub-io/notebooks.git
-         ref: ${{ env.RELEASE_VERSION }}
-      - name: Retrive latest weekly commit hash from the release branch
-        id: hash
+          repository: opendatahub-io/notebooks.git
+          ref: ${{ env.RELEASE_VERSION_N }}
+
+      - name: Retrieve latest weekly commit hash from the release branch
+        id: hash-n
         shell: bash
         run: |
-          echo "HASH=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
+          echo "HASH_N=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
 
       # Checkout the release branch to apply the updates
       - name: Checkout release branch
@@ -70,40 +72,94 @@ jobs:
 
       - name: Fetch digest, and update the param.env file
         run: |
+              echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}}
               IMAGES=("odh-minimal-notebook-image-n" "odh-minimal-gpu-notebook-image-n" "odh-pytorch-gpu-notebook-image-n" "odh-generic-data-science-notebook-image-n" "odh-tensorflow-gpu-notebook-image-n" "odh-trustyai-notebook-image-n")
               for ((i=0;i<${#IMAGES[@]};++i)); do
                 image=${IMAGES[$i]}
                 echo $image
                 img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
                 registry=$(echo $img | cut -d '@' -f1)
-                src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"')
-                regex="$src_tag-${{ env.RELEASE_VERSION}}-\d+-${{ steps.hash.outputs.HASH }}"
+                src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
+                regex="$src_tag-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}"
                 latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
                 digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
                 output=$registry@$digest
                 echo $output
                 sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
               done
-              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update file via  ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+
+  update-n-1-version:
+    needs: [initialize, update-n-version]
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Configure Git
+        run: |
+         git config --global user.email "github-actions[bot]@users.noreply.github.com"
+         git config --global user.name "GitHub Actions"
+
+      # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023a
+      - name: Checkout upstream notebooks repo
+        uses: actions/checkout@v3
+        with:
+          repository: opendatahub-io/notebooks.git
+          ref: ${{ env.RELEASE_VERSION_N_1 }}
+
+      - name: Retrieve latest weekly commit hash from the release branch
+        id: hash-n-1
+        shell: bash
+        run: |
+          echo "HASH_N_1=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT}
+
+      # Checkout the release branch to apply the updates
+      - name: Checkout release branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ env.DIGEST_UPDATER_BRANCH }}
+
+      - name: Fetch digest, and update the param.env file
+        run: |
+              echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}}
+              IMAGES=("odh-minimal-notebook-image-n-1" "odh-minimal-gpu-notebook-image-n-1" "odh-pytorch-gpu-notebook-image-n-1" "odh-generic-data-science-notebook-image-n-1" "odh-tensorflow-gpu-notebook-image-n-1" "odh-trustyai-notebook-image-n-1")
+              for ((i=0;i<${#IMAGES[@]};++i)); do
+                image=${IMAGES[$i]}
+                echo $image
+                img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2)
+                registry=$(echo $img | cut -d '@' -f1)
+                src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//')
+                regex="$src_tag-${{ env.RELEASE_VERSION_N_1}}-\d+-${{ steps.hash-n-1.outputs.HASH_N_1 }}"
+                latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]')
+                digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"')
+                output=$registry@$digest
+                echo $output
+                sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env
+              done
+              git fetch origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin  ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin  ${{ env.DIGEST_UPDATER_BRANCH }}
+
+  # Creates the Pull Request
   open-pull-request:
-    needs: [ update-n-version ]
+    needs: [update-n-version, update-n-1-version]
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
     steps:
       - name: Checkout repo
         uses: actions/checkout@v3
+
       - name: pull-request
         uses: repo-sync/pull-request@v2
         with:
-          source_branch:  ${{ env.DIGEST_UPDATER_BRANCH }}
+          source_branch: ${{ env.DIGEST_UPDATER_BRANCH }}
           destination_branch: ${{ env.BRANCH_NAME}}
           github_token: ${{ secrets.GITHUB_TOKEN }}
           pr_label: "automated pr"
           pr_title: "[Digest Updater Action] Update notebook's imageStreams image tag to digest format"
           pr_body: |
-            :rocket: This is a automated PR
+            :rocket: This is an automated Pull Request.
 
-            _Created by `/.github/workflows/digest-updater.yaml`
+            This PR updates the `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1).
+            Created by `/.github/workflows/notebooks-digest-updater-upstream.yaml`
 
             :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes

.github/workflows/piplock-renewal-2023a.yml

@@ -1,17 +1,18 @@
-# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks 
+---
+# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks
 
 name: Weekly Pipfile.locks renewal on [2023a] branch
 
-on:
- # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1
- schedule:
-   - cron: "0 22 * * 1"
- workflow_dispatch:  # for manual trigger workflow from GH Web UI
- 
+on:  # yamllint disable-line rule:truthy
+  # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1
+  schedule:
+    - cron: "0 22 * * 1"
+  workflow_dispatch:  # for manual trigger workflow from GH Web UI
+
 jobs:
   build:
     runs-on: ubuntu-latest
-     
+
     permissions:
       # Give the default GITHUB_TOKEN write permission to commit and push the
       # added or changed files to the repository.

.github/workflows/piplock-renewal-2023b.yml

@@ -0,0 +1,47 @@
+---
+# This GitHub action is meant to be triggered weekly in order to update the pipfile.locks
+
+name: Weekly Pipfile.locks renewal on [2023b] branch
+
+on:  # yamllint disable-line rule:truthy
+  # Triggers the workflow every Monday at 22pm UTC am 0 22 * * 1
+  schedule:
+    - cron: "0 22 * * 1"
+  workflow_dispatch:  # for manual trigger workflow from GH Web UI
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+
+    permissions:
+      # Give the default GITHUB_TOKEN write permission to commit and push the
+      # added or changed files to the repository.
+      contents: write
+
+    steps:
+      # Checkout the paricular branch
+      - name: Checkout code from the release branch
+        uses: actions/checkout@v3
+        with:
+          ref: 2023b
+          token: ${{ secrets.GH_ACCESS_TOKEN }}
+
+      # Setup Python environment
+      - name: Setup Python environment
+        uses: actions/setup-python@v4
+        with:
+          python-version: |
+            3.8
+            3.9
+      - name: Install pipenv
+        run: pip install pipenv
+
+      # Runs the makefile recipe `refresh-pipfilelock-files` and push the chances back to the branch
+      - name: Run make refresh-pipfilelock-files and push the chances back to the branch
+        run: |
+          make refresh-pipfilelock-files
+          git config --global user.email "github-actions[bot]@users.noreply.github.com"
+          git config --global user.name "GitHub Actions"
+          git add .
+          git commit -m "Update the pipfile.lock via the weekly workflow action"
+          git push

.github/workflows/pr-merge-image-delete.yml

@@ -1,5 +1,6 @@
+---
 name: Delete quay image of PR once merged
-on:
+on:  # yamllint disable-line rule:truthy
   push:
     branches:
       - 'main'
@@ -69,15 +70,15 @@ jobs:
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-datascience-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-pytorch-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-trustyai-ubi8-python-3.8-pr-${{ env.PR }}
-          skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-datascience-anaconda-python-3.8-pr-${{ env.PR }}  
+          skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-datascience-anaconda-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:cuda-jupyter-minimal-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:cuda-jupyter-datascience-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:cuda-jupyter-tensorflow-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:runtime-minimal-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:runtime-datascience-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:runtime-pytorch-ubi8-python-3.8-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:runtime-cuda-tensorflow-ubi8-python-3.8-pr-${{ env.PR }}
-          
+
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-minimal-ubi9-python-3.9-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-datascience-ubi9-python-3.9-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:jupyter-pytorch-ubi9-python-3.9-pr-${{ env.PR }}
@@ -91,4 +92,4 @@ jobs:
           skopeo delete docker://${QUAY_IMAGE_REPO}:runtime-cuda-tensorflow-ubi9-python-3.9-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:codeserver-c9s-python-3.9-pr-${{ env.PR }}
           skopeo delete docker://${QUAY_IMAGE_REPO}:rstudio-c9s-python-3.9-pr-${{ env.PR }}
-          skopeo delete docker://${QUAY_IMAGE_REPO}:cuda-rstudio-c9s-python-3.9-pr-${{ env.PR }}
+          skopeo delete docker://${QUAY_IMAGE_REPO}:cuda-rstudio-c9s-python-3.9-pr-${{ env.PR }}