diff --git a/.github/workflows/notebooks-digest-updater-upstream.yaml b/.github/workflows/notebooks-digest-updater-upstream.yaml index dae67ed20..8e0553b45 100644 --- a/.github/workflows/notebooks-digest-updater-upstream.yaml +++ b/.github/workflows/notebooks-digest-updater-upstream.yaml @@ -51,18 +51,13 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "GitHub Actions" - # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2024a - - name: Checkout upstream notebooks repo - uses: actions/checkout@v4 - with: - repository: opendatahub-io/notebooks.git - ref: ${{ env.RELEASE_VERSION_N }} - - - name: Retrieve latest weekly commit hash from the release branch + # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name + - name: Retrive latest commit hash from the release branch id: hash-n shell: bash run: | - echo "HASH_N=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT} + PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1) + echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT} # Checkout the release branch to apply the updates - name: Checkout release branch @@ -70,13 +65,22 @@ jobs: with: ref: ${{ env.DIGEST_UPDATER_BRANCH }} - - name: Fetch digest, and update the params.env file + - name: Update the param.env file run: | echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}} - IMAGES=("odh-minimal-notebook-image-n" "odh-minimal-gpu-notebook-image-n" "odh-pytorch-gpu-notebook-image-n" "odh-generic-data-science-notebook-image-n" "odh-tensorflow-gpu-notebook-image-n" "odh-trustyai-notebook-image-n") + IMAGES=("odh-minimal-notebook-image-n" + "odh-minimal-gpu-notebook-image-n" + "odh-pytorch-gpu-notebook-image-n" + "odh-generic-data-science-notebook-image-n" + "odh-tensorflow-gpu-notebook-image-n" + "odh-trustyai-notebook-image-n" + "odh-codeserver-notebook-n" + "odh-rstudio-notebook-n" + "odh-rstudio-gpu-notebook-n") + for ((i=0;i<${#IMAGES[@]};++i)); do image=${IMAGES[$i]} - echo $image + echo "CHECKING: " $image img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2) registry=$(echo $img | cut -d '@' -f1) src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//') @@ -84,20 +88,38 @@ jobs: latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"') output=$registry@$digest - echo $output + echo "NEW:" $output sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env done - git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + else + echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}" + fi - - name: Fetch digest, and update the commit.env file + - name: Update the commit.env file run: | echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}} - COMMIT=("odh-minimal-notebook-image-commit-n" "odh-minimal-gpu-notebook-image-commit-n" "odh-pytorch-gpu-notebook-image-commit-n" "odh-generic-data-science-notebook-image-commit-n" "odh-tensorflow-gpu-notebook-image-commit-n" "odh-trustyai-notebook-image-commit-n") + COMMIT=("odh-minimal-notebook-image-commit-n" + "odh-minimal-gpu-notebook-image-commit-n" + "odh-pytorch-gpu-notebook-image-commit-n" + "odh-generic-data-science-notebook-image-commit-n" + "odh-tensorflow-gpu-notebook-image-commit-n" + "odh-trustyai-notebook-image-commit-n" + "odh-codeserver-notebook-n" + "odh-rstudio-notebook-n" + "odh-rstudio-gpu-notebook-n") + for val in "${COMMIT[@]}"; do echo $val sed -i "s|${val}=.*|${val}=${{ steps.hash-n.outputs.HASH_N }}|" manifests/base/commit.env done - git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + else + echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}" + fi + update-n-1-version: needs: [initialize, update-n-version] runs-on: ubuntu-latest @@ -109,18 +131,13 @@ jobs: git config --global user.email "github-actions[bot]@users.noreply.github.com" git config --global user.name "GitHub Actions" - # Get the latest weekly build commit hash: https://github.com/opendatahub-io/notebooks/commits/2023b - - name: Checkout upstream notebooks repo - uses: actions/checkout@v4 - with: - repository: opendatahub-io/notebooks.git - ref: ${{ env.RELEASE_VERSION_N_1 }} - - - name: Retrieve latest weekly commit hash from the release branch + # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name + - name: Retrive latest commit hash from the release branch id: hash-n-1 shell: bash run: | - echo "HASH_N_1=$(git rev-parse --short HEAD)" >> ${GITHUB_OUTPUT} + PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N_1&per_page=1) + echo "HASH_N_1=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT} # Checkout the release branch to apply the updates - name: Checkout release branch @@ -128,13 +145,22 @@ jobs: with: ref: ${{ env.DIGEST_UPDATER_BRANCH }} - - name: Fetch digest, and update the params.env file + - name: Update the param.env file run: | echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}} - IMAGES=("odh-minimal-notebook-image-n-1" "odh-minimal-gpu-notebook-image-n-1" "odh-pytorch-gpu-notebook-image-n-1" "odh-generic-data-science-notebook-image-n-1" "odh-tensorflow-gpu-notebook-image-n-1" "odh-trustyai-notebook-image-n-1") + IMAGES=("odh-minimal-notebook-image-n-1" + "odh-minimal-gpu-notebook-image-n-1" + "odh-pytorch-gpu-notebook-image-n-1" + "odh-generic-data-science-notebook-image-n-1" + "odh-tensorflow-gpu-notebook-image-n-1" + "odh-trustyai-notebook-image-n-1" + "odh-codeserver-notebook-n-1" + "odh-rstudio-notebook-n-1" + "odh-rstudio-gpu-notebook-n-1") + for ((i=0;i<${#IMAGES[@]};++i)); do image=${IMAGES[$i]} - echo $image + echo "CHECKING: " $image img=$(cat manifests/base/params.env | grep -E "${image}=" | cut -d '=' -f2) registry=$(echo $img | cut -d '@' -f1) src_tag=$(skopeo inspect docker://$img | jq '.Env[] | select(startswith("OPENSHIFT_BUILD_NAME=")) | split("=")[1]' | tr -d '"' | sed 's/-amd64$//') @@ -142,21 +168,37 @@ jobs: latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"') output=$registry@$digest - echo $output + echo "NEW:" $output sed -i "s|${image}=.*|${image}=$output|" manifests/base/params.env done - git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} - - - name: Fetch digest, and update the commit.env file + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/params.env && git commit -m "Update images for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + else + echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N_1}}" + fi + - name: Update the commit.env file run: | echo Latest commit is: ${{ steps.hash-n-1.outputs.HASH_N_1 }} on ${{ env.RELEASE_VERSION_N_1}} - COMMIT=("odh-minimal-notebook-image-commit-n-1" "odh-minimal-gpu-notebook-image-commit-n-1" "odh-pytorch-gpu-notebook-image-commit-n-1" "odh-generic-data-science-notebook-image-commit-n-1" "odh-tensorflow-gpu-notebook-image-commit-n-1" "odh-trustyai-notebook-image-commit-n-1") + COMMIT=("odh-minimal-notebook-image-commit-n-1" + "odh-minimal-gpu-notebook-image-commit-n-1" + "odh-pytorch-gpu-notebook-image-commit-n-1" + "odh-generic-data-science-notebook-image-commit-n-1" + "odh-tensorflow-gpu-notebook-image-commit-n-1" + "odh-trustyai-notebook-image-commit-n-1" + "odh-codeserver-notebook-n-1" + "odh-rstudio-notebook-n-1" + "odh-rstudio-gpu-notebook-n-1") + for val in "${COMMIT[@]}"; do echo $val sed -i "s|${val}=.*|${val}=${{ steps.hash-n-1.outputs.HASH_N_1 }}|" manifests/base/commit.env done - git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N-1 via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} - # Creates the Pull Request + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add manifests/base/commit.env && git commit -m "Update image commits for release N via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + else + echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}" + fi + open-pull-request: needs: [update-n-version, update-n-1-version] runs-on: ubuntu-latest @@ -173,11 +215,13 @@ jobs: destination_branch: ${{ env.BRANCH_NAME}} github_token: ${{ secrets.GITHUB_TOKEN }} pr_label: "automated pr" - pr_title: "[Digest Updater Action] Update notebook's imageStreams image tag to digest format" + pr_title: "[Digest Updater Action] Update Notebook Images" pr_body: | :rocket: This is an automated Pull Request. - - This PR updates the `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1). Created by `/.github/workflows/notebooks-digest-updater-upstream.yaml` + This PR updates the following files: + - `manifests/base/params.env` file with the latest updated SHA digests of the notebooks (N & N-1). + - `manifests/base/commit.env` file with the latest commit (N & N-1). + :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes diff --git a/.github/workflows/runtimes-digest-updater-upstream.yaml b/.github/workflows/runtimes-digest-updater-upstream.yaml new file mode 100644 index 000000000..859362213 --- /dev/null +++ b/.github/workflows/runtimes-digest-updater-upstream.yaml @@ -0,0 +1,133 @@ +--- +# The aim of this GitHub workflow is to update the runtimes across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths. +name: Update runtime images +on: # yamllint disable-line rule:truthy + workflow_dispatch: + inputs: + branch: + required: true + description: "Provide the name of the branch you want to update ex main, vYYYYx etc: " + # Put the scheduler on comment until automate the full release procedure + # schedule: + # - cron: "0 0 * * 5" #Scheduled every Friday +env: + DIGEST_UPDATER_BRANCH: digest-updater-${{ github.run_id }} + BRANCH_NAME: ${{ github.event.inputs.branch || 'main' }} + RELEASE_VERSION_N: 2024a +jobs: + initialize: + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Install Skopeo CLI + shell: bash + run: | + sudo apt-get -y update + sudo apt-get -y install skopeo + + # Checkout the branch + - name: Checkout branch + uses: actions/checkout@v4 + with: + ref: ${{ env.BRANCH_NAME }} + + # Create a new branch + - name: Create a new branch + run: | + echo ${{ env.DIGEST_UPDATER_BRANCH }} + git checkout -b ${{ env.DIGEST_UPDATER_BRANCH }} + git push --set-upstream origin ${{ env.DIGEST_UPDATER_BRANCH }} + + update-runtimes: + needs: [initialize] + runs-on: ubuntu-latest + permissions: + contents: write + steps: + - name: Configure Git + run: | + git config --global user.email "github-actions[bot]@users.noreply.github.com" + git config --global user.name "GitHub Actions" + + # Get latest build commit from the https://github.com/opendatahub-io/notebooks/${release_branch} using this as identifier for the latest tag name + - name: Retrive latest commit hash from the release branch + id: hash-n + shell: bash + run: | + PAYLOAD=$(curl --silent -H 'Accept: application/vnd.github.v4.raw' https://api.github.com/repos/opendatahub-io/notebooks/commits?sha=$RELEASE_VERSION_N&per_page=1) + echo "HASH_N=$(echo $PAYLOAD | jq -r '.[0].sha[0:7]')" >> ${GITHUB_OUTPUT} + + # Checkout the release branch to apply the updates + - name: Checkout release branch + uses: actions/checkout@v4 + with: + ref: ${{ env.DIGEST_UPDATER_BRANCH }} + + - name: Update Runtimes + run: | + echo Latest commit is: ${{ steps.hash-n.outputs.HASH_N }} on ${{ env.RELEASE_VERSION_N}} + PATHS=("jupyter/datascience/anaconda-python-3.8/runtime-images/datascience-ubi8-py38.json" + "jupyter/datascience/anaconda-python-3.8/runtime-images/pytorch-ubi8-py38.json" + "jupyter/datascience/anaconda-python-3.8/runtime-images/tensorflow-ubi8-py38.json" + "jupyter/datascience/anaconda-python-3.8/runtime-images/ubi8-py38.json" + "jupyter/datascience/ubi8-python-3.8/runtime-images/datascience-ubi8-py38.json" + "jupyter/datascience/ubi8-python-3.8/runtime-images/pytorch-ubi8-py38.json" + "jupyter/datascience/ubi8-python-3.8/runtime-images/tensorflow-ubi8-py38.json" + "jupyter/datascience/ubi8-python-3.8/runtime-images/ubi8-py38.json" + "jupyter/datascience/ubi9-python-3.9/runtime-images/datascience-ubi9-py39.json" + "jupyter/datascience/ubi9-python-3.9/runtime-images/pytorch-ubi9-py39.json" + "jupyter/datascience/ubi9-python-3.9/runtime-images/tensorflow-ubi9-py39.json" + "jupyter/datascience/ubi9-python-3.9/runtime-images/ubi9-py39.json") + + for ((i=0;i<${#PATHS[@]};++i)); do + path=${PATHS[$i]} + img=$(cat ${path} | jq -r '.metadata.image_name') + name=$(echo "$path" | sed 's#.*runtime-images/\(.*\)-py.*#\1#') + py_version=$(echo "$path" | grep -o 'python-[0-9]\.[0-9]') + # Handling specific cases + if [[ $name == *tensorflow* ]]; then + name="cuda-$name" + elif [[ $name == ubi* ]]; then + name="minimal-$name" + fi + registry=$(echo $img | cut -d '@' -f1) + regex="runtime-$name-$py_version-${{ env.RELEASE_VERSION_N}}-\d+-${{ steps.hash-n.outputs.HASH_N }}" + echo "CHECKING: " $regex + latest_tag=$(skopeo inspect docker://$img | jq -r --arg regex "$regex" '.RepoTags | map(select(. | test($regex))) | .[0]') + digest=$(skopeo inspect docker://$registry:$latest_tag | jq .Digest | tr -d '"') + output=$registry@$digest + echo "NEW: " $output + jq --arg output "$output" '.metadata.image_name = $output' "$path" > "$path.tmp" && mv "$path.tmp" "$path" + done + if [[ $(git status --porcelain | wc -l) -gt 0 ]]; then + git fetch origin ${{ env.DIGEST_UPDATER_BRANCH }} && git pull origin ${{ env.DIGEST_UPDATER_BRANCH }} && git add jupyter/datascience/* && git commit -m "Update file via ${{ env.DIGEST_UPDATER_BRANCH }} GitHub action" && git push origin ${{ env.DIGEST_UPDATER_BRANCH }} + else + echo "There were no changes detected in the images for the ${{ env.RELEASE_VERSION_N}}" + fi + + open-pull-request: + needs: [update-runtimes] + runs-on: ubuntu-latest + permissions: + pull-requests: write + steps: + - name: Checkout repo + uses: actions/checkout@v4 + + - name: pull-request + uses: repo-sync/pull-request@v2 + with: + source_branch: ${{ env.DIGEST_UPDATER_BRANCH }} + destination_branch: ${{ env.BRANCH_NAME}} + github_token: ${{ secrets.GITHUB_TOKEN }} + pr_label: "automated pr" + pr_title: "[Digest Updater Action] Update Runtimes Images" + pr_body: | + :rocket: This is an automated Pull Request. + Created by `/.github/workflows/runtimes-digest-updater-upstream.yaml` + + This PR updates the following files: + - All the runtime images across `/jupyter/datascience/ubi*-python-*/runtime-images/*.json` paths + + :exclamation: **IMPORTANT NOTE**: Remember to delete the ` ${{ env.DIGEST_UPDATER_BRANCH }}` branch after merging the changes