From cff8ccb0e47382148ce8bca646c434e75b675259 Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Fri, 8 Nov 2024 13:36:33 +0200 Subject: [PATCH 1/4] adding the create relase workflow syncing publish and tools and excluding test.sh write empty receipts on poll usns fixing download usn asset pattern add check if there are new usns using -c flag on jq adding if the stack or the arch is new on the json object using arch_name instead of arch skipping steps in case of not previous release removing debug code fixing getting stacks and arch diffs --- .github/workflows/create-release.yml | 1194 +++++++++++++++++++++----- 1 file changed, 1000 insertions(+), 194 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index d410838..44be93f 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -22,66 +22,300 @@ on: concurrency: release env: - BUILD_RECEIPT_FILENAME: "build-receipt.cyclonedx.json" - RUN_RECEIPT_FILENAME: "run-receipt.cyclonedx.json" + STACKS_FILEPATH: "stacks/images.json" PATCHED_USNS_FILENAME: "patched-usns.json" jobs: + preparation: + name: Preparation + runs-on: ubuntu-22.04 + outputs: + stacks_added: ${{ steps.get-stacks.outputs.stacks_added }} + stacks: ${{ steps.get-stacks.outputs.stacks }} + support_usns: ${{ steps.polling-os-type.outputs.support_usns }} + architectures: ${{ steps.lookup.outputs.platforms }} + archs_added: ${{ steps.lookup.outputs.platforms_added }} + polling_type: ${{ steps.polling-os-type.outputs.polling_type }} + github_repo_name: ${{ steps.repo.outputs.github_repo_name }} + registry_repo_name: ${{ steps.repo.outputs.registry_repo_name }} + repo_owner: ${{ steps.repo.outputs.repo_owner }} + default_stack_dir: ${{ steps.lookup.outputs.default_stack_dir }} + stack_files_dir: ${{ steps.get-stacks.outputs.stack_files_dir }} + steps: + - name: Checkout repo + uses: actions/checkout@v4 + with: + fetch-depth: 0 # gets full history + + - name: Get stacks images + id: get-stacks + run: | + + if [[ -f ${{ env.STACKS_FILEPATH }} ]]; then + current_stacks=$(jq '[.images[] | + . + + { + "create_build_image": (.create_build_image // false), + "is_new": true + }]' ${{ env.STACKS_FILEPATH }} ) + + stack_files_dir="$(dirname "${{ env.STACKS_FILEPATH }}")" + + git show $(git describe --tags --abbrev=0):"${{ env.STACKS_FILEPATH }}" > ./previous_images.json + + if [ -f "./previous_images.json" ]; then + previous_stacks=$(jq '[.images[] | + . + + { + "create_build_image": (.create_build_image // false), + "is_new": false + }]' "./previous_images.json" ) + else + previous_stacks="[]" + fi + + stacks=$(echo "$current_stacks" | jq -c --argjson prev "$previous_stacks" ' + map( + . as $curr | + ($prev[] | select(.name == $curr.name)) // $curr + ) + ') + else + stacks=$( + cat <> "$GITHUB_OUTPUT" + + ## Filter stacks array to include the minimum number of attributes + stacks=$(echo "$stacks" | jq 'map({ + name, + config_dir, + output_dir, + build_image, + build_receipt_filename, + run_image, + run_receipt_filename, + create_build_image, + base_build_container_image, + base_run_container_image, + is_new + })') + + stacks=$(jq -c <<< "$stacks" ) + printf "stacks=%s\n" "${stacks}" >> "$GITHUB_OUTPUT" + + - name: Polling OS type + id: polling-os-type + run: | + support_usns=true + + if [[ -f ${{ env.STACKS_FILEPATH }} ]]; then + support_usns=$( jq '.support_usns' ${{ env.STACKS_FILEPATH }} ) + fi + + if [ $support_usns == true ]; then + echo "polling_type=usn" >> "$GITHUB_OUTPUT" + else + echo "polling_type=hash" >> "$GITHUB_OUTPUT" + fi + + echo "support_usns=${support_usns}" >> "$GITHUB_OUTPUT" + + - name: Get Repository Name + id: repo + run: | + full=${{ github.repository }} + # Strip off the org and slash from repo name + # paketo-buildpacks/jammy-base-stack --> jammy-base-stack + repo=$(echo "${full}" | sed 's/^.*\///') + echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + + # Strip off 'stack' suffix from repo name + # paketo-buildpacks/jammy-base-stack --> jammy-base + registry_repo="${repo//-stack/}" + echo "registry_repo_name=${registry_repo}" >> "$GITHUB_OUTPUT" + + # translates 'paketo-buildpacks' to 'paketobuildpacks' + repo_owner="${GITHUB_REPOSITORY_OWNER/-/}" + printf "repo_owner=%s\n" "${repo_owner}" >> "$GITHUB_OUTPUT" + + - name: Lookup Supported Architectures + id: lookup + run: | + #! /usr/bin/env bash + + set -euo pipefail + shopt -s inherit_errexit + + # install yj to parse TOML + curl -L $(curl -sL https://api.github.com/repos/sclevine/yj/releases/latest | jq -r '.assets[] | select(.name=="yj-linux-amd64").browser_download_url') -o yj + chmod +x yj + + default_stack_dir=$(echo '${{ steps.get-stacks.outputs.stacks }}' | jq -r '.[] | select(.name=="default") | .config_dir') + echo "default_stack_dir=${default_stack_dir}" >> "$GITHUB_OUTPUT" + + current_platforms="$(cat "$default_stack_dir/stack.toml" | ./yj -tj | jq -c '[.platforms[] | { name: sub("linux/"; "") , is_new: true }]')" + + # get previous platforms + git show $(git describe --tags --abbrev=0):"$default_stack_dir/stack.toml" > ./previous_stack.toml + + previous_platforms="$(cat ./previous_stack.toml | ./yj -tj | jq -c '[.platforms[] | { name: sub("linux/"; "") , is_new: false }]')" + + platforms=$(echo "$current_platforms" | jq -c --argjson prev "$previous_platforms" ' + map( + . as $curr | + ($prev[] | select(.name == $curr.name)) // $curr + ) + ') + + echo "platforms=${platforms}" >> "$GITHUB_OUTPUT" + + # The following job is specific to Ubuntu images. It checks for new + # USNs (Ubuntu Security Notices) and triggers the flow to create + # a new release with the latest images that have the USNs patched. poll_usns: name: Poll USNs runs-on: ubuntu-22.04 + needs: [preparation] + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} outputs: - usns: ${{ steps.usns.outputs.usns }} + usns: ${{ steps.new_usns.outputs.usns }} steps: + - name: Generate receipt asset patterns + id: receipt_pattern + run: | + if [ "${{ matrix.arch.name }}" = "amd64" ]; then + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + echo "build=${{ needs.preparation.outputs.github_repo_name }}-\\d+\\.\\d+(\\.\\d+)?-${{ matrix.stacks.build_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi + echo "run=${{ needs.preparation.outputs.github_repo_name }}-\\d+\\.\\d+(\\.\\d+)?-${{ matrix.stacks.run_receipt_filename }}" >> "$GITHUB_OUTPUT" + else + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + echo "build=${{ matrix.arch.name }}-${{ matrix.stacks.build_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi + echo "run=${{ matrix.arch.name }}-${{ matrix.stacks.run_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi + + - name: Write Empty Previous Receipts + if: ${{ matrix.arch.is_new == true || matrix.stacks.is_new == true }} + run: | + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + if [ ! -f "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + fi + fi + if [ ! -f "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + fi + + - name: Check for Previous Releases + id: check_previous + run: | + gh auth status + # shellcheck disable=SC2046 + if [ $(gh api "/repos/${{ github.repository }}/releases" | jq -r 'length') -eq 0 ]; then + echo "exists=false" >> "$GITHUB_OUTPUT" + exit 0 + fi + echo "exists=true" >> "$GITHUB_OUTPUT" + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + - name: Find and Download Previous Build Receipt + if: ${{ matrix.arch.is_new == false && matrix.stacks.is_new == false && steps.check_previous.outputs.exists == 'true' }} id: previous_build uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.BUILD_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.receipt_pattern.outputs.build }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.BUILD_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + strict: true - name: Find and Download Previous Run Receipt + if: ${{ matrix.arch.is_new == false && matrix.stacks.is_new == false && steps.check_previous.outputs.exists == 'true' }} id: previous_run uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.RUN_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.receipt_pattern.outputs.run }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.RUN_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + strict: true - name: Get Package List id: packages if: ${{ steps.previous_build.outputs.output_path != '' && steps.previous_run.outputs.output_path != '' }} uses: paketo-buildpacks/github-config/actions/stack/generate-package-list@main with: - build_receipt: "${{ github.workspace }}/${{ env.BUILD_RECEIPT_FILENAME }}" - run_receipt: "${{ github.workspace }}/${{ env.RUN_RECEIPT_FILENAME }}" + build_receipt: "${{ github.workspace }}/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + run_receipt: "${{ github.workspace }}/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + + - name: Generate USNs download asset pattern + id: usn_download_pattern + run: | + arch_prefix="" + if [ "${{ matrix.arch.name }}" = "amd64" ]; then + arch_prefix="" + else + arch_prefix="${{ matrix.arch.name }}" + fi + + stack_name_prefix="" + if [ "${{ matrix.stacks.name }}" = "default" ]; then + stack_name_prefix="" + else + stack_name_prefix="${{ matrix.stacks.name }}" + fi + + pattern=$(echo '["\\d+.\\d+(.\\d+)?","'"$stack_name_prefix"'", "'"$arch_prefix"'", "${{ env.PATCHED_USNS_FILENAME }}"]' | jq -r 'map(select(length > 0)) | join("-")') + + echo "pattern=$pattern" >> "$GITHUB_OUTPUT" - name: Find and Download Previous Patched USNs id: download_patched uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.PATCHED_USNS_FILENAME }}" + asset_pattern: "${{ steps.usn_download_pattern.outputs.pattern }}" search_depth: "-1" # Search all releases repo: ${{ github.repository }} - output_path: "/github/workspace/${{ env.PATCHED_USNS_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}-previous" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Output Patched USNs as JSON String id: patched if: ${{ steps.download_patched.outputs.output_path != '' }} run: | - patched=$(jq --compact-output . < "${GITHUB_WORKSPACE}/${{ env.PATCHED_USNS_FILENAME }}") + patched=$(jq --compact-output . < "${GITHUB_WORKSPACE}/${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}-previous") printf "patched=%s\n" "${patched}" >> "$GITHUB_OUTPUT" - name: Get Stack Distribution Name id: distro run: | # Extract distro from repo name: - # paketo-buildpacks/noble-tiny-stack --> noble + # paketo-buildpacks/jammy-tiny-stack --> jammy distro="$(echo "${{ github.repository }}" | sed 's/^.*\///' | sed 's/\-.*$//')" echo "Ubuntu distribution: ${distro}" printf "distro=%s\n" "${distro}" >> "$GITHUB_OUTPUT" @@ -94,16 +328,140 @@ jobs: packages: ${{ steps.packages.outputs.packages }} last_usns: ${{ steps.patched.outputs.patched }} + - name: Write USNs File + id: write_usns + run: | + jq . <<< "${USNS}" > "./${USNS_PATH}" + echo "usns=./${USNS_PATH}" >> "$GITHUB_OUTPUT" + env: + USNS_PATH: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + USNS: ${{ steps.usns.outputs.usns }} + + - name: Upload USNs file + uses: actions/upload-artifact@v4 + with: + name: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + path: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" + + - name: Are any new USNs + id: new_usns + run: | + new_usns_length=$(cat "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" | jq 'length') + + if [ "$new_usns_length" -ge 0 ]; then + echo "usns=true" >> "$GITHUB_OUTPUT" + fi + + # The job below checks if new images are available on the registry + # based on the sha256 checksum. If yes, it triggers the flow + # to create a new release with the latest images + poll_images: + name: Poll Images based on the hash code + runs-on: ubuntu-22.04 + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} + needs: preparation + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} + outputs: + images_need_update: ${{ steps.compare_previous_and_current_sha256_hash_codes.outputs.images_need_update }} + steps: + - name: Generate hash code asset patterns + id: hashcode_pattern + run: | + if [ "${{ matrix.arch.name }}" = "amd64" ]; then + echo "build=${{ needs.preparation.outputs.github_repo_name }}-\\d+.\\d+(.\\d+)?-${{ matrix.stacks.build_image }}.oci.sha256" >> "$GITHUB_OUTPUT" + echo "run=${{ needs.preparation.outputs.github_repo_name }}-\\d+.\\d+(.\\d+)?-${{ matrix.stacks.run_image }}.oci.sha256" >> "$GITHUB_OUTPUT" + else + echo "build=-${{ matrix.arch.name }}-${{ matrix.stacks.build_image }}.oci.sha256" >> "$GITHUB_OUTPUT" + echo "run=-${{ matrix.arch.name }}-${{ matrix.stacks.run_image }}.oci.sha256" >> "$GITHUB_OUTPUT" + fi + + - name: Find and Download Previous build image hash code of stack ${{ matrix.stacks.build_image }} + if: ${{ matrix.stacks.create_build_image == true }} + uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main + with: + asset_pattern: "${{ steps.hashcode_pattern.outputs.build }}" + search_depth: 1 + repo: ${{ github.repository }} + output_path: "./previous_${{ matrix.arch.name }}-${{ matrix.stacks.build_image }}.oci.sha256" + token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + + - name: Find and Download Previous run image hash code of stack ${{ matrix.stacks.run_image }} + uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main + with: + asset_pattern: "${{ steps.hashcode_pattern.outputs.run }}" + search_depth: 1 + repo: ${{ github.repository }} + output_path: "./previous_${{ matrix.arch.name }}-${{ matrix.stacks.run_image }}.oci.sha256" + token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} + + - name: Get current build image hash code of ${{ matrix.stacks.name }} stack with arch ${{ matrix.arch.name }} + if: ${{ matrix.stacks.create_build_image == true }} + run: | + skopeo inspect --format "{{.Digest}}" ${{ matrix.stacks.base_build_container_image }} > ./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + - name: Get current run image hash code of ${{ matrix.stacks.name }} stack with arch ${{ matrix.arch.name }} + run: | + skopeo inspect --format "{{.Digest}}" ${{ matrix.stacks.base_run_container_image }} > ./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + - name: Create empty image hash codes + run: | + if [ ! -f "./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" ]; then + touch "./hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" + fi + if [ ! -f "./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" ]; then + touch "./hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }}" + fi + + - name: Upload run image hash code + uses: actions/upload-artifact@v4 + with: + name: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error + + - name: Upload build image hash code + if: ${{ matrix.stacks.create_build_image == true }} + uses: actions/upload-artifact@v4 + with: + name: hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error + + - name: Compare previous and current hash codes + id: compare_previous_and_current_sha256_hash_codes + run: | + if [ "$(cat previous_${{ matrix.arch.name }}-${{ matrix.stacks.run_image }}.oci.sha256)" != "$(cat hash-code-current-run-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }})" ]; then + echo "images_need_update=true" >> "$GITHUB_OUTPUT" + fi + + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + + if [ "$(cat previous_${{ matrix.arch.name }}-${{ matrix.stacks.build_image }}.oci.sha256)" != "$(cat hash-code-current-build-image-${{ matrix.arch.name }}-${{ matrix.stacks.name }})" ]; then + echo "images_need_update=true" >> "$GITHUB_OUTPUT" + fi + fi + + # If there is no change on the usns, and there is no change on the image hash codes + # and the event is schedule, then there is no need to run below workflow as nothing has changed stack_files_changed: name: Determine If Stack Files Changed runs-on: ubuntu-22.04 - needs: poll_usns - if: ${{ ! ( needs.poll_usns.outputs.usns == '[]' && github.event_name == 'schedule' ) }} + needs: [ preparation, poll_usns, poll_images ] + if: ${{ + !failure() && !cancelled() && + !( + (needs.poll_images.outputs.images_need_update == null && + needs.poll_usns.outputs.usns == null ) && + github.event_name == 'schedule' + ) }} outputs: stack_files_changed: ${{ steps.compare.outputs.stack_files_changed }} steps: - name: Checkout With History - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # gets full history @@ -111,7 +469,7 @@ jobs: id: compare run: | # shellcheck disable=SC2046 - changed="$(git diff --name-only $(git describe --tags --abbrev=0) -- stack)" + changed=$(git diff --name-only $(git describe --tags --abbrev=0) -- "${{ needs.preparation.outputs.stack_files_dir }}") if [ -z "${changed}" ] then echo "No relevant files changed since previous release." @@ -126,86 +484,143 @@ jobs: name: Run If Stack Files Changed runs-on: ubuntu-22.04 needs: [stack_files_changed] - if: ${{ needs.stack_files_changed.outputs.stack_files_changed == 'true' }} + if: ${{ !failure() && !cancelled() && needs.stack_files_changed.outputs.stack_files_changed == 'true' }} steps: - name: Run if stack files changed run: | echo "stack files have changed" + usns_or_sha_changed: + name: USNs or SHAs have changed + runs-on: ubuntu-22.04 + outputs: + changed: ${{ steps.usns_or_sha_changed.outputs.changed }} + needs: [ preparation, poll_usns, poll_images ] + if: ${{ !failure() && !cancelled() }} + steps: + - name: Check USNs or SHAs have changed + id: usns_or_sha_changed + run: | + if [ '${{ needs.poll_images.result }}' != 'skipped' ]; then + echo "Poll images job did not skip" + if [ '${{ needs.poll_images.outputs.images_need_update }}' = "true" ]; then + echo "SHAs have changed" + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "SHAs have not changed" + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + + if [ '${{ needs.poll_usns.result }}' != 'skipped' ]; then + echo "Poll USNs did not skip" + if [ '${{ needs.poll_usns.outputs.usns }}' = "true" ]; then + echo "USNs have changed" + echo "changed=true" >> "$GITHUB_OUTPUT" + else + echo "USNs have not changed" + echo "changed=false" >> "$GITHUB_OUTPUT" + fi + exit 0 + fi + create_stack: name: Create Stack - needs: poll_usns - if: ${{ ! ( needs.poll_usns.outputs.usns == '[]' && github.event_name == 'schedule' ) }} + needs: [ preparation, usns_or_sha_changed ] + # If there is no change on the usns, and there is no change on the image hash codes + # and the event is schedule, then there is no need to run below workflow as nothing has changed + if: ${{ !failure() && !cancelled() && !( needs.usns_or_sha_changed.outputs.changed == 'false' && github.event_name == 'schedule') }} runs-on: ubuntu-22.04 + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 # https://github.com/docker/setup-qemu-action - name: Set up QEMU - uses: docker/setup-qemu-action@v1 + uses: docker/setup-qemu-action@v3 - - name: Create stack + - name: Create stack ${{ matrix.stacks.name }} id: create-stack run: | - ./scripts/create.sh + scripts/create.sh --stack-dir ${{ matrix.stacks.config_dir }} \ + --build-dir ${{ matrix.stacks.output_dir }} - name: Generate Package Receipts id: receipts run: | - scripts/receipts.sh --build-image "${GITHUB_WORKSPACE}/build/build.oci" \ - --run-image "${GITHUB_WORKSPACE}/build/run.oci" \ - --build-receipt ${{ env.BUILD_RECEIPT_FILENAME }} \ - --run-receipt ${{ env.RUN_RECEIPT_FILENAME }} - echo "build_receipt=${{ env.BUILD_RECEIPT_FILENAME }}" >> "$GITHUB_OUTPUT" - echo "run_receipt=${{ env.RUN_RECEIPT_FILENAME }}" >> "$GITHUB_OUTPUT" + scripts/receipts.sh --build-image "${{ matrix.stacks.output_dir }}/build.oci" \ + --run-image "${{ matrix.stacks.output_dir }}/run.oci" \ + --build-receipt current-build-receipt-${{ matrix.stacks.name }} \ + --run-receipt current-run-receipt-${{ matrix.stacks.name }} - name: Upload run image - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: current-run-image - path: build/run.oci + name: current-run-image-${{ matrix.stacks.name }} + path: "${{ matrix.stacks.output_dir }}/run.oci" + if-no-files-found: error - name: Upload build image - uses: actions/upload-artifact@v3 + if: ${{ matrix.stacks.create_build_image == true }} + uses: actions/upload-artifact@v4 with: - name: current-build-image - path: build/build.oci + name: current-build-image-${{ matrix.stacks.name }} + path: "${{ matrix.stacks.output_dir }}/build.oci" + if-no-files-found: error - name: Upload Build receipt - uses: actions/upload-artifact@v3 + if: ${{ matrix.stacks.create_build_image == true }} + uses: actions/upload-artifact@v4 with: - name: current-build-receipt - path: ${{ steps.receipts.outputs.build_receipt }} + name: current-build-receipt-${{ matrix.stacks.name }} + path: "*current-build-receipt-${{ matrix.stacks.name }}" + if-no-files-found: error - name: Upload Run receipt - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 with: - name: current-run-receipt - path: ${{ steps.receipts.outputs.run_receipt }} + name: current-run-receipt-${{ matrix.stacks.name }} + path: "*current-run-receipt-${{ matrix.stacks.name }}" + if-no-files-found: error diff: name: Diff Packages + if: ${{ !cancelled() && !failure() && needs.create_stack.result != 'skipped' }} outputs: - build_added: ${{ steps.build_diff.outputs.added }} - build_modified: ${{ steps.build_diff.outputs.modified }} - build_removed_with_force: ${{ steps.build_diff.outputs.removed }} - run_added: ${{ steps.run_diff.outputs.added }} - run_modified: ${{ steps.run_diff.outputs.modified }} - run_removed_with_force: ${{ steps.run_diff.outputs.removed }} removed_with_force: ${{ steps.removed_with_force.outputs.packages_removed }} - needs: [ create_stack ] + needs: [ create_stack, preparation ] runs-on: ubuntu-22.04 + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} steps: - - name: Download Build Receipt - uses: actions/download-artifact@v3 + - name: Download Current Receipt(s) + uses: actions/download-artifact@v4 with: - name: current-build-receipt + pattern: current-*-receipt-${{ matrix.stacks.name }} + path: receipt-files - - name: Download Run Receipt - uses: actions/download-artifact@v3 - with: - name: current-run-receipt + - name: Move/Rename Build receipt properly + if: ${{ matrix.stacks.create_build_image == true }} + run: | + arch_prefix="${{ matrix.arch.name }}-" + if [[ "${{ matrix.arch.name }}" == "amd64" ]]; then + arch_prefix="" + fi + mv receipt-files/current-build-receipt-${{ matrix.stacks.name }}/${arch_prefix}current-build-receipt-${{ matrix.stacks.name }} ./${{ matrix.arch.name }}-current-build-receipt-${{ matrix.stacks.name }} + + - name: Move/Rename Run receipt properly + run: | + arch_prefix="${{ matrix.arch.name }}-" + if [[ "${{ matrix.arch.name }}" == "amd64" ]]; then + arch_prefix="" + fi + mv receipt-files/current-run-receipt-${{ matrix.stacks.name }}/${arch_prefix}current-run-receipt-${{ matrix.stacks.name }} ./${{ matrix.arch.name }}-current-run-receipt-${{ matrix.stacks.name }} - name: Check for Previous Releases id: check_previous @@ -221,56 +636,81 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Write Empty Previous Receipts - if: ${{ steps.check_previous.outputs.exists == 'false' }} + if: ${{ steps.check_previous.outputs.exists == 'false' || matrix.arch.is_new == true || matrix.stacks.is_new == true }} + run: | + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + if [ ! -f "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + fi + fi + if [ ! -f "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" ]; then + echo '{"components":[]}' > "./${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + fi + + - name: Previous receipt download filename pattern + if: ${{ steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} + id: previous_receipt_download_pattern run: | - echo '{"components":[]}' > "${{ github.workspace }}/previous-${{ env.BUILD_RECEIPT_FILENAME }}" - echo '{"components":[]}' > "${{ github.workspace }}/previous-${{ env.RUN_RECEIPT_FILENAME }}" + if [ "${{ matrix.arch.name }}" = "amd64" ]; then + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + echo "build_filename=${{ needs.preparation.outputs.github_repo_name }}-\\d+\\.\\d+(\\.\\d+)?-${{ matrix.stacks.build_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi + echo "run_filename=${{ needs.preparation.outputs.github_repo_name }}-\\d+\\.\\d+(\\.\\d+)?-${{ matrix.stacks.run_receipt_filename }}" >> "$GITHUB_OUTPUT" + else + if [ ${{ matrix.stacks.create_build_image }} == true ]; then + echo "build_filename=${{ matrix.arch.name }}-${{ matrix.stacks.build_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi + echo "run_filename=${{ matrix.arch.name }}-${{ matrix.stacks.run_receipt_filename }}" >> "$GITHUB_OUTPUT" + fi - name: Find and Download Previous Build Receipt - if: ${{ steps.check_previous.outputs.exists == 'true' }} + if: ${{ matrix.stacks.create_build_image == true && steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.BUILD_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.previous_receipt_download_pattern.outputs.build_filename }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/previous-${{ env.BUILD_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Find and Download Previous Run Receipt - if: ${{ steps.check_previous.outputs.exists == 'true' }} + if: ${{ steps.check_previous.outputs.exists == 'true' && matrix.arch.is_new == false && matrix.stacks.is_new == false }} uses: paketo-buildpacks/github-config/actions/release/find-and-download-asset@main with: - asset_pattern: "${{ env.RUN_RECEIPT_FILENAME }}" + asset_pattern: "${{ steps.previous_receipt_download_pattern.outputs.run_filename }}" search_depth: 1 repo: ${{ github.repository }} - output_path: "/github/workspace/previous-${{ env.RUN_RECEIPT_FILENAME }}" + output_path: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} - name: Compare Build Packages id: build_diff + if: ${{ matrix.stacks.create_build_image == true }} uses: paketo-buildpacks/github-config/actions/stack/diff-package-receipts@main with: - previous: "/github/workspace/previous-${{ env.BUILD_RECEIPT_FILENAME }}" - current: "/github/workspace/${{ env.BUILD_RECEIPT_FILENAME }}" + previous: "/github/workspace/${{ matrix.arch.name }}-previous-build-receipt-${{ matrix.stacks.name }}" + current: "/github/workspace/${{ matrix.arch.name }}-current-build-receipt-${{ matrix.stacks.name }}" - name: Compare Run Packages id: run_diff uses: paketo-buildpacks/github-config/actions/stack/diff-package-receipts@main with: - previous: "/github/workspace/previous-${{ env.RUN_RECEIPT_FILENAME }}" - current: "/github/workspace/${{ env.RUN_RECEIPT_FILENAME }}" + previous: "/github/workspace/${{ matrix.arch.name }}-previous-run-receipt-${{ matrix.stacks.name }}" + current: "/github/workspace/${{ matrix.arch.name }}-current-run-receipt-${{ matrix.stacks.name }}" - name: Fail If Packages Removed id: removed_with_force run: | - build=$(jq '. | length' <<< "${BUILD_REMOVED}") - echo "Build packages removed: ${build}" + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + build=$(jq '. | length' <<< "${BUILD_REMOVED}") + echo "Build packages removed: ${build}" + fi run=$(jq '. | length' <<< "${RUN_REMOVED}") echo "Run packages removed: ${run}" # only fail if packages are removed AND the release has not been forced - if [ "${build}" -gt 0 ] || [ "${run}" -gt 0 ]; then + if ( [ "${build}" -gt 0 ] && [ "${{ matrix.stacks.create_build_image }}" == "true" ] ) || [ "${run}" -gt 0 ]; then if [ "${{ github.event.inputs.force }}" != 'true' ]; then echo "Packages removed without authorization. Stack cannot be released." exit 1 @@ -284,6 +724,111 @@ jobs: BUILD_REMOVED: ${{ steps.build_diff.outputs.removed }} RUN_REMOVED: ${{ steps.run_diff.outputs.removed }} + - name: Create/Upload variable artifacts + id: variable_artifacts + run: | + mkdir -p diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + cd diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + echo '${{ steps.build_diff.outputs.added }}' > build_added + echo '${{ steps.build_diff.outputs.modified }}' > build_modified + echo '${{ steps.build_diff.outputs.removed }}' > build_removed_with_force + fi + + echo '${{ steps.run_diff.outputs.added }}' > run_added + echo '${{ steps.run_diff.outputs.modified }}' > run_modified + echo '${{ steps.run_diff.outputs.removed }}' > run_removed_with_force + + - name: Upload diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + uses: actions/upload-artifact@v4 + with: + name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + path: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + if-no-files-found: error + + - name: Download USN File(s) + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + uses: actions/download-artifact@v4 + with: + path: "patched-usns" + pattern: ${{ matrix.arch.name }}-*${{ env.PATCHED_USNS_FILENAME }} + merge-multiple: true + + - name: Get USNs + id: get-usns + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + run: | + usns=$(cat "patched-usns/${{ matrix.arch.name }}-${{ matrix.stacks.name }}-${{ env.PATCHED_USNS_FILENAME }}" | jq -c ) + echo "usns=$usns" >> "$GITHUB_OUTPUT" + + - name: Get Repository Name + id: repo_name + run: | + full=${{ github.repository }} + # Strip off the org and slash from repo name + # paketo-buildpacks/repo-name --> repo-name + repo=$(echo "${full}" | sed 's/^.*\///') + echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + + # Strip off 'stack' suffix from repo name + # some-name-stack --> some-name + registry_repo="${repo//-stack/}" + echo "registry_repo_name=${registry_repo}" >> "$GITHUB_OUTPUT" + + - name: Increment Tag + if: github.event.inputs.version == '' + id: semver + uses: paketo-buildpacks/github-config/actions/tag/increment-tag@main + with: + allow_head_tagged: true + + - name: Set Release Tag + id: tag + run: | + tag="${{ github.event.inputs.version }}" + if [ -z "${tag}" ]; then + tag="${{ steps.semver.outputs.tag }}" + fi + echo "tag=${tag}" >> "$GITHUB_OUTPUT" + + - name: Get registry build and run image names + id: registry_names + run: | + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + echo "build_image=${{ needs.preparation.outputs.repo_owner }}/${{ matrix.stacks.build_image }}-${{ needs.preparation.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" >> "$GITHUB_OUTPUT" + else + echo "build_image=" >> "$GITHUB_OUTPUT" + fi + + echo "run_image=${{ needs.preparation.outputs.repo_owner }}/${{ matrix.stacks.run_image }}-${{ needs.preparation.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" >> "$GITHUB_OUTPUT" + + - name: Create Release Notes + id: notes + uses: paketo-buildpacks/github-config/actions/stack/release-notes@main + with: + build_image: ${{ steps.registry_names.outputs.build_image }} + run_image: ${{ steps.registry_names.outputs.run_image }} + build_packages_added: ${{ steps.build_diff.outputs.added }} + build_packages_modified: ${{ steps.build_diff.outputs.modified }} + build_packages_removed_with_force: ${{ steps.build_diff.outputs.removed }} + run_packages_added: ${{ steps.run_diff.outputs.added }} + run_packages_modified: ${{ steps.run_diff.outputs.modified }} + run_packages_removed_with_force: ${{ steps.run_diff.outputs.removed }} + supports_usns: ${{ needs.preparation.outputs.support_usns }} + patched_usns: ${{ needs.get-usns.outputs.usns }} + + - name: Release Notes File + id: release-notes-file + run: | + printf '%s\n' '${{ steps.notes.outputs.release_body }}' > "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + + - name: Upload ${{ matrix.arch.name }} release notes file for stack ${{ matrix.stacks.name }} + uses: actions/upload-artifact@v4 + with: + name: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + path: "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + run_if_packages_removed_with_force: name: Run If Packages Removed With Force needs: [ diff ] @@ -296,21 +841,52 @@ jobs: packages_changed: name: Determine If Packages Changed - needs: [ diff ] + needs: [ diff, preparation ] runs-on: ubuntu-22.04 + if: ${{ !cancelled() && !failure() && needs.diff.result != 'skipped' }} + strategy: + matrix: + stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} + arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} outputs: packages_changed: ${{ steps.compare.outputs.packages_changed }} steps: + - name: Download diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + uses: actions/download-artifact@v4 + with: + name: diff-${{ matrix.arch.name }}-${{ matrix.stacks.name }} + + - name: Set env diff variables + run: | + + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + BUILD_ADDED=$(cat ./build_added) + BUILD_MODIFIED=$(cat ./build_modified) + echo "BUILD_ADDED=$BUILD_ADDED" >> $GITHUB_ENV + echo "BUILD_MODIFIED=$BUILD_MODIFIED" >> $GITHUB_ENV + else + echo "BUILD_ADDED=$BUILD_ADDED" >> $GITHUB_ENV + echo "BUILD_MODIFIED=$BUILD_MODIFIED" >> $GITHUB_ENV + fi + + RUN_ADDED=$(cat ./run_added) + RUN_MODIFIED=$(cat ./run_modified) + echo "RUN_ADDED=$RUN_ADDED" >> $GITHUB_ENV + echo "RUN_MODIFIED=$RUN_MODIFIED" >> $GITHUB_ENV + - name: Compare With Previous Release id: compare run: | # shellcheck disable=SC2153 - build_added=$(jq '. | length' <<< "${BUILD_ADDED}") - echo "Build packages added: ${build_added}" - # shellcheck disable=SC2153 - build_modified=$(jq '. | length' <<< "${BUILD_MODIFIED}") - echo "Build packages modified: ${build_modified}" + if [ "${{ matrix.stacks.create_build_image }}" == "true" ]; then + build_added=$(cat ./build_added | jq 'length') + echo "Build packages added: ${build_added}" + + # shellcheck disable=SC2153 + build_modified=$(cat ./build_modified | jq 'length') + echo "Build packages modified: ${build_modified}" + fi # shellcheck disable=SC2153 run_added=$(jq '. | length' <<< "${RUN_ADDED}") @@ -322,23 +898,18 @@ jobs: if [ "${build_added}" -eq 0 ] && [ "${build_modified}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then echo "No packages changed." - echo "packages_changed=false" >> "$GITHUB_OUTPUT" + # We ommit setting "packages_changed" variable to false, + # as there is an edge case scenario overriding any true value due to parallelization else echo "Packages changed." echo "packages_changed=true" >> "$GITHUB_OUTPUT" fi - env: - BUILD_ADDED: ${{ needs.diff.outputs.build_added }} - BUILD_MODIFIED: ${{ needs.diff.outputs.build_modified }} - RUN_ADDED: ${{ needs.diff.outputs.run_added }} - RUN_MODIFIED: ${{ needs.diff.outputs.run_modified }} - run_if_packages_changed: name: Run If Packages Changed runs-on: ubuntu-22.04 needs: [packages_changed] - if: ${{ needs.packages_changed.outputs.packages_changed == 'true' }} + if: ${{ needs.packages_changed.outputs.packages_changed == 'true' && !cancelled() }} steps: - name: Run if packages changed run: | @@ -346,35 +917,43 @@ jobs: test: name: Acceptance Test - needs: [ create_stack ] + needs: [ preparation, create_stack ] + if: ${{ !cancelled() && !failure() && needs.create_stack.result != 'skipped' }} runs-on: ubuntu-22.04 steps: - name: Setup Go - uses: actions/setup-go@v3 + uses: actions/setup-go@v5 with: go-version: stable - name: Checkout - uses: actions/checkout@v3 - - - name: Create OCI artifacts destination directory - run: | - mkdir -p build + uses: actions/checkout@v4 - - name: Download Build Image - uses: actions/download-artifact@v3 + - name: Download Build Image(s) + uses: actions/download-artifact@v4 with: - name: current-build-image - path: build + pattern: current-build-image-* - - name: Download Run Image - uses: actions/download-artifact@v3 + - name: Download Run Image(s) + uses: actions/download-artifact@v4 with: - name: current-run-image - path: build + pattern: current-run-image-* + + - name: Create OCI artifacts destination directory + run: | + echo '${{ needs.preparation.outputs.stacks }}' | jq -c '.[]' | while read -r stack; do + name=$(echo "$stack" | jq -r '.name') + output_dir=$(echo "$stack" | jq -r '.output_dir') + create_build_image=$(echo "$stack" | jq -r '.create_build_image') + mkdir -p $output_dir + mv "current-run-image-${name}/run.oci" "${output_dir}/run.oci" + if [ $create_build_image == 'true' ]; then + mv "current-build-image-${name}/build.oci" "${output_dir}/build.oci" + fi + done - name: Run Acceptance Tests - run: ./scripts/test.sh + run: ./scripts/test.sh --validate-stack-builds force_release_creation: name: Force Release Creation @@ -388,8 +967,10 @@ jobs: release: name: Release runs-on: ubuntu-22.04 - needs: [poll_usns, create_stack, diff, run_if_stack_files_changed, run_if_packages_changed, run_if_packages_removed_with_force, test, force_release_creation ] + needs: [create_stack, diff, run_if_stack_files_changed, run_if_packages_changed, run_if_packages_removed_with_force, test, force_release_creation, preparation ] if: ${{ always() && needs.diff.result == 'success' && needs.test.result == 'success' && (needs.run_if_packages_changed.result == 'success' || needs.run_if_stack_files_changed.result == 'success' || needs.force_release_creation.result == 'success' ) }} + outputs: + tag: ${{ steps.tag.outputs.tag }} steps: - name: Print Release Reasoning run: | @@ -401,29 +982,75 @@ jobs: printf "Force Release: %s\n" "${{ github.event.inputs.force }}" - name: Checkout With History - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: fetch-depth: 0 # gets full history - - name: Download current build image - uses: actions/download-artifact@v3 + - name: Download current build image(s) + uses: actions/download-artifact@v4 with: - name: current-build-image + path: image-files + pattern: current-build-image-* - - name: Download current run image - uses: actions/download-artifact@v3 + - name: Download current run image(s) + uses: actions/download-artifact@v4 with: - name: current-run-image + path: image-files + pattern: current-run-image-* - - name: Download Build Receipt - uses: actions/download-artifact@v3 + - name: Display Build and Run Images + run: ls image-files + + - name: Download Build Receipt(s) + uses: actions/download-artifact@v4 with: - name: current-build-receipt + path: receipt-files + pattern: current-build-receipt-* + merge-multiple: true - - name: Download Run Receipt - uses: actions/download-artifact@v3 + - name: Download Run Receipt(s) + uses: actions/download-artifact@v4 with: - name: current-run-receipt + path: receipt-files + pattern: current-run-receipt-* + merge-multiple: true + + - name: Display Receipts + run: ls receipt-files + + - name: Download Release Note File(s) + uses: actions/download-artifact@v4 + with: + path: release-notes + pattern: "*release-notes.md" + merge-multiple: true + + - name: Display Release Note Files + run: ls release-notes + + - name: Download hash code Files + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} + uses: actions/download-artifact@v4 + with: + path: hash-code-files + pattern: hash-code-* + merge-multiple: true + + - name: Display hash code Files + if: ${{ needs.preparation.outputs.polling_type == 'hash' }} + run: ls hash-code-files + + - name: Download USN Files + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + uses: actions/download-artifact@v4 + with: + path: usn-files + pattern: "*${{ env.PATCHED_USNS_FILENAME }}" + merge-multiple: true + + - name: Display USN Files + if: ${{ needs.preparation.outputs.polling_type == 'usn' }} + run: ls usn-files - name: Increment Tag if: github.event.inputs.version == '' @@ -441,91 +1068,270 @@ jobs: fi echo "tag=${tag}" >> "$GITHUB_OUTPUT" - - name: Write USNs File - id: write_usns - if: ${{ needs.poll_usns.outputs.usns != '[]' }} + - name: Setup Release Assets + id: assets run: | - jq . <<< "${USNS}" > "${USNS_PATH}" - echo "usns=${USNS_PATH}" >> "$GITHUB_OUTPUT" - env: - USNS_PATH: "${{ env.PATCHED_USNS_FILENAME }}" - USNS: ${{ needs.poll_usns.outputs.usns }} - - name: Get Repository Name - id: repo_name - run: | - full=${{ github.repository }} - # Strip off the org and slash from repo name - # paketo-buildpacks/repo-name --> repo-name - repo=$(echo "${full}" | sed 's/^.*\///') - echo "github_repo_name=${repo}" >> "$GITHUB_OUTPUT" + stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c '.[]') + archs=$(echo '${{ needs.preparation.outputs.architectures }}' | jq -c -r '.[]') + repo="${{ needs.preparation.outputs.github_repo_name }}" + tag="${{ steps.tag.outputs.tag }}" - # Strip off 'stack' suffix from repo name - # some-name-stack --> some-name - registry_repo="${repo//-stack/}" - echo "registry_repo_name=${registry_repo}" >> "$GITHUB_OUTPUT" + # Start with an empty array + assets=$(jq -n -c '[]') + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + run_image=$(echo "$stack" | jq -r '.run_image') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + run_receipt_filename=$(echo "$stack" | jq -r '.run_receipt_filename') - - name: Create Release Notes - id: notes - uses: paketo-buildpacks/github-config/actions/stack/release-notes@main - with: - build_image: "paketobuildpacks/build-${{ steps.repo_name.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" - run_image: "paketobuildpacks/run-${{ steps.repo_name.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" - build_packages_added: ${{ needs.diff.outputs.build_added }} - build_packages_modified: ${{ needs.diff.outputs.build_modified }} - build_packages_removed_with_force: ${{ needs.diff.outputs.build_removed_with_force }} - run_packages_added: ${{ needs.diff.outputs.run_added }} - run_packages_modified: ${{ needs.diff.outputs.run_modified }} - run_packages_removed_with_force: ${{ needs.diff.outputs.run_removed_with_force }} - patched_usns: ${{ needs.poll_usns.outputs.usns }} + assets="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg run_image "${run_image}" \ + --arg tag "${{ steps.tag.outputs.tag }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + '. += [ + { + "path": ($image_filepath + "/" + "current-run-image-" + $stack_name + "/run" + ".oci"), + "name": ($repo + "-" + $tag + "-" + $run_image + ".oci"), + "content_type": "application/gzip" + } + ]' <<<"${assets}")" - - name: Setup Release Assets - id: assets - run: | - assets="$(jq --null-input --compact-output \ - --arg tag "${{ steps.tag.outputs.tag }}" \ - --arg repo "${{ steps.repo_name.outputs.github_repo_name }}" \ - --arg build_receipt "${{ env.BUILD_RECEIPT_FILENAME }}" \ - --arg run_receipt "${{ env.RUN_RECEIPT_FILENAME }}" \ - '[ - { - "path": "build.oci", - "name": ($repo + "-" + $tag + "-" + "build.oci"), - "content_type": "application/gzip" - }, - { - "path": "run.oci", - "name": ($repo + "-" + $tag + "-" + "run.oci"), - "content_type": "application/gzip" - }, - { - "path": $build_receipt, - "name": ($repo + "-" + $tag + "-" + $build_receipt), - "content_type": "text/plain" - }, - { - "path": $run_receipt, - "name": ($repo + "-" + $tag + "-" + $run_receipt), - "content_type": "text/plain" - }]')" + if [[ $create_build_image == true ]]; then + build_image=$(echo "$stack" | jq -r '.build_image') + assets="$(jq -c \ + --arg image_filepath "image-files" \ + --arg stack_name "${stack_name}" \ + --arg build_image "${build_image}" \ + --arg tag "${{ steps.tag.outputs.tag }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + '. += [ + { + "path": ($image_filepath + "/" + "current-build-image-" + $stack_name + "/build" + ".oci"), + "name": ($repo + "-" + $tag + "-" + $build_image + ".oci"), + "content_type": "application/gzip" + } + ]' <<<"${assets}")" + fi + + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="-${arch_name}-" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="-" + fi + + if [ "${{ needs.preparation.outputs.polling_type }}" = "hash" ]; then + ## Add the Hash code files of the run images to the assets + run_image=$(echo "$stack" | jq -r '.run_image') + assets="$(jq -c \ + --arg hash_code_filepath "hash-code-files" \ + --arg stack_name "${stack_name}" \ + --arg run_image "${run_image}" \ + --arg tag "${{ steps.tag.outputs.tag }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($hash_code_filepath + "/" + "hash-code-current-run-image-"+ $arch + "-" + $stack_name ), + "name": ($repo + "-" + $tag + $arch_prefix + $run_image + ".oci.sha256"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + + ## Add the Hash code files of the build images to the assets + if [[ $create_build_image == true ]]; then + build_image=$(echo "$stack" | jq -r '.build_image') + assets="$(jq -c \ + --arg hash_code_filepath "hash-code-files" \ + --arg stack_name "${stack_name}" \ + --arg build_image "${build_image}" \ + --arg tag "${{ steps.tag.outputs.tag }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($hash_code_filepath + "/" + "hash-code-current-build-image-"+ $arch + "-" + $stack_name ), + "name": ($repo + "-" + $tag + $arch_prefix + $build_image + ".oci.sha256"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + fi + fi + done + done + + ## Adding the SBOM files to the assets + for stack in $stacks; do + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + + stack_name=$(echo "$stack" | jq -r '.name') + run_receipt_filename=$(echo "$stack" | jq -r '.run_receipt_filename') + + receipt_file_name=$( + echo '[ "'"$arch_name"'" ,"current-run-receipt" , "'"$stack_name"'" ]' | + jq -r 'map(select(. != "amd64")) | join("-")' + ) + + receipt_asset_name=$( + echo '[ "'"$repo"'", "'"$tag"'", "'"$arch_name"'", "'"$run_receipt_filename"'" ]' | + jq -r 'map(select(. != "amd64")) | map(select(. != "default")) | join("-")' + ) + + assets="$(jq -c \ + --arg receipts_filepath "receipt-files" \ + --arg receipt_file_name "${receipt_file_name}" \ + --arg receipt_asset_name "${receipt_asset_name}" \ + '. += [ + { + "path": ($receipts_filepath + "/" + $receipt_file_name), + "name": $receipt_asset_name, + "content_type": "text/plain" + } + ]' <<<"${assets}")" + + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + + if [[ $create_build_image == true ]]; then + build_receipt_filename=$(echo "$stack" | jq -r '.build_receipt_filename') - if [ -n "${{ steps.write_usns.outputs.usns }}" ]; then - assets="$(jq --compact-output \ + receipt_file_name=$( + echo '[ "'"$arch_name"'" ,"current-build-receipt" , "'"$stack_name"'" ]' | + jq -r 'map(select(. != "amd64")) | join("-")' + ) + + receipt_asset_name=$( + echo '[ "'"$repo"'", "'"$tag"'", "'"$arch_name"'", "'"$build_receipt_filename"'" ]' | + jq -r 'map(select(. != "amd64")) | map(select(. != "default")) | join("-")' + ) + + assets="$(jq -c \ + --arg receipts_filepath "receipt-files" \ + --arg receipt_file_name "${receipt_file_name}" \ + --arg receipt_asset_name "${receipt_asset_name}" \ + --arg tag "${{ steps.tag.outputs.tag }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + --arg arch_prefix "${arch_prefix}" \ + '. += [ + { + "path": ($receipts_filepath + "/" + $receipt_file_name), + "name": $receipt_asset_name, + "content_type": "text/plain" + } + ]' <<<"${assets}")" + fi + done + done + + ## Add the usn files to the assets + if [ "${{ needs.preparation.outputs.polling_type }}" = "usn" ]; then + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + stack_name_prefix="${stack}" + if [[ $stack_name == "default" ]]; then + stack_name_prefix="" + fi + + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="${arch_name}" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="" + fi + + usn_asset_name=$( + echo '[ + "${{ needs.preparation.outputs.github_repo_name }}", + "${{ steps.tag.outputs.tag }}", + "'"$stack_name_prefix"'", + "'"$arch_prefix"'", + "${{ env.PATCHED_USNS_FILENAME }}" + ]' | jq -r 'map(select(length > 0)) | join("-")' + ) + + assets="$(jq -c \ + --arg usns_filepath "usn-files" \ + --arg stack_name "${stack_name}" \ + --arg arch "${arch_name}" \ + --arg usn_asset_name "${usn_asset_name}" \ + --arg patched_usns_suffix "${{ env.PATCHED_USNS_FILENAME }}" \ + '. += [ + { + "path": ($usns_filepath + "/" + $arch + "-" + $stack_name + "-" + $patched_usns_suffix), + "name": $usn_asset_name, + "content_type": "text/plain" + } + ]' <<< "${assets}")" + done + done + fi + + # Merge relase notes per architecture and add them to the assets + release_notes_dir="release-notes" + for arch in $archs; do + arch_name=$(echo "$arch" | jq -r '.name') + arch_prefix="-${arch_name}-" + if [[ $arch_name == "amd64" ]]; then + arch_prefix="-" + fi + + # Merge the release notes per architecture + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + filename="${arch_name}-${stack_name}-release-notes.md" + cat "${release_notes_dir}/${filename}" >>"${release_notes_dir}/${arch_name}-release-notes" + done + + # add release notes of the arch on the assets + assets="$(jq -c \ + --arg release_notes_dir "${release_notes_dir}" \ --arg tag "${{ steps.tag.outputs.tag }}" \ - --arg repo "${{ steps.repo_name.outputs.github_repo_name }}" \ - --arg usn_path "${{ steps.write_usns.outputs.usns }}" \ - --arg usn_name "${{ env.PATCHED_USNS_FILENAME }}" \ + --arg repo "${{ needs.preparation.outputs.github_repo_name }}" \ + --arg arch "${arch_name}" \ + --arg arch_prefix "${arch_prefix}" \ '. += [ - { - "path": $usn_path, - "name": ($repo + "-" + $tag + "-" + $usn_name), - "content_type": "text/plain" - } - ]' <<< "${assets}")" + { + "path": ($release_notes_dir + "/" + $arch + "-" + "release-notes"), + "name": ($repo + "-" + $tag + $arch_prefix + "release-notes.md"), + "content_type": "text/plain" + } + ]' <<<"${assets}")" + done + + # If there is only one architecture + # add this as a relase notes description + archs_length=$(echo '${{ needs.preparation.outputs.architectures }}' | jq 'length') + if [ $archs_length -eq 1 ]; then + arch_name=$(echo '${{ needs.preparation.outputs.architectures }}' | jq -r '.[0].name') + cat "${release_notes_dir}/${arch_name}-release-notes" > release_notes.md fi - printf "assets=%s\n" "${assets}" >> "$GITHUB_OUTPUT" + echo "assets=${assets}" >> "$GITHUB_OUTPUT" + echo "archs_length=${archs_length}" >> "$GITHUB_OUTPUT" + + - name: Generate Release Notes Description + if: ${{ steps.assets.outputs.archs_length > 1 }} + run: | + echo "## Images" > release_notes.md + echo "" >> release_notes.md + stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c -r '.[]') + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + run_image=$(echo "$stack" | jq -r '.run_image') + build_image=$(echo "$stack" | jq -r '.build_image') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + + if [ ${create_build_image} == true ]; then + echo "Build: \`${{ needs.preparation.outputs.repo_owner }}/${build_image}-${stack_name}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md + fi + echo "Run: \`${{ needs.preparation.outputs.repo_owner }}/${run_image}-${stack_name}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md + done - name: Create Release uses: paketo-buildpacks/github-config/actions/release/create@main @@ -535,15 +1341,15 @@ jobs: tag_name: v${{ steps.tag.outputs.tag }} target_commitish: ${{ github.sha }} name: v${{ steps.tag.outputs.tag }} - body: ${{ steps.notes.outputs.release_body }} + body_filepath: release_notes.md draft: false assets: ${{ steps.assets.outputs.assets }} failure: name: Alert on Failure runs-on: ubuntu-22.04 - needs: [poll_usns, create_stack, diff, test, release, packages_changed, stack_files_changed] - if: ${{ always() && needs.poll_usns.result == 'failure' || needs.create_stack.result == 'failure' || needs.diff.result == 'failure' || needs.test.result == 'failure' || needs.release.result == 'failure' || needs.packages_changed.result == 'failure' || needs.stack_files_changed.result == 'failure' }} + needs: [ preparation, poll_usns, poll_images, create_stack, stack_files_changed, diff, test, packages_changed, release ] + if: ${{ always() && needs.preparation.result == 'failure' || needs.poll_images.result == 'failure' || needs.poll_usns.result == 'failure' || needs.create_stack.result == 'failure' || needs.diff.result == 'failure' || needs.test.result == 'failure' || needs.release.result == 'failure' || needs.packages_changed.result == 'failure' || needs.stack_files_changed.result == 'failure' }} steps: - name: File Failure Alert Issue uses: paketo-buildpacks/github-config/actions/issue/file@main @@ -555,6 +1361,6 @@ jobs: issue_title: "Failure: Create Release workflow" issue_body: | Create Release workflow [failed](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). - Please take a look to ensure CVE patches can be released. (cc @paketo-buildpacks/stacks-maintainers). + Unable to update images. comment_body: | Another failure occurred: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} From 0e47c19bbea5bb7a968874c4698a773759506127 Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Tue, 12 Nov 2024 12:58:05 +0200 Subject: [PATCH 2/4] removing arm64 --- stack/stack.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/stack.toml b/stack/stack.toml index 01297a2..1fc0464 100644 --- a/stack/stack.toml +++ b/stack/stack.toml @@ -2,7 +2,7 @@ id = "io.buildpacks.stacks.noble" homepage = "https://github.com/paketo-buildpacks/noble-base-stack" maintainer = "Paketo Buildpacks" -platforms = ["linux/amd64", "linux/arm64"] +platforms = ["linux/amd64"] [build] description = "ubuntu:noble with compilers and shell utilities" From f77305908b8e9b3b29adffa316e63ed276c02ead Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Tue, 12 Nov 2024 16:50:23 +0200 Subject: [PATCH 3/4] adding arm64 arch --- stack/stack.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stack/stack.toml b/stack/stack.toml index 1fc0464..01297a2 100644 --- a/stack/stack.toml +++ b/stack/stack.toml @@ -2,7 +2,7 @@ id = "io.buildpacks.stacks.noble" homepage = "https://github.com/paketo-buildpacks/noble-base-stack" maintainer = "Paketo Buildpacks" -platforms = ["linux/amd64"] +platforms = ["linux/amd64", "linux/arm64"] [build] description = "ubuntu:noble with compilers and shell utilities" From 0e6ec90b780f51584bc55f10e9c82fcb13c23a84 Mon Sep 17 00:00:00 2001 From: Costas Papastathis Date: Tue, 10 Dec 2024 15:28:16 +0200 Subject: [PATCH 4/4] passing readme file with filepath --- .github/workflows/create-release.yml | 62 ++++++++++++++++------------ 1 file changed, 35 insertions(+), 27 deletions(-) diff --git a/.github/workflows/create-release.yml b/.github/workflows/create-release.yml index 44be93f..f931686 100644 --- a/.github/workflows/create-release.yml +++ b/.github/workflows/create-release.yml @@ -599,6 +599,11 @@ jobs: stacks: ${{ fromJSON(needs.preparation.outputs.stacks) }} arch: ${{ fromJSON(needs.preparation.outputs.architectures) }} steps: + - name: Checkout With History + uses: actions/checkout@v4 + with: + fetch-depth: 0 # gets full history + - name: Download Current Receipt(s) uses: actions/download-artifact@v4 with: @@ -803,8 +808,8 @@ jobs: echo "run_image=${{ needs.preparation.outputs.repo_owner }}/${{ matrix.stacks.run_image }}-${{ needs.preparation.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}" >> "$GITHUB_OUTPUT" - - name: Create Release Notes - id: notes + - name: Create Release Notes per Arch and per Stack + id: notes_arc_stack uses: paketo-buildpacks/github-config/actions/stack/release-notes@main with: build_image: ${{ steps.registry_names.outputs.build_image }} @@ -821,7 +826,7 @@ jobs: - name: Release Notes File id: release-notes-file run: | - printf '%s\n' '${{ steps.notes.outputs.release_body }}' > "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" + printf '%s\n' '${{ steps.notes_arc_stack.outputs.release_body }}' > "${{ matrix.arch.name }}-${{ matrix.stacks.name }}-release-notes.md" - name: Upload ${{ matrix.arch.name }} release notes file for stack ${{ matrix.stacks.name }} uses: actions/upload-artifact@v4 @@ -896,7 +901,7 @@ jobs: run_modified=$(jq '. | length' <<< "${RUN_MODIFIED}") echo "Run packages modified: ${run_modified}" - if [ "${build_added}" -eq 0 ] && [ "${build_modified}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then + if [ "${build_added:-0}" -eq 0 ] && [ "${build_modified:-0}" -eq 0 ] && [ "${run_added}" -eq 0 ] && [ "${run_modified}" -eq 0 ]; then echo "No packages changed." # We ommit setting "packages_changed" variable to false, # as there is an edge case scenario overriding any true value due to parallelization @@ -1304,34 +1309,37 @@ jobs: ]' <<<"${assets}")" done + echo "assets=${assets}" >> "$GITHUB_OUTPUT" + + - name: Generate Release Notes Description + id: notes + run: | + + release_notes_dir="release-notes" + # If there is only one architecture - # add this as a relase notes description archs_length=$(echo '${{ needs.preparation.outputs.architectures }}' | jq 'length') if [ $archs_length -eq 1 ]; then arch_name=$(echo '${{ needs.preparation.outputs.architectures }}' | jq -r '.[0].name') cat "${release_notes_dir}/${arch_name}-release-notes" > release_notes.md - fi - - echo "assets=${assets}" >> "$GITHUB_OUTPUT" - echo "archs_length=${archs_length}" >> "$GITHUB_OUTPUT" + else + echo "## Images" > release_notes.md + echo "" >> release_notes.md + stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c -r '.[]') + for stack in $stacks; do + stack_name=$(echo "$stack" | jq -r '.name') + run_image=$(echo "$stack" | jq -r '.run_image') + build_image=$(echo "$stack" | jq -r '.build_image') + create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') - - name: Generate Release Notes Description - if: ${{ steps.assets.outputs.archs_length > 1 }} - run: | - echo "## Images" > release_notes.md - echo "" >> release_notes.md - stacks=$(echo '${{ needs.preparation.outputs.stacks }}' | jq -c -r '.[]') - for stack in $stacks; do - stack_name=$(echo "$stack" | jq -r '.name') - run_image=$(echo "$stack" | jq -r '.run_image') - build_image=$(echo "$stack" | jq -r '.build_image') - create_build_image=$(echo "$stack" | jq -r '.create_build_image // false') + if [ ${create_build_image} == true ]; then + echo "Build: \`${{ needs.preparation.outputs.repo_owner }}/${build_image}-${{ needs.preparation.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md + fi + echo "Run: \`${{ needs.preparation.outputs.repo_owner }}/${run_image}-${{ needs.preparation.outputs.registry_repo_name }}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md + done + fi - if [ ${create_build_image} == true ]; then - echo "Build: \`${{ needs.preparation.outputs.repo_owner }}/${build_image}-${stack_name}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md - fi - echo "Run: \`${{ needs.preparation.outputs.repo_owner }}/${run_image}-${stack_name}:${{ steps.tag.outputs.tag }}\`" >> release_notes.md - done + echo "release_body=release_notes.md" >> "$GITHUB_OUTPUT" - name: Create Release uses: paketo-buildpacks/github-config/actions/release/create@main @@ -1341,14 +1349,14 @@ jobs: tag_name: v${{ steps.tag.outputs.tag }} target_commitish: ${{ github.sha }} name: v${{ steps.tag.outputs.tag }} - body_filepath: release_notes.md + body_filepath: ${{ steps.notes.outputs.release_body }} draft: false assets: ${{ steps.assets.outputs.assets }} failure: name: Alert on Failure runs-on: ubuntu-22.04 - needs: [ preparation, poll_usns, poll_images, create_stack, stack_files_changed, diff, test, packages_changed, release ] + needs: [ preparation, poll_usns, poll_images, create_stack, diff, test, release, packages_changed, stack_files_changed ] if: ${{ always() && needs.preparation.result == 'failure' || needs.poll_images.result == 'failure' || needs.poll_usns.result == 'failure' || needs.create_stack.result == 'failure' || needs.diff.result == 'failure' || needs.test.result == 'failure' || needs.release.result == 'failure' || needs.packages_changed.result == 'failure' || needs.stack_files_changed.result == 'failure' }} steps: - name: File Failure Alert Issue