Scheduled #13650
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Scheduled | |
| on: | |
| schedule: | |
| - cron: '45 * * * *' | |
| workflow_dispatch: | |
| concurrency: | |
| group: "scheduled" | |
| cancel-in-progress: true | |
| jobs: | |
| check: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| configs: ${{ steps.repo_check.outputs.configs }} | |
| steps: | |
| - uses: actions/checkout@v4 | |
| - uses: mikefarah/yq@v4.35.1 | |
| - name: Login to Docker Hub | |
| uses: docker/login-action@v3 | |
| with: | |
| username: ${{ vars.DOCKER_USERNAME }} | |
| password: ${{ secrets.DOCKER_PASSWORD }} | |
| - id: repo_check | |
| env: | |
| CONCURRENT_IMAGE_PULL: ${{ vars.CONCURRENT_IMAGE_PULL }} | |
| run: | | |
| # This script reads config.yaml, platforms.yaml and runners.yaml to generate a list of configurations to build and deploy | |
| # It will; | |
| # - check the source respository for the latest commit hash | |
| # - check if the built image exists in our dockerhub registry | |
| # - generate a list of configurations to build and deploy | |
| # | |
| # Here is an example of generate output; | |
| # [{ | |
| # "source": { | |
| # "repository": "hyperledger/besu", | |
| # "ref": "main" | |
| # }, | |
| # "build_script": "./besu/build.sh", | |
| # "target": { | |
| # "tag": "main", | |
| # "repository": "ethpandaops/besu" | |
| # }, | |
| # "platforms": [ | |
| # { | |
| # "platform": "linux/amd64", | |
| # "runner": "ubuntu-latest", | |
| # "slug": "linux-amd64" | |
| # }, | |
| # { | |
| # "platform": "linux/arm64", | |
| # "runner": "ARM64", | |
| # "slug": "linux-arm64" | |
| # } | |
| # ] | |
| # },{ | |
| # "source": { | |
| # "repository": "sigp/lighthouse", | |
| # "ref": "unstable" | |
| # }, | |
| # "target": { | |
| # "tag": "unstable", | |
| # "repository": "ethpandaops/lighthouse", | |
| # "dockerfile": "./lighthouse/Dockerfile" | |
| # }, | |
| # "platforms": [ | |
| # { | |
| # "platform": "linux/amd64", | |
| # "runner": "ubuntu-latest", | |
| # "slug": "linux-amd64" | |
| # }, | |
| # { | |
| # "platform": "linux/arm64", | |
| # "runner": "ARM64", | |
| # "slug": "linux-arm64" | |
| # } | |
| # ] | |
| # }] | |
| CONFIG_FILE="config.yaml" | |
| PLATFORMS_FILE="platforms.yaml" | |
| RUNNERS_FILE="runners.yaml" | |
| # Create a temporary directory for storing intermediate results | |
| TEMP_DIR=$(mktemp -d) | |
| # Ensure the temporary directory is removed when the script exits | |
| trap "rm -rf $TEMP_DIR" EXIT | |
| process_commits() { | |
| local LINE=$1 | |
| local SOURCE_REPOSITORY=$2 | |
| local SOURCE_REF=$3 | |
| local TARGET_REPOSITORY=$4 | |
| local TARGET_TAG=$5 | |
| local CLIENT="${TARGET_REPOSITORY#*/}" | |
| local RESPONSE=$(curl -s -H "Accept: application/vnd.github+json" \ | |
| -H "Authorization: Bearer ${{ secrets.GITHUB_TOKEN }}" \ | |
| "https://api.github.com/repos/${SOURCE_REPOSITORY}/commits/${SOURCE_REF}?per_page=1") | |
| local COMMIT_HASH=$(echo "$RESPONSE" | jq -r '.sha' | cut -c1-7) | |
| if [[ -z "$COMMIT_HASH" || "$COMMIT_HASH" == "null" ]]; then | |
| # Log error but don't exit; just skip this configuration | |
| echo "[LINE:$LINE] Error fetching commit hash for ${SOURCE_REPOSITORY}#${SOURCE_REF}, skipping." | |
| return | |
| fi | |
| local configOutput="${TEMP_DIR}/${LINE}_commits.json" | |
| touch $configOutput | |
| echo "{\"line\": \"$LINE\", \"commit_hash\": \"$COMMIT_HASH\"}," >> $configOutput | |
| } | |
| process_image() { | |
| local LINE=$1 | |
| local IMAGE=$2 | |
| local URL=$3 | |
| local imageOutput="${TEMP_DIR}/${LINE}_image.json" | |
| touch $imageOutput | |
| local exists=$(curl -s $URL | jq '.results | length > 0') | |
| # check if exists == true | |
| if [ "$exists" == "true" ]; then | |
| exists=true | |
| else | |
| exists=false | |
| fi | |
| echo "{\"line\": \"$LINE\", \"image\": \"$IMAGE\", \"exists\": $exists}" >> $imageOutput | |
| } | |
| # Get commit hashes for each configuration in parallel | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do | |
| process_commits "$LINE" "$SOURCE_REPOSITORY" "$SOURCE_REF" "$TARGET_REPOSITORY" "$TARGET_TAG" & | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| wait | |
| # Initialize JSON arrays | |
| COMMITS="[" | |
| # Concatenate results, ensuring files exist before attempting to read | |
| for file in $TEMP_DIR/*_commits.json; do | |
| if [ -f "$file" ]; then | |
| COMMITS+=$(cat "$file") | |
| fi | |
| done | |
| # Remove trailing commas and close JSON arrays | |
| COMMITS="${COMMITS%,}]" | |
| echo "Checking if images exist in dockerhub..." | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do | |
| # get the image commit hash from LINE | |
| COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash') | |
| IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}" | |
| IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}" | |
| URL="https://hub.docker.com/v2/repositories/${TARGET_REPOSITORY}/tags?page_size=25&page=1&ordering=&name=${IMAGE_TAG}" | |
| process_image $LINE $IMAGE $URL & | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| wait | |
| declare -A images | |
| # Concatenate results, ensuring files exist before attempting to read | |
| for file in $TEMP_DIR/*_image.json; do | |
| if [ -f "$file" ]; then | |
| LINE=$(cat "$file" | jq -r '.line') | |
| IMAGE=$(cat "$file" | jq -r '.image') | |
| EXISTS=$(cat "$file" | jq -r '.exists') | |
| images[$IMAGE]=$EXISTS | |
| fi | |
| done | |
| CONFIGS="configs=[" | |
| echo "Generating configuration files..." | |
| while IFS=$'\t' read -r LINE SOURCE_REPOSITORY SOURCE_REF TARGET_REPOSITORY TARGET_TAG; do | |
| # get the image commit hash from LINE | |
| COMMIT_HASH=$(echo "$COMMITS" | jq -r --arg LINE "$LINE" '.[] | select(.line == $LINE) | .commit_hash') | |
| IMAGE_TAG="${TARGET_TAG}-${COMMIT_HASH}" | |
| IMAGE="${TARGET_REPOSITORY}:${IMAGE_TAG}" | |
| CLIENT="${TARGET_REPOSITORY#*/}" | |
| if [ "${images[$IMAGE]}" == "false" ]; then | |
| # Handle platforms and runners, ensuring output files are created even if empty | |
| platforms=$(yq e ".$CLIENT[]" "$PLATFORMS_FILE") | |
| platformsArr="" | |
| for platform in $platforms; do | |
| runner=$(yq e ".\"$platform\"" "$RUNNERS_FILE") | |
| slug=$(echo "$platform" | tr '/' '-') | |
| platformsArr+="{\\\"platform\\\": \\\"$platform\\\", \\\"runner\\\": \\\"$runner\\\", \\\"slug\\\": \\\"$slug\\\"}," | |
| done | |
| platformsArr="${platformsArr%,}" | |
| # convert to string | |
| platformsOutput="{\"platforms\": \"[$platformsArr]\"}" | |
| CONFIGS+=$(echo "$(yq -r -o=json ".[${LINE}]" "$CONFIG_FILE" | jq --argjson plat "$platformsOutput" '. + $plat'),") | |
| fi | |
| done < <(yq -r 'to_entries | map_values({"value":.value, "index":.key}) | .[] | [.index, .value.source.repository, .value.source.ref, .value.target.repository, .value.target.tag] | @tsv' "$CONFIG_FILE") | |
| # Remove trailing commas and close JSON arrays | |
| CONFIGS="${CONFIGS%,}]" | |
| echo "CONFIGS: $CONFIGS" | |
| echo $CONFIGS >> $GITHUB_OUTPUT | |
| deploy: | |
| needs: check | |
| if: ${{ needs.check.outputs.configs != '[]' && needs.check.outputs.configs != '' }} | |
| uses: ./.github/workflows/deploy.yml | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| config: ${{fromJson(needs.check.outputs.configs)}} | |
| name: ${{ matrix.config.source.repository }}#${{ matrix.config.source.ref }} ${{ matrix.config.target.tag }} | |
| with: | |
| source_repository: ${{ matrix.config.source.repository }} | |
| source_ref: ${{ matrix.config.source.ref }} | |
| build_script: ${{ matrix.config.build_script }} | |
| build_args: "${{ matrix.config.build_args }}" | |
| target_tag: ${{ matrix.config.target.tag }} | |
| target_repository: ${{ matrix.config.target.repository }} | |
| target_dockerfile: ${{ matrix.config.target.dockerfile }} | |
| platforms: ${{ matrix.config.platforms }} | |
| DOCKER_USERNAME: "${{ vars.DOCKER_USERNAME }}" | |
| GOPROXY: "${{ vars.GOPROXY }}" | |
| secrets: | |
| DOCKER_PASSWORD: "${{ secrets.DOCKER_PASSWORD }}" | |
| MACOS_PASSWORD: "${{ secrets.MACOS_PASSWORD }}" | |
| notify: | |
| name: Discord Notification | |
| runs-on: ubuntu-latest | |
| needs: | |
| - check | |
| - deploy | |
| if: cancelled() || failure() | |
| steps: | |
| - name: Notify | |
| uses: nobrayner/discord-webhook@v1 | |
| with: | |
| github-token: ${{ secrets.github_token }} | |
| discord-webhook: ${{ secrets.DISCORD_WEBHOOK }} |