From 804bf47484cc7fe5850223f26c5359f5cde3ed36 Mon Sep 17 00:00:00 2001 From: Topher Lubaway Date: Fri, 14 Oct 2022 12:58:27 -0500 Subject: [PATCH] Major overhaul of check images (#17979) * Major overhaul of check images actually fail on missed images detect rate limit for dockerhub sleep when hit rate limit and registry more clear user messaging Reduce output from 5000 lines to ~ 1000 More clear help messages for failures * Addresses GH feedback minor bugfixes --- tools/bin/check_images_exist.sh | 115 ++++++++++++++++++++++++-------- 1 file changed, 87 insertions(+), 28 deletions(-) diff --git a/tools/bin/check_images_exist.sh b/tools/bin/check_images_exist.sh index efbab4272f04..839af118a377 100755 --- a/tools/bin/check_images_exist.sh +++ b/tools/bin/check_images_exist.sh @@ -1,63 +1,122 @@ #!/usr/bin/env bash -set -e +# ------------- Import some defaults for the shell + +# Source shell defaults +# $0 is the currently running program (this file) +this_file_directory=$(dirname $0) +relative_path_to_defaults=$this_file_directory/../shell_defaults + +# if a file exists there, source it. otherwise complain +if test -f $relative_path_to_defaults; then + # source and '.' are the same program + source $relative_path_to_defaults +else + echo -e "\033[31m\nFAILED TO SOURCE TEST RUNNING OPTIONS.\033[39m" + echo -e "\033[31mTried $relative_path_to_defaults\033[39m" + exit 1 +fi + +set +o xtrace # +x easier human reading here . tools/lib/lib.sh + function docker_tag_exists() { - # Added check for images pushed to github container registry - if [[ $1 == ghcr* ]] + # Is true for images stored in the Github Container Registry + repo=$1 + tag=$2 + # we user [[ here because test doesn't support globbing well + if [[ $repo == ghcr* ]] then TOKEN_URL=https://ghcr.io/token\?scope\="repository:$1:pull" - token=$(curl $TOKEN_URL | jq -r '.token') + token=$(curl $TOKEN_URL | jq -r '.token' > /dev/null) URL=https://ghcr.io/v2/$1/manifests/$2 - printf "\tURL: %s\n" "$URL" - curl --silent -H "Authorization: Bearer $token" -f -lSL "$URL" > /dev/null + echo -e "$blue_text""\n\n\tURL: $URL""$default_text" + curl -H "Authorization: Bearer $token" --location --silent --show-error --dump-header header.txt "$URL" > /dev/null + curl_success=$? else URL=https://hub.docker.com/v2/repositories/"$1"/tags/"$2" - printf "\tURL: %s\n" "$URL" - curl --silent -f -lSL "$URL" > /dev/null + echo -e "$blue_text""\n\n\tURL: $URL""$default_text" + curl --silent --show-error --location --dump-header header.txt "$URL" > /dev/null + curl_success=$? + # some bullshit to get the number out of a header that looks like this + # < content-length: 1039 + # < x-ratelimit-limit: 180 + # < x-ratelimit-reset: 1665683196 + # < x-ratelimit-remaining: 180 + docker_rate_limit_remaining=$(grep 'x-ratelimit-remaining: ' header.txt | grep --only-matching --extended-regexp "\d+") + # too noisy when set to < 1. Dockerhub starts complaining somewhere around 10 + if test "$docker_rate_limit_remaining" -lt 20; then + echo -e "$red_text""We are close to a sensitive dockerhub rate limit!""$default_text" + echo -e "$red_text""SLEEPING 60s sad times""$default_text" + sleep 60 + docker_tag_exists $1 $2 + elif test $docker_rate_limit_remaining -lt 50; then + echo -e "$red_text""Rate limit reported as $docker_rate_limit_remaining""$default_text" + fi + fi + if test $curl_success -ne 0; then + echo -e "$red_text""Curl Said this didn't work. Please investigate""$default_text" + exit 1 fi } checkPlatformImages() { - echo "Checking platform images exist..." - docker-compose pull || exit 1 - echo "Success! All platform images exist!" + echo -e "$blue_text""Checking platform images exist...""$default_text" + #Pull without printing progress information and send error stream because that where image names are + docker-compose pull --quiet 2> compose_output + docker_compose_success=$? + # quiet output is just SHAs ie: f8a3d002a8a6 + images_pulled_count=$(docker images --quiet | wc -l) + if test $images_pulled_count -eq 0; then + echo -e "$red_text""Nothing was pulled! This script may be broken! We expect to pull something""$default_text" + exit 1 + elif test $docker_compose_success -eq 0; then + echo -e "$blue_text""Docker successfully pulled all images""$default_text" + else + echo -e "$red_text""docker-compose failed to pull all images""$default_text" + cat compose_output + exit 1 + fi } checkNormalizationImages() { + echo -e "$blue_text""Checking Normalization images exist...""$default_text" # the only way to know what version of normalization the platform is using is looking in NormalizationRunnerFactory. local image_version; factory_path=airbyte-commons-worker/src/main/java/io/airbyte/workers/normalization/NormalizationRunnerFactory.java - # This check should fail if the file doesn't exist, so just try to ls the file - ls $factory_path > /dev/null + # -f True if file exists and is a regular file + if ! test -f $factory_path; then + echo -e "$red_text""No NormalizationRunnerFactory found at path! H4LP!!!""$default_text" + fi image_version=$(cat $factory_path | grep 'NORMALIZATION_VERSION =' | cut -d"=" -f2 | sed 's:;::' | sed -e 's:"::g' | sed -e 's:[[:space:]]::g') - echo "Checking normalization images with version $image_version exist..." - VERSION=$image_version docker-compose -f airbyte-integrations/bases/base-normalization/docker-compose.yaml pull || exit 1 - echo "Success! All normalization images exist!" + echo -e "$blue_text""Checking normalization images with version $image_version exist...""$default_text" + VERSION=$image_version docker-compose --file airbyte-integrations/bases/base-normalization/docker-compose.yaml pull --quiet + docker_compose_success=$? + if test $docker_compose_success -eq 0; then + echo -e "$blue_text""Docker successfully pulled all images for normalization""$default_text" + else + echo -e "$red_text""docker-compose failed to pull all images for normalization""$default_text" + exit 1 + fi } checkConnectorImages() { - echo "Checking connector images exist..." - + echo -e "$blue_text""Checking connector images exist...""$default_text" CONNECTOR_DEFINITIONS=$(grep "dockerRepository" -h -A1 airbyte-config/init/src/main/resources/seed/*.yaml | grep -v -- "^--$" | tr -d ' ') [ -z "CONNECTOR_DEFINITIONS" ] && echo "ERROR: Could not find any connector definition." && exit 1 while IFS=":" read -r _ REPO; do IFS=":" read -r _ TAG - printf "${REPO}: ${TAG}\n" + printf "\t${REPO}: ${TAG}\n" if docker_tag_exists "$REPO" "$TAG"; then printf "\tSTATUS: found\n" else printf "\tERROR: not found!\n" && exit 1 fi - # Docker hub has a rate limit of 180 requests per minute, so slow down our rate of calling the API - # https://docs.docker.com/docker-hub/api/latest/#tag/rate-limiting - sleep 1 done <<< "${CONNECTOR_DEFINITIONS}" - - echo "Success! All connector images exist!" + echo -e "$blue_text""Success! All connector images exist!""$default_text" } main() { @@ -65,14 +124,14 @@ main() { SUBSET=${1:-all} # default to all. [[ ! "$SUBSET" =~ ^(all|platform|connectors)$ ]] && echo "Usage ./tools/bin/check_image_exists.sh [all|platform|connectors]" && exit 1 - - echo "checking images for: $SUBSET" + echo -e "$blue_text""checking images for: $SUBSET""$default_text" [[ "$SUBSET" =~ ^(all|platform)$ ]] && checkPlatformImages [[ "$SUBSET" =~ ^(all|platform|connectors)$ ]] && checkNormalizationImages [[ "$SUBSET" =~ ^(all|connectors)$ ]] && checkConnectorImages - - echo "Image check complete." + echo -e "$blue_text""Image check complete.""$default_text" + test -f header.txt && rm header.txt + test -f compose_output && rm compose_output } main "$@"