diff --git a/.cicd/generate-base-images.sh b/.cicd/generate-base-images.sh index 94abce495c..7a141640a9 100755 --- a/.cicd/generate-base-images.sh +++ b/.cicd/generate-base-images.sh @@ -9,7 +9,7 @@ TAG=$(echo $FULL_TAG | cut -d: -f2) EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG") # build, if neccessary if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' ]]; then # if we cannot pull the image, we build and push it first - docker build -t $FULL_TAG -f $CICD_DIR/docker/${IMAGE_TAG}.dockerfile . + docker build -t $FULL_TAG -f $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile . docker push $FULL_TAG else echo "$FULL_TAG already exists." diff --git a/.cicd/helpers/docker-hash.sh b/.cicd/helpers/docker-hash.sh index 6dcd24b3c2..de838e8e65 100644 --- a/.cicd/helpers/docker-hash.sh +++ b/.cicd/helpers/docker-hash.sh @@ -16,7 +16,7 @@ function determine-hash() { } if [[ ! -z $IMAGE_TAG ]]; then - determine-hash "$CICD_DIR/docker/${IMAGE_TAG}.dockerfile" + determine-hash "$CICD_DIR/platforms/${IMAGE_TAG}.dockerfile" export FULL_TAG="eosio/ci:eosio-cdt-$HASHED_IMAGE_TAG" else echo "Please set ENV::IMAGE_TAG to match the name of a platform dockerfile..." diff --git a/.cicd/pipeline-upload.sh b/.cicd/pipeline-upload.sh new file mode 100755 index 0000000000..202cefcec1 --- /dev/null +++ b/.cicd/pipeline-upload.sh @@ -0,0 +1,9 @@ +#!/bin/bash +set -eou pipefail + +export MACOS_10_14_TAG="eosio-cdt-macos-10.14-$(sha1sum ./.cicd/platforms/macos-10.14.sh | awk '{print $1}')" +export MACOS_10_15_TAG="eosio-cdt-macos-10.15-$(sha1sum ./.cicd/platforms/macos-10.15.sh | awk '{print $1}')" +export VARS='$MACOS_10_14_TAG:$MACOS_10_15_TAG' +envsubst "$VARS" < "./.cicd/pipeline.yml" > "./.cicd/pipeline.yml.out" +buildkite-agent artifact upload ./.cicd/pipeline.yml.out +buildkite-agent pipeline upload ./.cicd/pipeline.yml.out diff --git a/.cicd/pipeline.yml b/.cicd/pipeline.yml index 9b23082af5..7d5fd6e084 100644 --- a/.cicd/pipeline.yml +++ b/.cicd/pipeline.yml @@ -1,5 +1,4 @@ steps: - - wait - label: ":aws: Amazon_Linux 2 - Build" @@ -70,8 +69,6 @@ steps: - label: ":darwin: macOS 10.14 - Build" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -82,22 +79,35 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.14.6_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}" modify-cpu: 12 modify-ram: 24 always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' + pre-commands: + - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" - EOSIO/skip-checkout#v0.1.1: cd: ~ + env: + PROJECT_TAG: ${MACOS_10_14_TAG} + REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} + REPO_COMMIT: $BUILDKITE_COMMIT + TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eosio.cdt && cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && . ./.cicd/platforms/macos-10.14.sh && cd ~/eosio.cdt && cd .. && rm -rf eosio.cdt" + TEMPLATE: 10.14.6_6C_14G_80G + TEMPLATE_TAG: clean::cicd::git-ssh::nas::brew::buildkite-agent + timeout: ${TIMEOUT:-120} agents: - "queue=mac-anka-large-node-fleet" - skip: $SKIP_MACOS_10_14 + skip: ${SKIP_MACOS_10_14}${SKIP_MAC} - label: ":darwin: macOS 10.15 - Build" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -108,16 +118,32 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.15.5_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}" modify-cpu: 12 modify-ram: 24 always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' + pre-commands: + - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'" + - EOSIO/skip-checkout#v0.1.1: + cd: ~ + env: + PROJECT_TAG: ${MACOS_10_15_TAG} + REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} + REPO_COMMIT: $BUILDKITE_COMMIT + TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eosio.cdt && cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && . ./.cicd/platforms/macos-10.15.sh && cd ~/eosio.cdt && cd .. && rm -rf eosio.cdt" + TEMPLATE: 10.15.5_6C_14G_80G + TEMPLATE_TAG: clean::cicd::git-ssh::nas::brew::buildkite-agent + timeout: ${TIMEOUT:-120} agents: - "queue=mac-anka-large-node-fleet" - skip: $SKIP_MACOS_10_15 - + skip: ${SKIP_MACOS_10_15}${SKIP_MAC} - wait @@ -189,8 +215,6 @@ steps: - label: ":darwin: macOS 10.14 - Unit Tests" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -201,22 +225,23 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.14.6_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - modify-cpu: 12 - modify-ram: 24 + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}" always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' - EOSIO/skip-checkout#v0.1.1: cd: ~ agents: - "queue=mac-anka-node-fleet" - skip: ${SKIP_MACOS_10_14}${SKIP_UNIT_TESTS} + skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_UNIT_TESTS} - label: ":darwin: macOS 10.15 - Unit Tests" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -227,15 +252,20 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.15.5_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - modify-cpu: 12 - modify-ram: 24 + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}" always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' + - EOSIO/skip-checkout#v0.1.1: + cd: ~ agents: - "queue=mac-anka-node-fleet" - skip: ${SKIP_MACOS_10_15}${SKIP_UNIT_TESTS} + skip: ${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_UNIT_TESTS} - label: ":aws: Amazon_Linux 2 - Toolchain Tests" command: @@ -305,8 +335,6 @@ steps: - label: ":darwin: macOS 10.14 - Toolchain Tests" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -317,22 +345,23 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.14.6_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - modify-cpu: 12 - modify-ram: 24 + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}" always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' - EOSIO/skip-checkout#v0.1.1: cd: ~ agents: - "queue=mac-anka-node-fleet" - skip: ${SKIP_MACOS_10_14}${SKIP_TOOLCHAIN_TESTS} + skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_TOOLCHAIN_TESTS} - label: ":darwin: macOS 10.15 - Toolchain Tests" command: - - "brew update && brew upgrade" - - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3" - "git clone $BUILDKITE_REPO eosio.cdt" - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi" - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive" @@ -343,17 +372,20 @@ steps: no-volume: true inherit-environment-vars: true vm-name: 10.15.5_6C_14G_80G - vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent" - modify-cpu: 12 - modify-ram: 24 + vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}" always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' - EOSIO/skip-checkout#v0.1.1: cd: ~ agents: - "queue=mac-anka-node-fleet" - skip: ${SKIP_MACOS_10_14}${SKIP_TOOLCHAIN_TESTS} + skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_TOOLCHAIN_TESTS} - label: ":ubuntu: Ubuntu 18.04 - Integration Tests" command: @@ -482,12 +514,17 @@ steps: always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' - EOSIO/skip-checkout#v0.1.1: cd: ~ agents: - "queue=mac-anka-node-fleet" timeout: ${TIMEOUT:-20} - skip: ${SKIP_MACOS_10_14}${SKIP_PACKAGE_BUILDER} + skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_PACKAGE_BUILDER} - label: ":darwin: Catalina - Package Builder" command: @@ -505,10 +542,17 @@ steps: always-pull: true debug: true wait-network: true + pre-execute-sleep: 5 + pre-execute-ping-sleep: github.com + failover-registries: + - 'registry-1' + - 'registry-2' + - EOSIO/skip-checkout#v0.1.1: + cd: ~ agents: - "queue=mac-anka-node-fleet" timeout: ${TIMEOUT:-20} - skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER} + skip: ${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_PACKAGE_BUILDER} - wait @@ -529,7 +573,7 @@ steps: agents: queue: "automation-basic-builder-fleet" timeout: "${TIMEOUT:-5}" - skip: ${SKIP_MACOS_10_14}${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER} + skip: ${SKIP_MACOS_10_14}${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_PACKAGE_BUILDER} - label: ":git: Git Submodule Regression Check" command: diff --git a/.cicd/docker/amazonlinux-2.dockerfile b/.cicd/platforms/amazonlinux-2.dockerfile similarity index 100% rename from .cicd/docker/amazonlinux-2.dockerfile rename to .cicd/platforms/amazonlinux-2.dockerfile diff --git a/.cicd/docker/centos-7.7.dockerfile b/.cicd/platforms/centos-7.7.dockerfile similarity index 100% rename from .cicd/docker/centos-7.7.dockerfile rename to .cicd/platforms/centos-7.7.dockerfile diff --git a/.cicd/docker/centos-8.dockerfile b/.cicd/platforms/centos-8.dockerfile similarity index 100% rename from .cicd/docker/centos-8.dockerfile rename to .cicd/platforms/centos-8.dockerfile diff --git a/.cicd/platforms/macos-10.14.sh b/.cicd/platforms/macos-10.14.sh new file mode 100755 index 0000000000..3c91271a9c --- /dev/null +++ b/.cicd/platforms/macos-10.14.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eou pipefail +VERSION=1 + +brew update && brew upgrade +brew install automake cmake doxygen gettext git gmp graphviz lcov libtool python@3 wget diff --git a/.cicd/platforms/macos-10.15.sh b/.cicd/platforms/macos-10.15.sh new file mode 100755 index 0000000000..3c91271a9c --- /dev/null +++ b/.cicd/platforms/macos-10.15.sh @@ -0,0 +1,6 @@ +#!/bin/bash +set -eou pipefail +VERSION=1 + +brew update && brew upgrade +brew install automake cmake doxygen gettext git gmp graphviz lcov libtool python@3 wget diff --git a/.cicd/docker/ubuntu-16.04.dockerfile b/.cicd/platforms/ubuntu-16.04.dockerfile similarity index 100% rename from .cicd/docker/ubuntu-16.04.dockerfile rename to .cicd/platforms/ubuntu-16.04.dockerfile diff --git a/.cicd/docker/ubuntu-18.04.dockerfile b/.cicd/platforms/ubuntu-18.04.dockerfile similarity index 100% rename from .cicd/docker/ubuntu-18.04.dockerfile rename to .cicd/platforms/ubuntu-18.04.dockerfile diff --git a/.cicd/docker/ubuntu-20.04.dockerfile b/.cicd/platforms/ubuntu-20.04.dockerfile similarity index 100% rename from .cicd/docker/ubuntu-20.04.dockerfile rename to .cicd/platforms/ubuntu-20.04.dockerfile