Anka prebuilt image for 1.8.x branch.

.cicd/generate-base-images.sh

@@ -9,7 +9,7 @@ TAG=$(echo $FULL_TAG | cut -d: -f2)
 EXISTS=$(curl -s -H "Authorization: Bearer $(curl -sSL "https://auth.docker.io/token?service=registry.docker.io&scope=repository:${ORG_REPO}:pull" | jq --raw-output .token)" "https://registry.hub.docker.com/v2/${ORG_REPO}/manifests/$TAG")
 # build, if neccessary
 if [[ $EXISTS =~ '404 page not found' || $EXISTS =~ 'manifest unknown' ]]; then # if we cannot pull the image, we build and push it first
-    docker build -t $FULL_TAG -f $CICD_DIR/docker/${IMAGE_TAG}.dockerfile .
+    docker build -t $FULL_TAG -f $CICD_DIR/platforms/${IMAGE_TAG}.dockerfile .
     docker push $FULL_TAG
 else
     echo "$FULL_TAG already exists."

.cicd/helpers/docker-hash.sh

@@ -16,7 +16,7 @@ function determine-hash() {
 }
 
 if [[ ! -z $IMAGE_TAG ]]; then
-    determine-hash "$CICD_DIR/docker/${IMAGE_TAG}.dockerfile"
+    determine-hash "$CICD_DIR/platforms/${IMAGE_TAG}.dockerfile"
     export FULL_TAG="eosio/ci:eosio-cdt-$HASHED_IMAGE_TAG"
 else
     echo "Please set ENV::IMAGE_TAG to match the name of a platform dockerfile..."

.cicd/pipeline-upload.sh

@@ -0,0 +1,9 @@
+#!/bin/bash
+set -eou pipefail
+
+export MACOS_10_14_TAG="eosio-cdt-macos-10.14-$(sha1sum ./.cicd/platforms/macos-10.14.sh | awk '{print $1}')"
+export MACOS_10_15_TAG="eosio-cdt-macos-10.15-$(sha1sum ./.cicd/platforms/macos-10.15.sh | awk '{print $1}')"
+export VARS='$MACOS_10_14_TAG:$MACOS_10_15_TAG'
+envsubst "$VARS" < "./.cicd/pipeline.yml" > "./.cicd/pipeline.yml.out"
+buildkite-agent artifact upload ./.cicd/pipeline.yml.out
+buildkite-agent pipeline upload ./.cicd/pipeline.yml.out

.cicd/pipeline.yml

@@ -1,5 +1,4 @@
 steps:
-
   - wait
 
   - label: ":aws: Amazon_Linux 2 - Build"
@@ -70,8 +69,6 @@ steps:
 
   - label: ":darwin: macOS 10.14 - Build"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -82,22 +79,35 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.14.6_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}"
           modify-cpu: 12
           modify-ram: 24
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
+          pre-commands:
+            - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'"
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
+    env:
+      PROJECT_TAG: ${MACOS_10_14_TAG}
+      REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}
+      REPO_COMMIT: $BUILDKITE_COMMIT
+      TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eosio.cdt && cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && . ./.cicd/platforms/macos-10.14.sh && cd ~/eosio.cdt && cd .. && rm -rf eosio.cdt"
+      TEMPLATE: 10.14.6_6C_14G_80G
+      TEMPLATE_TAG: clean::cicd::git-ssh::nas::brew::buildkite-agent
+    timeout: ${TIMEOUT:-120}
     agents:
       - "queue=mac-anka-large-node-fleet"
-    skip: $SKIP_MACOS_10_14
+    skip: ${SKIP_MACOS_10_14}${SKIP_MAC}
 
   - label: ":darwin: macOS 10.15 - Build"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -108,16 +118,32 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.15.5_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}"
           modify-cpu: 12
           modify-ram: 24
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
+          pre-commands:
+            - "rm -rf mac-anka-fleet; git clone git@github.com:EOSIO/mac-anka-fleet.git && cd mac-anka-fleet && . ./ensure-tag.bash -u 12 -r 25G -a '-n'"
+      - EOSIO/skip-checkout#v0.1.1:
+          cd: ~
+    env:
+      PROJECT_TAG: ${MACOS_10_15_TAG}
+      REPO: ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}
+      REPO_COMMIT: $BUILDKITE_COMMIT
+      TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} eosio.cdt && cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive && . ./.cicd/platforms/macos-10.15.sh && cd ~/eosio.cdt && cd .. && rm -rf eosio.cdt"
+      TEMPLATE: 10.15.5_6C_14G_80G
+      TEMPLATE_TAG: clean::cicd::git-ssh::nas::brew::buildkite-agent
+    timeout: ${TIMEOUT:-120}
     agents:
       - "queue=mac-anka-large-node-fleet"
-    skip: $SKIP_MACOS_10_15
-
+    skip: ${SKIP_MACOS_10_15}${SKIP_MAC}
 
   - wait
 
@@ -189,8 +215,6 @@ steps:
 
   - label: ":darwin: macOS 10.14 - Unit Tests"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -201,22 +225,23 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.14.6_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
-          modify-cpu: 12
-          modify-ram: 24
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}"
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
-    skip: ${SKIP_MACOS_10_14}${SKIP_UNIT_TESTS}
+    skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_UNIT_TESTS}
 
   - label: ":darwin: macOS 10.15 - Unit Tests"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -227,15 +252,20 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.15.5_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
-          modify-cpu: 12
-          modify-ram: 24
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}"
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
+      - EOSIO/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
-    skip: ${SKIP_MACOS_10_15}${SKIP_UNIT_TESTS}
+    skip: ${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_UNIT_TESTS}
 
   - label: ":aws: Amazon_Linux 2 - Toolchain Tests"
     command:
@@ -305,8 +335,6 @@ steps:
 
   - label: ":darwin: macOS 10.14 - Toolchain Tests"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -317,22 +345,23 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.14.6_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
-          modify-cpu: 12
-          modify-ram: 24
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_14_TAG}"
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
-    skip: ${SKIP_MACOS_10_14}${SKIP_TOOLCHAIN_TESTS}
+    skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_TOOLCHAIN_TESTS}
 
   - label: ":darwin: macOS 10.15 - Toolchain Tests"
     command:
-      - "brew update && brew upgrade"
-      - "brew install git automake libtool wget cmake gmp gettext doxygen graphviz lcov python@3"
       - "git clone $BUILDKITE_REPO eosio.cdt"
       - "cd eosio.cdt && if [[ $BUILDKITE_BRANCH =~ ^pull/[0-9]+/head: ]]; then git fetch -v --prune origin refs/pull/$(echo $BUILDKITE_BRANCH | cut -d/ -f2)/head; fi"
       - "cd eosio.cdt && git checkout -f $BUILDKITE_COMMIT && git submodule update --init --recursive"
@@ -343,17 +372,20 @@ steps:
           no-volume: true
           inherit-environment-vars: true
           vm-name: 10.15.5_6C_14G_80G
-          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent"
-          modify-cpu: 12
-          modify-ram: 24
+          vm-registry-tag: "clean::cicd::git-ssh::nas::brew::buildkite-agent::${MACOS_10_15_TAG}"
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
-    skip: ${SKIP_MACOS_10_14}${SKIP_TOOLCHAIN_TESTS}
+    skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_TOOLCHAIN_TESTS}
 
   - label: ":ubuntu: Ubuntu 18.04 - Integration Tests"
     command:
@@ -482,12 +514,17 @@ steps:
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
       - EOSIO/skip-checkout#v0.1.1:
           cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
     timeout: ${TIMEOUT:-20}
-    skip: ${SKIP_MACOS_10_14}${SKIP_PACKAGE_BUILDER}
+    skip: ${SKIP_MACOS_10_14}${SKIP_MAC}${SKIP_PACKAGE_BUILDER}
 
   - label: ":darwin: Catalina - Package Builder"
     command:
@@ -505,10 +542,17 @@ steps:
           always-pull: true
           debug: true
           wait-network: true
+          pre-execute-sleep: 5
+          pre-execute-ping-sleep: github.com
+          failover-registries:
+            - 'registry-1'
+            - 'registry-2'
+      - EOSIO/skip-checkout#v0.1.1:
+          cd: ~
     agents:
       - "queue=mac-anka-node-fleet"
     timeout: ${TIMEOUT:-20}
-    skip: ${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}
+    skip: ${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_PACKAGE_BUILDER}
 
   - wait
 
@@ -529,7 +573,7 @@ steps:
     agents:
       queue: "automation-basic-builder-fleet"
     timeout: "${TIMEOUT:-5}"
-    skip: ${SKIP_MACOS_10_14}${SKIP_MACOS_10_15}${SKIP_PACKAGE_BUILDER}
+    skip: ${SKIP_MACOS_10_14}${SKIP_MACOS_10_15}${SKIP_MAC}${SKIP_PACKAGE_BUILDER}
 
   - label: ":git: Git Submodule Regression Check"
     command:

.cicd/platforms/macos-10.14.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eou pipefail
+VERSION=1
+
+brew update && brew upgrade
+brew install automake cmake doxygen gettext git gmp graphviz lcov libtool python@3 wget

.cicd/platforms/macos-10.15.sh

@@ -0,0 +1,6 @@
+#!/bin/bash
+set -eou pipefail
+VERSION=1
+
+brew update && brew upgrade
+brew install automake cmake doxygen gettext git gmp graphviz lcov libtool python@3 wget