Skip to content
This repository has been archived by the owner on Aug 2, 2022. It is now read-only.

Anka for 1.8.x + new linux vm image labels #7335

Merged
merged 5 commits into from
May 12, 2019
Merged

Anka for 1.8.x + new linux vm image labels #7335

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
b6bba5b
anka pipeline changes + 1.8.0-rc1 for linux
May 11, 2019
9d6d39d
test script fixes
May 11, 2019
5762363
ctest fix
May 11, 2019
df62b58
test revert for testing
May 11, 2019
1158c86
testing new path fix
May 11, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
193 changes: 125 additions & 68 deletions .buildkite/pipeline.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,19 +1,53 @@
env:
ANKA_MOJAVE_TEMPLATE: "10.14.4_6C_14G_40G"
ANKA_TEMPLATE_TAG: "clean::cicd::git-ssh::nas::brew::buildkite-agent-3.11.2"
CHECKSUMABLE: "scripts/eosio_build*"
MAC_TAG: "eosio_1-8-0-rc1"
WORKDIR: "/data/job"

steps:
- command: | # macOS Mojave Build
echo "--- Creating symbolic link to job directory :file_folder:"
sleep 5
ln -s "$(pwd)" /data/job
cd /data/job
echo "+++ Building :hammer:"
./scripts/eosio_build.sh -y -P -m
echo "--- Compressing build directory :compression:"
tar -pczf build.tar.gz build
if [[ ! -f build.tar.gz ]]; then echo 'ERROR: No build.tar.gz artifact found!' && exit 1; fi
label: ":darwin: Mojave Build"

- label: ":darwin: Ensure Mojave Anka Template Dependency Tag/Layer Exists"
command:
- "git clone git@github.com:EOSIO/mac-anka-fleet.git"
- "cd mac-anka-fleet && . ./ensure_tag.bash -a \"-n\""
agents:
- "role=builder-v2-2"
- "os=mojave"
artifact_paths: "build.tar.gz"
- "queue=mac-anka-templater-fleet"
env:
REPO: "${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO}"
REPO_COMMIT: "${BUILDKITE_COMMIT}"
CHECKSUMABLE: "${CHECKSUMABLE}"
TEMPLATE: "${ANKA_MOJAVE_TEMPLATE}"
TEMPLATE_TAG: "${ANKA_TEMPLATE_TAG}"
TAG_COMMANDS: "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} && cd eos && git checkout ${BUILDKITE_COMMIT} && git submodule update --init --recursive && ./scripts/eosio_build.sh -y -P -f -m && cd .. && rm -rf eos" # CLONED_REPO_DIR IS REQUIRED and is where the repo is always cloned into
PROJECT_TAG: "${MAC_TAG}"
timeout: 320


- wait

############################
# BUILDING #################
############################

- label: ":darwin: [Darwin] Mojave Build"
command:
- "git clone ${BUILDKITE_PULL_REQUEST_REPO:-$BUILDKITE_REPO} && cd eos && git checkout ${BUILDKITE_COMMIT} && git submodule update --init --recursive && ./scripts/eosio_build.sh -y -P -m"
- "tar -pczf /Network/NAS/MAC_FLEET/BUILDKITE/artifacts/${ANKA_MOJAVE_TEMPLATE}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_ID}.tar.gz eos"
plugins:
chef/anka#v0.4.4:
no-volume: true
workdir: $WORKDIR
inherit-environment-vars: true
vm-name: $ANKA_MOJAVE_TEMPLATE
vm-registry-tag: "${ANKA_TEMPLATE_TAG}::${MAC_TAG}"
modify-cpu: 12
modify-ram: 24
always-pull: true
debug: true
wait-network: true
agents:
- "queue=mac-anka-large-node-fleet"
timeout: 120

- command: | # Amazon Linux 2 Build
Expand All @@ -34,7 +68,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 120
Expand All @@ -57,7 +91,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 120
Expand All @@ -80,7 +114,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 120
Expand All @@ -103,13 +137,57 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 120

- wait


################################
# TESTS ########################
################################

# MOJAVE #
- label: ":darwin: [Darwin] Mojave Tests"
command:
- "tar -xzf /Network/NAS/MAC_FLEET/BUILDKITE/artifacts/${ANKA_MOJAVE_TEMPLATE}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_ID}.tar.gz"
- "cd eos && ./scripts/parallel-test.sh"
agents:
- "queue=mac-anka-node-fleet"
plugins:
chef/anka#v0.4.4:
no-volume: true
workdir: $WORKDIR
inherit-environment-vars: true
bash-interactive: true
vm-name: $ANKA_MOJAVE_TEMPLATE
vm-registry-tag: "${ANKA_TEMPLATE_TAG}::${MAC_TAG}"
always-pull: true
debug: true
wait-network: true
timeout: 120

- label: ":darwin: [Darwin] Mojave NP Tests"
command:
- "tar -xzf /Network/NAS/MAC_FLEET/BUILDKITE/artifacts/${ANKA_MOJAVE_TEMPLATE}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_ID}.tar.gz"
- "cd eos && ./scripts/serial-test.sh"
agents:
- "queue=mac-anka-node-fleet"
plugins:
chef/anka#v0.4.4:
no-volume: true
workdir: $WORKDIR
inherit-environment-vars: true
bash-interactive: true
vm-name: $ANKA_MOJAVE_TEMPLATE
vm-registry-tag: "${ANKA_TEMPLATE_TAG}::${MAC_TAG}"
always-pull: true
debug: true
wait-network: true
timeout: 120

# Amazon Linux 2 Tests
- command: |
echo "--- :arrow_down: Downloading Build Directory"
Expand All @@ -127,7 +205,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -148,7 +226,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_2-2"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:amazonlinux2_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -170,7 +248,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -191,7 +269,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_2-2"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -213,7 +291,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -234,7 +312,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_2-2"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -256,7 +334,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60
Expand All @@ -277,35 +355,11 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_2-2"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
timeout: 60

# Mojave Tests
- command: |
echo "--- :arrow_down: Downloading Build Directory"
buildkite-agent artifact download "build.tar.gz" . --step ":darwin: Mojave Build"
echo "+++ :microscope: Running Tests"
ln -s "$(pwd)" /data/job
./scripts/parallel-test.sh
label: ":darwin: Mojave Tests"
agents:
- "role=tester-v2-2"
- "os=mojave"
timeout: 60
- command: |
echo "--- :arrow_down: Downloading Build Directory"
buildkite-agent artifact download "build.tar.gz" . --step ":darwin: Mojave Build"
echo "+++ :microscope: Running Tests"
ln -s "$(pwd)" /data/job
./scripts/serial-test.sh
label: ":darwin: Mojave NP Tests"
agents:
- "role=tester-v2-1"
- "os=mojave"
timeout: 60

- wait:
continue_on_failure: true

Expand Down Expand Up @@ -347,7 +401,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:centos7_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
env:
Expand Down Expand Up @@ -375,7 +429,7 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu16_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
env:
Expand Down Expand Up @@ -403,30 +457,33 @@ steps:
region: "us-west-2"
docker#v2.1.0:
debug: true
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_2-3"
image: "436617320021.dkr.ecr.us-west-2.amazonaws.com/ci:ubuntu18_1-8-0-rc1"
propagate-environment: true
workdir: /data/job
env:
OS: "ubuntu-18.04"
PKGTYPE: "deb"
timeout: 60

- command: | # macOS Mojave Package Builder
echo "--- :arrow_down: Downloading build directory"
buildkite-agent artifact download "build.tar.gz" . --step ":darwin: Mojave Build"
tar -zxf build.tar.gz
echo "+++ :microscope: Starting package build"
ln -s "$(pwd)" /data/job
cd /data/job/build/packages
bash generate_package.sh brew
label: ":darwin: Mojave Package Builder"
- label: ":darwin: Mojave Package Builder"
command:
- "tar -xzf /Network/NAS/MAC_FLEET/BUILDKITE/artifacts/${ANKA_MOJAVE_TEMPLATE}-${BUILDKITE_PIPELINE_SLUG}-${BUILDKITE_BUILD_ID}.tar.gz"
- "cd eos/build/packages && bash generate_package.sh brew"
- "cd eos && buildkite-agent artifact upload build/packages/*.tar.gz"
- "cd eos && buildkite-agent artifact upload build/packages/*.rb"
agents:
- "role=builder-v2-2"
- "os=mojave"
artifact_paths:
- "build/packages/*.tar.gz"
- "build/packages/*.rb"
timeout: 60
- "queue=mac-anka-node-fleet"
plugins:
chef/anka#v0.4.4:
no-volume: true
inherit-environment-vars: true
bash-interactive: true
vm-name: $ANKA_MOJAVE_TEMPLATE
vm-registry-tag: "${ANKA_TEMPLATE_TAG}::${MAC_TAG}"
always-pull: true
debug: true
wait-network: true
timeout: 120

- wait

Expand Down
4 changes: 2 additions & 2 deletions scripts/parallel-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@ set -e # exit on failure of any "simple" command (excludes &&, ||, or | chains)
# prepare environment
PATH=$PATH:~/opt/mongodb/bin
echo "Extracting build directory..."
[[ -z "${1}" ]] && tar -zxf build.tar.gz || tar -xzf $1
cd /data/job/build
[[ -f build.tar.gz ]] && tar -xzf build.tar.gz
[[ $(uname) == "Darwin" ]] && cd /data/job/eos/build || cd /data/job/build
# run tests
echo "Running tests..."
CPU_CORES=$(getconf _NPROCESSORS_ONLN)
Expand Down
4 changes: 2 additions & 2 deletions scripts/serial-test.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,10 +3,10 @@ set -e # exit on failure of any "simple" command (excludes &&, ||, or | chains)
# prepare environment
PATH=$PATH:~/opt/mongodb/bin
echo "Extracting build directory..."
[[ -z "${1}" ]] && tar -zxf build.tar.gz || tar -xzf $1
[[ -f build.tar.gz ]] && tar -xzf build.tar.gz
[[ $(uname) == "Darwin" ]] && cd /data/job/eos/build || cd /data/job/build
echo "Starting MongoDB..."
mongod --fork --dbpath ~/data/mongodb -f ~/etc/mongod.conf --logpath "$(pwd)"/mongod.log
cd /data/job/build
# run tests
echo "Running tests..."
TEST_COUNT=$(ctest -N -L nonparallelizable_tests | grep -i 'Total Tests: ' | cut -d ':' -f 2 | awk '{print $1}')
Expand Down