diff --git a/.circleci/config.yml b/.circleci/config.yml index 828cc9bbe4..a860da02e4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -199,11 +199,10 @@ commands: working_directory: integration-tests name: Slack notification on dev branch only command: | - if [ "$CIRCLE_NODE_INDEX" -eq 0 ] && [ "$CIRCLE_BRANCH" = "dev" ]; then + if [ "$CIRCLE_BRANCH" = "alex-ci-testing" ]; then bash ../.circleci/wait-for-job-finish.sh node --require ../.pnp.cjs ./slack/notify-circleci-test-results.js fi - no_output_timeout: 30m when: always jobs: diff --git a/.circleci/wait-for-job-finish.sh b/.circleci/wait-for-job-finish.sh index 7d9531dfa5..79792eace2 100755 --- a/.circleci/wait-for-job-finish.sh +++ b/.circleci/wait-for-job-finish.sh @@ -6,26 +6,35 @@ set -o nounset # Use the error status of the first failure, rather than that of the last item in a pipeline. Also fail explicitly on any exit code. set -eo pipefail +trap 's=$?; echo -e >&2 "\nError in $0:\nat line "$LINENO": $BASH_COMMAND"; exit $s' ERR + +echo "Starting wait-for-job-finish script" + +date counter=0 +URL_BASE="https://circleci.com/api/v2/project/github/DataBiosphere/terra-ui" # Wait up to 25 minutes for job to finish. A job can run on multiple nodes: parallelism > 1 while [ "$counter" -le 1500 ]; do - # Find number of nodes in running - job_detail=$(curl -s "https://circleci.com/api/v2/project/github/DataBiosphere/terra-ui/job/$CIRCLE_BUILD_NUM") - job_running_nodes_count=$(echo "$job_detail" | jq -r '[.parallel_runs[] | select(.status == "running") | select(.index != '"$CIRCLE_NODE_INDEX"')] | length') - - if [ "$job_running_nodes_count" -eq 0 ]; then - exit 0 + # Get job details + job_detail=$(curl -s "$URL_BASE/job/$CIRCLE_BUILD_NUM") + + # Wait for all nodes with status==running excluding self node + nodes=$(echo "$job_detail" | jq -r '.parallel_runs[]') + running_nodes=$(echo "$nodes" | jq -r --arg IDX "$CIRCLE_NODE_INDEX" 'select(.status=="running") | select(.index|tostring!=$IDX)') + count=$(echo "$running_nodes" | grep -c -e "running" || test $? = 1;) + if [ "$count" -eq 0 ]; then + echo "Checking from NODE_INDEX #$CIRCLE_NODE_INDEX: Parallel running nodes have finished. Waited $counter seconds." + echo "$nodes" + exit 0 fi + echo "Waiting for parallel running nodes to finish. Sleep 10 seconds." sleep 10 counter=$(($counter + 10)) done -echo "Waited total $counter seconds" -date - # Something is wrong. Log response for error troubleshooting curl -s "https://circleci.com/api/v2/project/github/DataBiosphere/terra-ui/job/$CIRCLE_BUILD_NUM" | jq -r '.' -echo "ERROR: Exceeded maximum waiting time 25 minutes." +echo "ERROR: Exceeded maximum wait time 25 minutes." exit 1