Support a nicer way of fetching node process metrics. #154
Workflow file for this run
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Test debian-11-slim-primary | |
| on: | |
| push: | |
| branches: | |
| - develop | |
| - v*-releases | |
| pull_request: | |
| branches: | |
| - develop | |
| - v*-releases | |
| jobs: | |
| debian-11-slim-primary: | |
| runs-on: self-hosted | |
| timeout-minutes: 120 | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| # NOTE(mikal): git repos are checked out to /srv/github/_work/{repo}/{repo} | |
| # which is available as GITHUB_WORKSPACE. You can find other environment | |
| # variables at https://docs.github.com/en/actions/learn-github-actions/environment-variables | |
| steps: | |
| - name: Set environment variables | |
| run: | | |
| echo "SF_HEAD_SHA=${{ github.event.pull_request.head.sha }}" >> $GITHUB_ENV | |
| echo "SF_PRIMARY_REPO=$( echo ${{ github.repository }} | cut -f 2 -d '/' )" >> $GITHUB_ENV | |
| echo "SHAKENFIST_NAMESPACE=$(hostname)" >> $GITHUB_ENV | |
| - name: Checkout client-python | |
| uses: actions/checkout@v3 | |
| with: | |
| path: client-python | |
| fetch-depth: 0 | |
| - name: Determine if there is any dependency between the repositories | |
| run: | | |
| python3 ${GITHUB_WORKSPACE}/client-python/tools/clone_with_depends.py | |
| - name: Build infrastructure | |
| run: | | |
| cd ${GITHUB_WORKSPACE}/shakenfist | |
| ansible-playbook -i /home/debian/ansible-hosts \ | |
| --extra-vars "identifier=${SHAKENFIST_NAMESPACE} source_path=${GITHUB_WORKSPACE} \ | |
| base_image=sf://label/ci-images/debian-11 base_image_user=debian" \ | |
| deploy/ansible/ci-topology-slim-primary.yml | |
| - name: Log github actions buffering status | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| cd ${GITHUB_WORKSPACE}/shakenfist | |
| scp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| tools/buffer.py debian@$primary:buffer.py | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary python3 buffer.py | |
| - name: Run getsf installer on primary | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary /tmp/getsf-wrapper | |
| echo "" | |
| echo "" | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| 'sudo rm /etc/apache2/sites-enabled/*; sudo a2ensite sf-example.conf; sudo apachectl graceful' | |
| - name: Wait for API to start answering | |
| run: | | |
| set +e | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary 'sudo chmod ugo+r /etc/sf/sfrc /etc/sf/shakenfist.json /var/log/syslog' | |
| count=0 | |
| while [ $count -lt 60 ] | |
| do | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary '. /etc/sf/sfrc; sf-client instance list' | |
| if [ $? == 0 ]; then | |
| exit 0 | |
| fi | |
| count=$(( $count + 1 )) | |
| sleep 5 | |
| done | |
| exit 1 | |
| - name: Import cached images | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| '. /etc/sf/sfrc; sf-client artifact upload ubuntu-1804 /srv/ci/ubuntu:18.04 --shared' | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| '. /etc/sf/sfrc; sf-client artifact upload ubuntu-2004 /srv/ci/ubuntu:20.04 --shared' | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| '. /etc/sf/sfrc; sf-client artifact upload debian-11 /srv/ci/debian:11 --shared' | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| '. /etc/sf/sfrc; sf-client artifact upload cirros /srv/ci/cirros --shared' | |
| - name: Create a base level of activity in the cluster | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| 'echo "==== sfrc ===="; cat /etc/sf/sfrc; echo "==== end sfrc ===="' | |
| echo "" | |
| echo "" | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary \ | |
| '. /etc/sf/sfrc; for i in `seq 100`; do sf-client --async=continue network create background-$i 10.$i.0.0/24 > /dev/null; echo -n "."; done' | |
| echo "" | |
| - name: Run functional tests | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| $source_path/shakenfist \ | |
| debian@$primary:shakenfist | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary "cd shakenfist/deploy; . /etc/sf/sfrc; sudo pip3 install -r requirements.txt; stestr run --concurrency=5" | |
| - name: Check logs | |
| if: ${{ ! cancelled() }} | |
| run: | | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary sudo chmod ugo+r /var/log/syslog | |
| scp -rp -i /srv/github/id_ci -o StrictHostKeyChecking=no \ | |
| -o UserKnownHostsFile=/dev/null \ | |
| debian@$primary:/var/log/syslog \ | |
| ${GITHUB_WORKSPACE}/syslog | |
| failures=0 | |
| echo | |
| etcd_conns=`grep -c "Building new etcd connection" ${GITHUB_WORKSPACE}/syslog || true` | |
| echo "This CI run created $etcd_conns etcd connections." | |
| if [ $etcd_conns -gt 5000 ]; then | |
| echo "FAILURE: Too many etcd clients!" | |
| failures=1 | |
| fi | |
| FORBIDDEN=("Traceback (most recent call last):" | |
| "ERROR sf" | |
| " died" | |
| "Extra vxlan present" | |
| "Fork support is only compatible with the epoll1 and poll polling strategies" | |
| "not using configured address" | |
| "Dumping thread traces" | |
| "because it is leased to" | |
| "not committing online upgrade" | |
| "Received a GOAWAY with error code ENHANCE_YOUR_CALM" | |
| "ConnectionFailedError") | |
| IFS="" | |
| for forbid in ${FORBIDDEN[*]} | |
| do | |
| if [ `grep -c "$forbid" ${GITHUB_WORKSPACE}/syslog || true` -gt 0 ] | |
| then | |
| echo "FAILURE: Forbidden string found in logs: $forbid" | |
| failures=1 | |
| fi | |
| done | |
| if [ $failures -gt 0 ]; then | |
| echo "...failures detected." | |
| exit 1 | |
| fi | |
| - name: Gather logs | |
| if: ${{ ! cancelled() }} | |
| run: | | |
| set -x | |
| . ${GITHUB_WORKSPACE}/ci-environment.sh | |
| cd ${GITHUB_WORKSPACE} | |
| rm -rf artifacts bundle | |
| mkdir -p artifacts bundle | |
| echo "==============================" | |
| for node in $nodes | |
| do | |
| echo "=== $node ===" | |
| safe_node=$(echo $node | tr "-" "_") | |
| ssh -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@${!safe_node} sudo /srv/shakenfist/venv/bin/clingwrap gather \ | |
| --target /srv/shakenfist/venv/share/clingwrap/shakenfist-ci-failure.cwd --output /tmp/$node.zip | |
| scp -i /srv/github/id_ci -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null \ | |
| debian@${!safe_node}:/tmp/$node.zip bundle/ | |
| mkdir -p bundle/$node | |
| # We ignore the exit code of unzip, as it will return 1 if a warning is | |
| # emitted during expansion, even though expansion worked correctly. For | |
| # example stripping a leading / from paths. | |
| unzip -q bundle/$node.zip -d bundle/$node/ || true | |
| rm bundle/$node.zip | |
| done | |
| zip -rq artifacts/bundle.zip bundle/* | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ ! cancelled() }} | |
| with: | |
| name: bundle.zip | |
| retention-days: 90 | |
| if-no-files-found: error | |
| path: ${{ github.workspace }}/artifacts/bundle.zip |