Tests #890
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| --- | |
| name: Tests | |
| on: | |
| pull_request: | |
| push: | |
| branches: [main] | |
| # Run at 4am everyday (see https://crontab.guru) | |
| schedule: | |
| - cron: "0 4 * * *" | |
| env: | |
| PYTHON_VERSION: 3.8.8 | |
| PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python | |
| SECONDS: 120 | |
| jobs: | |
| tests: | |
| name: Run tests | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| matrix: | |
| os: [ubuntu-latest] | |
| steps: | |
| - name: Set up git repository | |
| uses: actions/checkout@v3.1.0 | |
| - name: Install OS dependencies | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| sudo apt update | |
| sudo apt install -y expect \ | |
| python3-opengl \ | |
| xvfb | |
| echo 'export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python' >> "${HOME}/.bashrc" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| brew install expect | |
| brew install --cask anaconda | |
| export PATH="/usr/local/anaconda3/bin:$PATH" | |
| echo 'export PATH="/usr/local/anaconda3/bin:$PATH"' >> "${HOME}/.zshrc" | |
| echo 'export PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION=python' >> "${HOME}/.zshrc" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Populate acceptance tests | |
| run: | | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/install_test.exp" ./install_test.exp | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/cra_test.exp" ./cra_test.exp | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/ispa_test.exp" ./ispa_test.exp | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/satellite_hsj_test.exp" ./satellite_hsj_test.exp | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/creditfraud_hsj_test.exp" ./creditfraud_hsj_test.exp | |
| cp "${GITHUB_WORKSPACE}/.github/helpers/quality-gates/digits_hsj_test.exp" ./digits_hsj_test.exp | |
| - name: Setup Anaconda | |
| uses: conda-incubator/setup-miniconda@v2 | |
| with: | |
| activate-environment: counterfit | |
| auto-update-conda: true | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Setup Conda | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| echo 'conda activate "counterfit"' >> "${HOME}/.profile" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| echo 'conda activate "counterfit"' >> "${HOME}/.zshrc" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Install Pycld2 | |
| run: | | |
| pip install https://github.com/aboSamoor/pycld2/zipball/e3ac86ed4d4902e912691c1531d0c5645382a726 | |
| - name: Setup counterfit | |
| shell: bash -l {0} | |
| run: | | |
| echo "PYTHON_PATH=$(which python)" >> $GITHUB_ENV | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| cd "${GITHUB_WORKSPACE}/" | |
| pip install --no-input .[dev] | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| cd "${GITHUB_WORKSPACE}/" | |
| pip install --no-input .[dev] | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test counterfit install | |
| shell: bash -l {0} | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./install_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| expect -d ./install_test.exp Darwin ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test Initial State Perturbation Attack for n minutes | |
| shell: bash -l {0} | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./ispa_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| expect -d ./ispa_test.exp Darwin ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test Corrupted Replay Attack (CRA) for n minutes | |
| shell: bash -l {0} | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./cra_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| expect -d ./cra_test.exp Darwin ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test Hop Skip Jump Attack (HSJ) attack on Satellite target for n minutes | |
| shell: bash -l {0} | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./satellite_hsj_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| expect -d ./satellite_hsj_test.exp Darwin ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test Hop Skip Jump Attack (HSJ) attack on CreditFraud target for n minutes | |
| shell: bash -l {0} | |
| run: | | |
| if [ "${RUNNER_OS}" == "Linux" ]; then | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./creditfraud_hsj_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| elif [ "${RUNNER_OS}" == "macOS" ]; then | |
| conda activate counterfit | |
| expect -d ./creditfraud_hsj_test.exp Darwin ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| else | |
| echo "${RUNNER_OS} not supported" | |
| exit 1 | |
| fi | |
| - name: Test Hop Skip Jump Attack (HSJ) attack on Digits Keras | |
| shell: bash -l {0} | |
| run: | | |
| source "${HOME}/.profile" | |
| conda activate counterfit | |
| expect -d ./digits_hsj_test.exp Linux ${{ env.SECONDS }} "${GITHUB_WORKSPACE}/examples/terminal/terminal.py" | |
| - name: Report failure | |
| uses: nashmaniac/create-issue-action@v1.1 | |
| # Only report failures of pushes (PRs have are visible through the Checks section) to the default branch | |
| if: failure() && github.event_name == 'main' && github.ref == 'refs/heads/main' | |
| with: | |
| title: 🐛 Coverage report failed for ${{ github.sha }} | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| labels: kind/bug | |
| body: https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }} |