Full Test Suite #5250
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Full Test Suite | |
| on: | |
| workflow_call: | |
| secrets: | |
| GCP_STORAGE_BUCKET: | |
| description: 'GCP STORAGE_BUCKET.' | |
| required: false | |
| schedule: | |
| - cron: '0 */4 * * *' | |
| env: | |
| timeout: 7200 | |
| GHA_MACHINE_TYPE: "n2-standard-4" | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| Matrix: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrices: ${{ steps.generate.outputs.matrices }} | |
| link: ${{ steps.link.outputs.link }} | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| - name: RW Jars link | |
| id: link | |
| run: | | |
| sudo apt update | |
| sudo apt install -y curl wget | |
| ./.github/scripts/get_rapidwright_link.sh | tee -a $GITHUB_OUTPUT | |
| - name: Generate Matrix | |
| id: generate | |
| run: | | |
| python3 -m pip install -r conf/requirements.txt | |
| PYTHONPATH=$(pwd) python3 ./.github/scripts/generate_job_matrix.py all | |
| Build_nextpnr-fpga_interchange-experimental: | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - name: Build Tools | |
| run: | | |
| ./.github/scripts/build_nextpnr_fpga_interchange.sh | |
| cp ./third_party/nextpnr/nextpnr-fpga_interchange nextpnr-fpga_interchange-experimental | |
| - name: Upload nextpnr-fpga_interchange-experimental | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: nextpnr-fpga_interchange-experimental | |
| path: nextpnr-fpga_interchange-experimental | |
| Build_nextpnr-fpga_interchange-experimental-single-thread: | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - name: Build Tools | |
| run: | | |
| ./.github/scripts/build_nextpnr_fpga_interchange.sh single_thread | |
| cp ./third_party/nextpnr/nextpnr-fpga_interchange nextpnr-fpga_interchange-experimental-single-thread | |
| - name: Upload nextpnr-fpga_interchange-experimental-single-thread | |
| uses: actions/upload-artifact@v3 | |
| if: always() | |
| with: | |
| name: nextpnr-fpga_interchange-experimental-single-thread | |
| path: nextpnr-fpga_interchange-experimental-single-thread | |
| # Symbiflow tests | |
| VPR: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).vpr }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: xilinx-a35t | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| VPRFASM2Bels: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).vpr_fasm2bels }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: xilinx-a35t | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v | |
| # Quicklogic tests | |
| QuickLogic: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).quicklogic }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: quicklogic | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| # Vivado | |
| Vivado: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).vivado }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: xilinx-a35t | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v | |
| YosysVivado: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).yosys_vivado }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: xilinx-a35t | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v | |
| YosysVivadoUHDM: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).yosys_vivado_uhdm }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: xilinx-a35t | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v | |
| VivadoAlreadySynthesized: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).vivado_already_synth }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - uses: actions/checkout@v3 | |
| with: | |
| submodules: recursive | |
| - name: Install | |
| run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
| - name: Run Test | |
| run: > | |
| source env.sh && | |
| python3 exhaust.py | |
| --verbose | |
| --project ${{ matrix.project }} | |
| --toolchain ${{ matrix.toolchain }} | |
| --board ${{ matrix.board }} | |
| --build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
| --fail | |
| --timeout ${{ env.timeout }} | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| path: | | |
| **/results*.gz | |
| **/plot_*.svg | |
| # Nextpnr | |
| NextpnrFPGAInterchange: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v -l $(echo ${{needs.Matrix.outputs.link}}) | |
| NextpnrFPGAInterchangeAlreadySynthesized: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_already_synth }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v -l $(echo ${{needs.Matrix.outputs.link}}) | |
| NextpnrFPGAInterchangeExperimentalAlreadySynthesized: | |
| needs: | |
| - Matrix | |
| - Build_nextpnr-fpga_interchange-experimental | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_experimental_already_synth }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - name: Download nextpnr-fpga_interchange-experimental | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nextpnr-fpga_interchange-experimental | |
| - name: Install nextpnr-fpga_interchange-experimental | |
| run: | | |
| chmod +x nextpnr-fpga_interchange-experimental | |
| mv nextpnr-fpga_interchange-experimental /usr/local/bin | |
| DEBIAN_FRONTEND=noninteractive apt update -qq | |
| DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ | |
| libtcl8.6 \ | |
| libboost-all-dev \ | |
| libcapnp-0.7.0 | |
| - name: Install | |
| run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
| - name: Run Test | |
| run: > | |
| source env.sh nextpnr && | |
| python3 exhaust.py | |
| --verbose | |
| --project ${{ matrix.project }} | |
| --toolchain ${{ matrix.toolchain }} | |
| --board ${{ matrix.board }} | |
| --build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
| --fail | |
| --timeout ${{ env.timeout }} | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| path: | | |
| **/results*.gz | |
| **/plot_*.svg | |
| NextpnrFPGAInterchangeExperimentalAlreadySynthesizedSingleThread: | |
| needs: | |
| - Matrix | |
| - Build_nextpnr-fpga_interchange-experimental-single-thread | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_experimental_already_synth_single_thread }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - name: Download nextpnr-fpga_interchange-experimental-single-thread | |
| uses: actions/download-artifact@v3 | |
| with: | |
| name: nextpnr-fpga_interchange-experimental-single-thread | |
| - name: Install nextpnr-fpga_interchange-experimental-single-thread | |
| run: | | |
| chmod +x nextpnr-fpga_interchange-experimental-single-thread | |
| mv nextpnr-fpga_interchange-experimental-single-thread /usr/local/bin | |
| DEBIAN_FRONTEND=noninteractive apt update -qq | |
| DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ | |
| libtcl8.6 \ | |
| libboost-all-dev \ | |
| libcapnp-0.7.0 | |
| - name: Install | |
| run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
| - name: Run Test | |
| run: > | |
| source env.sh nextpnr && | |
| python3 exhaust.py | |
| --verbose | |
| --project ${{ matrix.project }} | |
| --toolchain ${{ matrix.toolchain }} | |
| --board ${{ matrix.board }} | |
| --build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
| --fail | |
| --timeout ${{ env.timeout }} | |
| - uses: actions/upload-artifact@v3 | |
| if: ${{ always() }} | |
| with: | |
| path: | | |
| **/results*.gz | |
| **/plot_*.svg | |
| NextpnrIce40: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_ice40 }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| NextpnrNexus: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_nexus }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| NextpnrXilinx: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_xilinx }} | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| NextpnrXilinxFASM2Bels: | |
| needs: Matrix | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_xilinx_fasm2bels }} | |
| env: | |
| GHA_EXTERNAL_DISK: "tools" | |
| container: ubuntu:focal-20221130 | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| steps: | |
| - {uses: actions/checkout@v3, with: { submodules: recursive }} | |
| - uses: ./.github/RunTest | |
| with: | |
| environment: nextpnr | |
| project: ${{ matrix.project }} | |
| toolchain: ${{ matrix.toolchain }} | |
| board: ${{ matrix.board }} | |
| install: -v | |
| AggregateResults: | |
| # Restore all the others once this works | |
| needs: | |
| - VPR | |
| - VPRFASM2Bels | |
| - QuickLogic | |
| - Vivado | |
| - YosysVivado | |
| - YosysVivadoUHDM | |
| - VivadoAlreadySynthesized | |
| - NextpnrFPGAInterchange | |
| - NextpnrFPGAInterchangeAlreadySynthesized | |
| - NextpnrFPGAInterchangeExperimentalAlreadySynthesized | |
| - NextpnrFPGAInterchangeExperimentalAlreadySynthesizedSingleThread | |
| - NextpnrIce40 | |
| - NextpnrNexus | |
| - NextpnrXilinx | |
| - NextpnrXilinxFASM2Bels | |
| runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
| container: ubuntu:focal-20221130 | |
| env: | |
| GHA_SA: gh-sa-fpga-tool-perf-ci | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - name: Install prerequisites | |
| run: | | |
| apt update -qqy | |
| apt install -qqy curl python3 python3-pip gnupg2 | |
| echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list | |
| curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - | |
| apt update -qqy | |
| # TODO: remove version pindown after https://github.com/GoogleCloudPlatform/gsutil/issues/1663 has been resolved | |
| apt install -qqy google-cloud-cli=413.0.0-0 | |
| - uses: actions/download-artifact@v3 | |
| - name: Results Aggregation | |
| run: | | |
| find . -name "*.json.gz" | |
| mkdir upload | |
| python3 ./utils/aggregate_results.py artifact/build/_exhaust-runs upload | |
| - uses: actions/upload-artifact@v3 | |
| name: aggregated-results | |
| with: | |
| path: | | |
| **/results*.json.gz | |
| - name: Upload to GCP | |
| if: ${{ (github.event_name == 'push' || github.event_name == 'schedule') && | |
| github.ref == 'refs/heads/main' }} | |
| run: | | |
| gsutil -m cp upload/* gs://${{ secrets.GCP_STORAGE_BUCKET }} |