Full Test Suite #5311
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
name: Full Test Suite | |
on: | |
workflow_call: | |
secrets: | |
GCP_STORAGE_BUCKET: | |
description: 'GCP STORAGE_BUCKET.' | |
required: false | |
schedule: | |
- cron: '0 */4 * * *' | |
env: | |
timeout: 7200 | |
GHA_MACHINE_TYPE: "n2-standard-4" | |
defaults: | |
run: | |
shell: bash | |
jobs: | |
Matrix: | |
runs-on: ubuntu-latest | |
outputs: | |
matrices: ${{ steps.generate.outputs.matrices }} | |
link: ${{ steps.link.outputs.link }} | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- name: Setup Python | |
uses: actions/setup-python@v4 | |
- name: RW Jars link | |
id: link | |
run: | | |
sudo apt update | |
sudo apt install -y curl wget | |
./.github/scripts/get_rapidwright_link.sh | tee -a $GITHUB_OUTPUT | |
- name: Generate Matrix | |
id: generate | |
run: | | |
python3 -m pip install -r conf/requirements.txt | |
PYTHONPATH=$(pwd) python3 ./.github/scripts/generate_job_matrix.py all | |
Build_nextpnr-fpga_interchange-experimental: | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- name: Build Tools | |
run: | | |
./.github/scripts/build_nextpnr_fpga_interchange.sh | |
cp ./third_party/nextpnr/nextpnr-fpga_interchange nextpnr-fpga_interchange-experimental | |
- name: Upload nextpnr-fpga_interchange-experimental | |
uses: actions/upload-artifact@v3 | |
if: always() | |
with: | |
name: nextpnr-fpga_interchange-experimental | |
path: nextpnr-fpga_interchange-experimental | |
Build_nextpnr-fpga_interchange-experimental-single-thread: | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- name: Build Tools | |
run: | | |
./.github/scripts/build_nextpnr_fpga_interchange.sh single_thread | |
cp ./third_party/nextpnr/nextpnr-fpga_interchange nextpnr-fpga_interchange-experimental-single-thread | |
- name: Upload nextpnr-fpga_interchange-experimental-single-thread | |
uses: actions/upload-artifact@v3 | |
if: always() | |
with: | |
name: nextpnr-fpga_interchange-experimental-single-thread | |
path: nextpnr-fpga_interchange-experimental-single-thread | |
# Symbiflow tests | |
VPR: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).vpr }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: xilinx-a35t | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
VPRFASM2Bels: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).vpr_fasm2bels }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: xilinx-a35t | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v | |
# Quicklogic tests | |
QuickLogic: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).quicklogic }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: quicklogic | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
# Vivado | |
Vivado: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).vivado }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: xilinx-a35t | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v | |
YosysVivado: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).yosys_vivado }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: xilinx-a35t | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v | |
YosysVivadoUHDM: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).yosys_vivado_uhdm }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: xilinx-a35t | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v | |
VivadoAlreadySynthesized: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).vivado_already_synth }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- uses: actions/checkout@v3 | |
with: | |
submodules: recursive | |
- name: Install | |
run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
- name: Run Test | |
run: > | |
source env.sh && | |
python3 exhaust.py | |
--verbose | |
--project ${{ matrix.project }} | |
--toolchain ${{ matrix.toolchain }} | |
--board ${{ matrix.board }} | |
--build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
--fail | |
--timeout ${{ env.timeout }} | |
- uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
path: | | |
**/results*.gz | |
**/plot_*.svg | |
# Nextpnr | |
NextpnrFPGAInterchange: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v -l $(echo ${{needs.Matrix.outputs.link}}) | |
NextpnrFPGAInterchangeAlreadySynthesized: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_already_synth }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v -l $(echo ${{needs.Matrix.outputs.link}}) | |
NextpnrFPGAInterchangeExperimentalAlreadySynthesized: | |
needs: | |
- Matrix | |
- Build_nextpnr-fpga_interchange-experimental | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_experimental_already_synth }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- name: Download nextpnr-fpga_interchange-experimental | |
uses: actions/download-artifact@v3 | |
with: | |
name: nextpnr-fpga_interchange-experimental | |
- name: Install nextpnr-fpga_interchange-experimental | |
run: | | |
chmod +x nextpnr-fpga_interchange-experimental | |
mv nextpnr-fpga_interchange-experimental /usr/local/bin | |
DEBIAN_FRONTEND=noninteractive apt update -qq | |
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ | |
libtcl8.6 \ | |
libboost-all-dev \ | |
libcapnp-0.7.0 | |
- name: Install | |
run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
- name: Run Test | |
run: > | |
source env.sh nextpnr && | |
python3 exhaust.py | |
--verbose | |
--project ${{ matrix.project }} | |
--toolchain ${{ matrix.toolchain }} | |
--board ${{ matrix.board }} | |
--build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
--fail | |
--timeout ${{ env.timeout }} | |
- uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
path: | | |
**/results*.gz | |
**/plot_*.svg | |
NextpnrFPGAInterchangeExperimentalAlreadySynthesizedSingleThread: | |
needs: | |
- Matrix | |
- Build_nextpnr-fpga_interchange-experimental-single-thread | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_fpga_interchange_experimental_already_synth_single_thread }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- name: Download nextpnr-fpga_interchange-experimental-single-thread | |
uses: actions/download-artifact@v3 | |
with: | |
name: nextpnr-fpga_interchange-experimental-single-thread | |
- name: Install nextpnr-fpga_interchange-experimental-single-thread | |
run: | | |
chmod +x nextpnr-fpga_interchange-experimental-single-thread | |
mv nextpnr-fpga_interchange-experimental-single-thread /usr/local/bin | |
DEBIAN_FRONTEND=noninteractive apt update -qq | |
DEBIAN_FRONTEND=noninteractive apt install -y --no-install-recommends \ | |
libtcl8.6 \ | |
libboost-all-dev \ | |
libcapnp-0.7.0 | |
- name: Install | |
run: ./.github/scripts/install.sh -v -l $(echo ${{needs.Matrix.outputs.link}}) -t ${{ matrix.toolchain }} -b ${{ matrix.board }} | |
- name: Run Test | |
run: > | |
source env.sh nextpnr && | |
python3 exhaust.py | |
--verbose | |
--project ${{ matrix.project }} | |
--toolchain ${{ matrix.toolchain }} | |
--board ${{ matrix.board }} | |
--build_type ${{ matrix.project }}-${{ matrix.toolchain }}-${{ matrix.board }} | |
--fail | |
--timeout ${{ env.timeout }} | |
- uses: actions/upload-artifact@v3 | |
if: ${{ always() }} | |
with: | |
path: | | |
**/results*.gz | |
**/plot_*.svg | |
NextpnrIce40: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_ice40 }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
NextpnrNexus: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_nexus }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
NextpnrXilinx: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_xilinx }} | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
NextpnrXilinxFASM2Bels: | |
needs: Matrix | |
strategy: | |
fail-fast: false | |
matrix: | |
include: ${{ fromJson(needs.Matrix.outputs.matrices).nextpnr_xilinx_fasm2bels }} | |
env: | |
GHA_EXTERNAL_DISK: "tools" | |
container: ubuntu:focal-20221130 | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
steps: | |
- {uses: actions/checkout@v3, with: { submodules: recursive }} | |
- uses: ./.github/RunTest | |
with: | |
environment: nextpnr | |
project: ${{ matrix.project }} | |
toolchain: ${{ matrix.toolchain }} | |
board: ${{ matrix.board }} | |
install: -v | |
AggregateResults: | |
# Restore all the others once this works | |
needs: | |
- VPR | |
- VPRFASM2Bels | |
- QuickLogic | |
- Vivado | |
- YosysVivado | |
- YosysVivadoUHDM | |
- VivadoAlreadySynthesized | |
- NextpnrFPGAInterchange | |
- NextpnrFPGAInterchangeAlreadySynthesized | |
- NextpnrFPGAInterchangeExperimentalAlreadySynthesized | |
- NextpnrFPGAInterchangeExperimentalAlreadySynthesizedSingleThread | |
- NextpnrIce40 | |
- NextpnrNexus | |
- NextpnrXilinx | |
- NextpnrXilinxFASM2Bels | |
runs-on: [self-hosted, Linux, X64, gcp-custom-runners] | |
container: ubuntu:focal-20221130 | |
env: | |
GHA_SA: gh-sa-fpga-tool-perf-ci | |
steps: | |
- uses: actions/checkout@v3 | |
- name: Install prerequisites | |
run: | | |
apt update -qqy | |
apt install -qqy curl python3 python3-pip gnupg2 | |
echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] https://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list | |
curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg add - | |
apt update -qqy | |
# TODO: remove version pindown after https://github.com/GoogleCloudPlatform/gsutil/issues/1663 has been resolved | |
apt install -qqy google-cloud-cli=413.0.0-0 | |
- uses: actions/download-artifact@v3 | |
- name: Results Aggregation | |
run: | | |
find . -name "*.json.gz" | |
mkdir upload | |
python3 ./utils/aggregate_results.py artifact/build/_exhaust-runs upload | |
- uses: actions/upload-artifact@v3 | |
name: aggregated-results | |
with: | |
path: | | |
**/results*.json.gz | |
- name: Upload to GCP | |
if: ${{ (github.event_name == 'push' || github.event_name == 'schedule') && | |
github.ref == 'refs/heads/main' }} | |
run: | | |
gsutil -m cp upload/* gs://${{ secrets.GCP_STORAGE_BUCKET }} |