From 44bcf0f2c46598247075526dbb89988e2bdc1631 Mon Sep 17 00:00:00 2001 From: Vaclav Pavlin Date: Tue, 6 Jun 2023 17:41:41 +0200 Subject: [PATCH] chore(ci): properly set os and architecture for nightly and release (#1780) --- .github/workflows/nightly.yml | 17 ++++++++----- .github/workflows/release-assets.yml | 36 +++++++++++++++++++--------- Makefile | 4 ---- config.nims | 34 +++++++++++++++++++------- 4 files changed, 61 insertions(+), 30 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c96fd6766f..daf7a6a4ce 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -18,6 +18,11 @@ jobs: matrix: os: [ubuntu-latest, macos-latest] experimental: [true, false] + arch: [amd64] + include: + - os: macos-latest + experimental: false + arch: arm64 runs-on: ${{ matrix.os }} steps: - name: Checkout code @@ -34,20 +39,18 @@ jobs: path: | vendor/ .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + key: ${{ runner.os }}-${{matrix.arch}}-submodules-${{ steps.submodules.outputs.hash }} - name: prep variables id: vars run: | - ARCH=$(uname -m) EXPERIMENTAL=$([[ "${{ matrix.experimental }}" == "true" ]] && echo "-experimental" || echo "") - echo "arch=${ARCH}" >> $GITHUB_OUTPUT echo "experimental=${EXPERIMENTAL}" >> $GITHUB_OUTPUT - NWAKU_ARTIFACT_NAME=$(echo "nwaku${EXPERIMENTAL}-${ARCH}-${{runner.os}}.tar.gz" | tr "[:upper:]" "[:lower:]") - NWAKU_TOOLS_ARTIFACT_NAME=$(echo "nwaku-tools${EXPERIMENTAL}-${ARCH}-${{runner.os}}.tar.gz" | tr "[:upper:]" "[:lower:]") + NWAKU_ARTIFACT_NAME=$(echo "nwaku${EXPERIMENTAL}-${{matrix.arch}}-${{runner.os}}.tar.gz" | tr "[:upper:]" "[:lower:]") + NWAKU_TOOLS_ARTIFACT_NAME=$(echo "nwaku-tools${EXPERIMENTAL}-${{matrix.arch}}-${{runner.os}}.tar.gz" | tr "[:upper:]" "[:lower:]") echo "nwaku=${NWAKU_ARTIFACT_NAME}" >> $GITHUB_OUTPUT echo "nwakutools=${NWAKU_TOOLS_ARTIFACT_NAME}" >> $GITHUB_OUTPUT @@ -55,7 +58,9 @@ jobs: - name: build artifacts id: build run: | - make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false EXPERIMENTAL=${{matrix.experimental}} wakunode2 tools + OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux") + + make QUICK_AND_DIRTY_COMPILER=1 V=1 CI=false EXPERIMENTAL=${{matrix.experimental}} NIMFLAGS="-d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" wakunode2 tools tar -cvzf ${{steps.vars.outputs.nwaku}} ./build/wakunode2 tar -cvzf ${{steps.vars.outputs.nwakutools}} ./build/wakucanary ./build/networkmonitor diff --git a/.github/workflows/release-assets.yml b/.github/workflows/release-assets.yml index d217b4ed33..dde8c744cd 100644 --- a/.github/workflows/release-assets.yml +++ b/.github/workflows/release-assets.yml @@ -13,12 +13,17 @@ jobs: matrix: env: - { NPROC: 2 } - platform: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.platform }} + os: [ubuntu-latest, macos-latest] + arch: [amd64] + include: + - os: macos-latest + experimental: false + arch: arm64 + runs-on: ${{ matrix.os }} env: ${{ matrix.env }} timeout-minutes: 60 - name: ${{ matrix.platform }} - ${{ matrix.env.NPROC }} processes + name: ${{ matrix.os }} - ${{ matrix.env.NPROC }} processes steps: - name: Checkout code @@ -35,19 +40,28 @@ jobs: path: | vendor/ .git/modules - key: ${{ runner.os }}-vendor-modules-${{ steps.submodules.outputs.hash }} + key: ${{ runner.os }}-${{matrix.arch}}-submodules-${{ steps.submodules.outputs.hash }} + + - name: prep variables + id: vars + run: | + NWAKU_ARTIFACT_NAME=$(echo "nwaku${EXPERIMENTAL}-${{matrix.arch}}-${{runner.os}}.tar.gz" | tr "[:upper:]" "[:lower:]") + + echo "nwaku=${NWAKU_ARTIFACT_NAME}" >> $GITHUB_OUTPUT - name: Install dependencies run: | - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" V=1 update - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" CI=false wakunode1 - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" CI=false wakunode2 - make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC}" CI=false chat2 - tar -cvzf nim-waku-${{ matrix.platform }}.tar.gz ./build/ + OS=$([[ "${{runner.os}}" == "macOS" ]] && echo "macosx" || echo "linux") + + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" V=1 update + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" CI=false wakunode1 + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" CI=false wakunode2 + make -j${NPROC} NIMFLAGS="--parallelBuild:${NPROC} -d:disableMarchNative --os:${OS} --cpu:${{matrix.arch}}" CI=false chat2 + tar -cvzf ${{steps.vars.outputs.nwaku}} ./build/ - name: Upload asset uses: actions/upload-artifact@v2.2.3 with: - name: nim-waku-${{ matrix.platform }}.tar.gz - path: nim-waku-${{ matrix.platform }}.tar.gz + name: ${{steps.vars.outputs.nwaku}} + path: ${{steps.vars.outputs.nwaku}} if-no-files-found: error diff --git a/Makefile b/Makefile index a210303d2b..131718d245 100644 --- a/Makefile +++ b/Makefile @@ -64,10 +64,6 @@ TARGET ?= prod GIT_VERSION ?= $(shell git describe --abbrev=6 --always --tags) NIM_PARAMS := $(NIM_PARAMS) -d:git_version=\"$(GIT_VERSION)\" -## Pass CPU architecture to C compiler, use basic x86-64 instruction set by default -ARCHITECTURE ?= "x86-64" -NIM_PARAMS := $(NIM_PARAMS) --passC:\"-march=$(ARCHITECTURE)\" - ## Heaptracker options HEAPTRACKER ?= 0 HEAPTRACKER_INJECT ?= 0 diff --git a/config.nims b/config.nims index 2ca66c398f..65f42538dc 100644 --- a/config.nims +++ b/config.nims @@ -18,19 +18,34 @@ if defined(windows): # because these require direct manipulations of the stdout File object. switch("define", "chronicles_colors=off") -# This helps especially for 32-bit x86, which sans SSE2 and newer instructions -# requires quite roundabout code generation for cryptography, and other 64-bit -# and larger arithmetic use cases, along with register starvation issues. When -# engineering a more portable binary release, this should be tweaked but still -# use at least -msse2 or -msse3. +# https://github.com/status-im/nimbus-eth2/blob/stable/docs/cpu_features.md#ssse3-supplemental-sse3 +# suggests that SHA256 hashing with SSSE3 is 20% faster than without SSSE3, so +# given its near-ubiquity in the x86 installed base, it renders a distribution +# build more viable on an overall broader range of hardware. +# if defined(disableMarchNative): if defined(i386) or defined(amd64): if defined(macosx): + # macOS Catalina is EOL as of 2022-09 + # https://support.apple.com/kb/sp833 + # "macOS Big Sur - Technical Specifications" lists current oldest + # supported models: MacBook (2015 or later), MacBook Air (2013 or later), + # MacBook Pro (Late 2013 or later), Mac mini (2014 or later), iMac (2014 + # or later), iMac Pro (2017 or later), Mac Pro (2013 or later). + # + # These all have Haswell or newer CPUs. + # + # This ensures AVX2, AES-NI, PCLMUL, BMI1, and BMI2 instruction set support. switch("passC", "-march=haswell -mtune=generic") switch("passL", "-march=haswell -mtune=generic") else: - switch("passC", "-msse3") - switch("passL", "-msse3") + if defined(marchOptimized): + # https://github.com/status-im/nimbus-eth2/blob/stable/docs/cpu_features.md#bmi2--adx + switch("passC", "-march=broadwell -mtune=generic") + switch("passL", "-march=broadwell -mtune=generic") + else: + switch("passC", "-mssse3") + switch("passL", "-mssse3") elif defined(macosx) and defined(arm64): # Apple's Clang can't handle "-march=native" on M1: https://github.com/status-im/nimbus-eth2/issues/2758 switch("passC", "-mcpu=apple-m1") @@ -41,8 +56,9 @@ else: if defined(windows): # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=65782 # ("-fno-asynchronous-unwind-tables" breaks Nim's exception raising, sometimes) - switch("passC", "-mno-avx512vl") - switch("passL", "-mno-avx512vl") + switch("passC", "-mno-avx512f") + switch("passL", "-mno-avx512f") + --threads:on --opt:speed