diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 000000000000..88774ec180b9 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,30 @@ +[alias] +b = "build" +c = "check" +d = "doc" +t = "test" +r = "run" + +[build] +incremental = true +# might be helpful but we don't enforce sccache installation atm +# https://github.com/mozilla/sccache +# rustc-wrapper = "sccache" + +# For details checkout https://jondot.medium.com/8-steps-for-troubleshooting-your-rust-build-times-2ffc965fd13e +[target.x86_64-unknown-linux-gnu] +# clang has been listed as prerequisite in the doc +linker = "clang" +# enable avx2 by default since it's avaiable on almost all x86_64 CPUs +rustflags = ["-Clink-arg=-fuse-ld=lld", "-Ctarget-feature=+avx2,+fma", "-Zshare-generics=y"] + +[target.x86_64-apple-darwin] +# zld might help here +# brew install michaeleisel/zld/zld +# "-Clink-arg=-fuse-ld=zld" +# For details checkout https://jondot.medium.com/8-steps-for-troubleshooting-your-rust-build-times-2ffc965fd13e +rustflags = ["-Ctarget-feature=+avx2,+fma", "-Zshare-generics=y"] + +[net] +git-fetch-with-cli = true +retry = 5 diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 1fd638793893..dcfb5a455c01 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -11,22 +11,25 @@ jobs: runs-on: buildjet-8vcpu-ubuntu-2004 timeout-minutes: 30 steps: - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - name: Checkout Sources uses: actions/checkout@v3 with: submodules: true - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: 5 - continue-on-error: true + - name: Apt Dependencies + run: sudo make install-deps + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-codecov-${{ hashFiles('rust-toolchain.toml') }} + cache-update: ${{ github.event_name != 'pull_request' }} - name: Run cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov - name: Generate code coverage run: cargo llvm-cov --workspace --lcov --output-path lcov.info + env: + CC: "sccache clang" + CXX: "sccache clang++" - name: Upload CodeCov uses: codecov/codecov-action@v3 with: diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 4be832ac617c..a02827e1d9aa 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -17,28 +17,44 @@ jobs: name: Publish runs-on: ubuntu-latest steps: - - name: Apt Dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - name: Checkout Sources uses: actions/checkout@v3 + - name: Apt Dependencies + run: sudo make install-deps - name: Link Checker (Repo) uses: lycheeverse/lychee-action@v1.5.1 with: args: --verbose --no-progress 'README.md' # Fail action on broken links fail: true - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: 5 - continue-on-error: true + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + key: ${{ runner.os }}-docs-cargo-bin-${{ hashFiles('rust-toolchain.toml') }} - name: Install MDBook run: cargo install mdbook mdbook-linkcheck --locked + env: + RUSTFLAGS: "-Cstrip=symbols" + continue-on-error: true + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-docs-${{ hashFiles('rust-toolchain.toml') }} + # true: always save cache + # false: only save when cache is not found + # this option is only available in the forked action + # the original action has no activities for years so we + # just keep using the fork + cache-update: ${{ github.event_name != 'pull_request' }} - name: Execute MDBook run: make mdbook-build - name: Build rustdoc run: make rustdoc + env: + CC: "sccache clang" + CXX: "sccache clang++" - name: Link Checker (Docs) uses: lycheeverse/lychee-action@v1.5.1 with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index bba16dfebbee..68f106696943 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,17 +17,15 @@ jobs: - os: macos-latest file: forest-${{ github.ref_name }}-macos-amd64.zip steps: + - name: Checkout Sources + uses: actions/checkout@v3 - name: Apt Dependencies if: startsWith(matrix.os, 'Ubuntu') - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev + run: sudo make install-deps - name: Homebrew Utils if: startsWith(matrix.os, 'macOS') run: | brew install --verbose coreutils - - name: Checkout Sources - uses: actions/checkout@v3 - name: Rust Cache uses: Swatinem/rust-cache@v2 timeout-minutes: 5 diff --git a/.github/workflows/rust.yml b/.github/workflows/rust.yml index 8bd2347093dd..b846b3f17c07 100644 --- a/.github/workflows/rust.yml +++ b/.github/workflows/rust.yml @@ -1,6 +1,7 @@ name: Rust on: + workflow_dispatch: pull_request: branches: - main @@ -14,96 +15,55 @@ env: CI: 1 CARGO_INCREMENTAL: 1 CACHE_TIMEOUT_MINUTES: 5 - RUSTFLAGS: -Ctarget-feature=+avx2,+fma jobs: test: name: Test runs-on: buildjet-8vcpu-ubuntu-2004 timeout-minutes: 30 - env: - RUSTFLAGS: "-Ctarget-feature=+avx2,+fma -D warnings" steps: - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - name: Checkout Sources uses: actions/checkout@v3 + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-test-${{ hashFiles('rust-toolchain.toml') }} + cache-update: ${{ github.event_name != 'pull_request' }} + - name: Apt Dependencies + run: sudo make install-deps - name: install nextest uses: taiki-e/install-action@nextest - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true - - name: Cargo Check - run: cargo check - - name: Make Test-All - run: make test-all - - lint: - name: Lint - runs-on: ubuntu-latest - steps: - - name: Apt Dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - - name: Checkout Sources - uses: actions/checkout@v3 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true - - name: Install taplo (TOML linter) - run: cargo install taplo-cli --locked - - name: Run Linters - run: make lint - - audit: - name: Audit - runs-on: ubuntu-latest - steps: - - name: Checkout Sources - uses: actions/checkout@v3 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true - - name: Install Audit - run: cargo install cargo-audit --locked - - name: Run Audit - run: make audit - - unused_dependencies: - name: Unused dependencies - runs-on: ubuntu-latest - steps: - - name: Checkout Sources - uses: actions/checkout@v3 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true - - name: Install udeps - run: cargo install cargo-udeps --locked - - name: Run udeps - run: make udeps + - run: make test-all + env: + CC: "sccache clang" + CXX: "sccache clang++" - spellcheck: - name: Spellcheck + lint-all: + name: All lint checks (lint audit spellcheck udeps) runs-on: ubuntu-latest steps: - name: Checkout Sources uses: actions/checkout@v3 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} + - name: Apt Dependencies + run: sudo make install-deps + - uses: actions/cache@v3 + with: + path: | + ~/.cargo/bin/ + key: ${{ runner.os }}-lints-cargo-bin-${{ hashFiles('rust-toolchain.toml') }} + - run: make install-lint-tools continue-on-error: true - - name: Install spellcheck - run: cargo install cargo-spellcheck --locked - - name: Run Spellcheck - run: make spellcheck + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-lints-${{ hashFiles('rust-toolchain.toml') }} + cache-update: ${{ github.event_name != 'pull_request' }} + - run: make lint-all + env: + CC: "sccache clang" + CXX: "sccache clang++" build: name: Build @@ -111,22 +71,32 @@ jobs: strategy: matrix: os: [ubuntu-latest, macos-latest] - #rv: [1.58.1, stable, beta, nightly] - rv: [nightly] steps: - - name: Install Dependencies - if: startsWith(matrix.os, 'Ubuntu') - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - name: Checkout Sources uses: actions/checkout@v3 - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true - - name: Cargo Build - run: cargo build --profile dev + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + # hard code release-name for macos, it always get rate limited when calling github api + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-${{ hashFiles('rust-toolchain.toml') }} + cache-update: ${{ github.event_name != 'pull_request' }} + - name: Install Apt Dependencies + if: startsWith(matrix.os, 'Ubuntu') + run: | + lscpu # the job may run on different CPUs, list cpu here for analysing build time + sudo make install-deps + - name: Cargo Check + run: cargo check --timings + env: + CC: "sccache clang" + CXX: "sccache clang++" + - uses: actions/upload-artifact@v3 + with: + name: build-timings + path: | + target/cargo-timings/* + if-no-files-found: error calibnet-check: name: Calibnet sync check @@ -134,20 +104,23 @@ jobs: runs-on: buildjet-8vcpu-ubuntu-2004 timeout-minutes: 30 steps: - - name: Install Dependencies - run: | - sudo apt-get update - sudo apt-get install -y build-essential clang ocl-icd-opencl-dev - name: Checkout Sources uses: actions/checkout@v3 with: submodules: true - - name: Rust Cache - uses: Swatinem/rust-cache@v2 - timeout-minutes: ${{ fromJSON(env.CACHE_TIMEOUT_MINUTES) }} - continue-on-error: true + - name: Apt Dependencies + run: sudo make install-deps + - name: Setup sccache + uses: hanabi1224/sccache-action@v1.2.0 + with: + release-name: v0.3.0 + cache-key: ${{ runner.os }}-sccache-calibnet-check-${{ hashFiles('rust-toolchain.toml') }} + cache-update: ${{ github.event_name != 'pull_request' }} - name: build and install binaries run: make install + env: + CC: "sccache clang" + CXX: "sccache clang++" - name: download snapshot run: forest-cli --chain calibnet chain fetch -s /tmp/snapshots/ - name: import snapshot and run Forest diff --git a/.gitignore b/.gitignore index 388ae9912a7b..bf9879be92a0 100644 --- a/.gitignore +++ b/.gitignore @@ -9,6 +9,7 @@ config*.toml *.orig criterion genesis.json -/.cargo +/.cargo/* +!/.cargo/*.toml node/rpc-api/static/ast.ron documentation/book/**/* \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 6a86de3157be..3ecc05290543 100644 --- a/Dockerfile +++ b/Dockerfile @@ -20,14 +20,13 @@ FROM buildpack-deps:buster AS build-env # Install dependencies -RUN apt-get update && apt-get install --no-install-recommends -y build-essential clang ocl-icd-opencl-dev cmake ca-certificates curl +RUN apt-get update && apt-get install --no-install-recommends -y build-essential clang lld ocl-icd-opencl-dev cmake ca-certificates curl RUN update-ca-certificates # Install rustup # https://rustup.rs/ RUN curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | sh -s -- -y ENV PATH="/root/.cargo/bin:${PATH}" -ENV RUSTFLAGS="-Ctarget-feature=+avx2,+fma" WORKDIR /usr/src/forest COPY . . diff --git a/Makefile b/Makefile index 152429cb3d83..61ac80760a13 100644 --- a/Makefile +++ b/Makefile @@ -8,6 +8,13 @@ install-daemon: install: install-cli install-daemon +install-deps: + apt-get update -y + apt-get install --no-install-recommends -y build-essential clang lld ocl-icd-opencl-dev cmake + +install-lint-tools: + RUSTFLAGS="-Cstrip=symbols" cargo install --locked taplo-cli cargo-audit cargo-spellcheck cargo-udeps + clean-all: cargo clean @@ -39,7 +46,7 @@ clean: @echo "Done cleaning." # Lints with everything we have in our CI arsenal -lint-all: lint audit udeps spellcheck +lint-all: lint audit spellcheck udeps audit: cargo audit --ignore RUSTSEC-2020-0071 --ignore RUSTSEC-2022-0040 @@ -114,4 +121,4 @@ mdbook-build: rustdoc: cargo doc --workspace --all-features --no-deps -.PHONY: clean clean-all lint build release test test-all test-release license test-vectors run-vectors pull-serialization-tests install-cli install-daemon install docs run-serialization-vectors rustdoc +.PHONY: clean clean-all lint build release test test-all test-release license test-vectors run-vectors pull-serialization-tests install-cli install-daemon install install-deps install-lint-tools docs run-serialization-vectors rustdoc diff --git a/README.md b/README.md index 776c045112aa..567cae3d9d6e 100644 --- a/README.md +++ b/README.md @@ -66,6 +66,8 @@ rustup install nightly ```shell # Ubuntu +sudo make install-deps +# Or sudo apt install build-essential clang ocl-icd-opencl-dev libssl-dev # Archlinux @@ -82,15 +84,10 @@ git clone --recursive https://github.com/chainsafe/forest cd forest # Install binary to $HOME/.cargo/bin and run node +# avx2 is enabled by default. To disable it, remove +# -Ctarget-feature from RUSTFLAGS in .cargo/config.toml make install -# Simd is supported by many crypto / hashing crates -# Install with avx2 cpu features -RUSTFLAGS="-Ctarget-feature=+avx2,+fma" make install - -# Or install with local cpu features -RUSTFLAGS="-Ctarget-cpu=native" make install - forest ``` diff --git a/taplo.toml b/taplo.toml index 4d8d77f351c3..f6d3a3ef63f2 100644 --- a/taplo.toml +++ b/taplo.toml @@ -1,7 +1,8 @@ [formatting] -align_entries = true -column_width = 120 -array_auto_expand = true +align_entries = true +column_width = 120 +array_auto_expand = true +allowed_blank_lines = 1 [[rule]] keys = ["dependencies", "dev-dependencies", "toolchain", "workspace.dependencies"]