Skip to content

[ci] Roll pinned nightly toolchain #4018

[ci] Roll pinned nightly toolchain

[ci] Roll pinned nightly toolchain #4018

Workflow file for this run

# Copyright 2022 The Fuchsia Authors
#
# Licensed under a BSD-style license <LICENSE-BSD>, Apache License, Version 2.0
# <LICENSE-APACHE or https://www.apache.org/licenses/LICENSE-2.0>, or the MIT
# license <LICENSE-MIT or https://opensource.org/licenses/MIT>, at your option.
# This file may not be copied, modified, or distributed except according to
# those terms.
name: Build & Tests
on:
pull_request:
push:
branches:
- main
- v0.6.x
merge_group:
permissions: read-all
env:
CARGO_TERM_COLOR: always
RUSTFLAGS: -Dwarnings
RUSTDOCFLAGS: -Dwarnings
# `ZC_NIGHTLY_XXX` are flags that we add to `XXX` only on the nightly
# toolchain.
ZC_NIGHTLY_RUSTFLAGS: -Zrandomize-layout
ZC_NIGHTLY_MIRIFLAGS: "-Zmiri-strict-provenance -Zmiri-backtrace=full"
jobs:
build_test:
runs-on: ubuntu-latest
# Generate and populate the global Cargo registry and cache first. Each
# job in the matrix runs in parallel, so without populating the cache
# first, most jobs would duplicate the work of downloading crates from
# the internet. Populating the cache first ensures that this work only
# happens once.
needs: generate_cache
strategy:
# By default, this is set to `true`, which means that a single CI job
# failure will cause all outstanding jobs to be canceled. This slows down
# development because it means that errors need to be encountered and
# fixed one at a time.
fail-fast: false
matrix:
# See `INTERNAL.md` for an explanation of these pinned toolchain
# versions.
toolchain: [ "msrv", "stable", "nightly", "zerocopy-generic-bounds-in-const-fn", "zerocopy-aarch64-simd", "zerocopy-panic-in-const", ]
target: [
"i686-unknown-linux-gnu",
"x86_64-unknown-linux-gnu",
"arm-unknown-linux-gnueabi",
"aarch64-unknown-linux-gnu",
"powerpc-unknown-linux-gnu",
"powerpc64-unknown-linux-gnu",
"riscv64gc-unknown-linux-gnu",
"s390x-unknown-linux-gnu",
"wasm32-wasi"
]
features: [ "--no-default-features", "", "--features __internal_use_only_features_that_work_on_stable", "--all-features" ]
crate: [ "zerocopy", "zerocopy-derive" ]
event_name: [ "${{ github.event_name }}" ]
exclude:
# Exclude any combination which uses a non-nightly toolchain but
# enables nightly features.
- toolchain: "msrv"
features: "--all-features"
- toolchain: "stable"
features: "--all-features"
- toolchain: "zerocopy-generic-bounds-in-const-fn"
features: "--all-features"
- toolchain: "zerocopy-aarch64-simd"
features: "--all-features"
- toolchain: "zerocopy-panic-in-const"
features: "--all-features"
# Exclude any combination for the zerocopy-derive crate which
# uses zerocopy features.
- crate: "zerocopy-derive"
features: "--no-default-features"
- crate: "zerocopy-derive"
features: "--features __internal_use_only_features_that_work_on_stable"
- crate: "zerocopy-derive"
features: "--all-features"
# Exclue any combination of zerocopy-derive and any toolchain version
# other than "msrv", "stable", and "nightly". These other versions
# exist to exercise zerocopy behavior which differs by toolchain;
# zerocopy-derive doesn't behave different on these toolchains.
- crate: "zerocopy-derive"
toolchain: "zerocopy-generic-bounds-in-const-fn"
- crate: "zerocopy-derive"
toolchain: "zerocopy-aarch64-simd"
- crate: "zerocopy-derive"
toolchain: "zerocopy-panic-in-const"
# Exclude most targets during PR development, but allow them in the
# merge queue. This speeds up our development flow, while still
# ensuring that errors on these targets are caught before a PR is
# merged to main.
- target: "arm-unknown-linux-gnueabi"
event_name: "pull_request"
- target: "aarch64-unknown-linux-gnu"
event_name: "pull_request"
- target: "powerpc-unknown-linux-gnu"
event_name: "pull_request"
- target: "powerpc64-unknown-linux-gnu"
event_name: "pull_request"
- target: "riscv64gc-unknown-linux-gnu"
event_name: "pull_request"
- target: "s390x-unknown-linux-gnu"
event_name: "pull_request"
- target: "wasm32-wasi"
event_name: "pull_request"
name: Build & Test (${{ matrix.crate }} / ${{ matrix.toolchain }} / ${{ matrix.features }} / ${{ matrix.target }})
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Populate cache
uses: ./.github/actions/cache
- name: Configure environment variables
run: |
set -eo pipefail
# We use toolchain descriptors ("msrv", "stable", "nightly", and values
# from the "metadata.build-rs" key in Cargo.toml) in the matrix. This
# step converts the current descriptor to a particular toolchain version
# by looking up the corresponding key in `Cargo.toml`. It sets the
# `ZC_TOOLCHAIN` environment variable for use in the next step
# (toolchain installation) because GitHub variable interpolation doesn't
# support running arbitrary commands. In other words, we can't rewrite:
#
# toolchain: $ {{ env.ZC_TOOLCHAIN }}
#
# ...to:
#
# toolchain: $ {{ ./cargo.sh --version matrix.toolchain }} # hypothetical syntax
ZC_TOOLCHAIN="$(./cargo.sh --version ${{ matrix.toolchain }})"
echo "Found that the '${{ matrix.toolchain }}' toolchain is $ZC_TOOLCHAIN" | tee -a $GITHUB_STEP_SUMMARY
echo "ZC_TOOLCHAIN=$ZC_TOOLCHAIN" >> $GITHUB_ENV
if [[ '${{ matrix.toolchain }}' == 'nightly' ]]; then
RUSTFLAGS="$RUSTFLAGS $ZC_NIGHTLY_RUSTFLAGS"
MIRIFLAGS="$MIRIFLAGS $ZC_NIGHTLY_MIRIFLAGS"
echo "Using nightly toolchain; setting RUSTFLAGS='$RUSTFLAGS' and MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY
echo "RUSTFLAGS=$RUSTFLAGS" >> $GITHUB_ENV
echo "MIRIFLAGS=$MIRIFLAGS" >> $GITHUB_ENV
else
echo "Using non-nightly toolchain; not modifying RUSTFLAGS='$RUSTFLAGS' or MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY
fi
# On our MSRV, `cargo` does not know about the `rust-version` field. As a
# result, in `cargo.sh`, if we use our MSRV toolchain in order to run `cargo
# metadata`, we will not be able to extract the `rust-version` field. Thus,
# in `cargo.sh`, we explicitly do `cargo +stable metadata`. This requires a
# (more recent) stable toolchain to be installed. As of this writing, this
# toolchain is not used for anything else.
- name: Install stable Rust for use in 'cargo.sh'
uses: dtolnay/rust-toolchain@00b49be78f40fba4e87296b2ead62868750bdd83 # stable
with:
toolchain: stable
- name: Install Rust with ${{ matrix.toolchain }} toolchain (${{ env.ZC_TOOLCHAIN }}) and target ${{ matrix.target }}
uses: dtolnay/rust-toolchain@00b49be78f40fba4e87296b2ead62868750bdd83 # stable
with:
toolchain: ${{ env.ZC_TOOLCHAIN }}
targets: ${{ matrix.target }}
# We require the `rust-src` component to ensure that the compiler
# error output generated during UI tests matches that generated on
# local developer machines; see
# https://github.com/rust-lang/rust/issues/116433.
#
# Only nightly has a working Miri, so we skip installing on all other
# toolchains. This expression is effectively a ternary expression -
# see [1] for details.
#
# [1] https://github.com/actions/runner/issues/409#issuecomment-752775072
components: clippy, rust-src ${{ matrix.toolchain == 'nightly' && ', miri' || '' }}
- name: Rust Cache
uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
with:
key: "${{ matrix.target }}"
- name: Check tests
run: ./cargo.sh +${{ matrix.toolchain }} check --tests --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
- name: Build
run: ./cargo.sh +${{ matrix.toolchain }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
# When building tests for the i686 target, we need certain libraries which
# are not installed by default; `gcc-multilib` includes these libraries.
- name: Install gcc-multilib
# Per [1]:
#
# Note: Always run `sudo apt-get update` before installing a package. In
# case the `apt` index is stale, this command fetches and re-indexes any
# available packages, which helps prevent package installation failures.
#
# [1] https://docs.github.com/en/actions/using-github-hosted-runners/customizing-github-hosted-runners
run: |
sudo apt-get update
sudo apt-get install gcc-multilib
if: contains(matrix.target, 'i686')
- name: Run tests
run: |
./cargo.sh +${{ matrix.toolchain }} test \
--package ${{ matrix.crate }} \
--target ${{ matrix.target }} \
${{ matrix.features }} \
--verbose \
-- \
--skip ui
# Only run tests when targetting x86 (32- or 64-bit) - we're executing on
# x86_64, so we can't run tests for any non-x86 target.
#
# TODO(https://github.com/dtolnay/trybuild/issues/184#issuecomment-1269097742):
# Run compile tests when building for other targets.
if: contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')
- name: Run UI tests
run: |
# Run UI tests separately, treating warnings as warnings (rather than
# as errors, as we do everywhere else in our CI tests). This allows
# our UI tests to more accurately reflect what users will see, and
# also ensures that we're not spuriously relying on warnings being
# errors to ensure compilation failure (if we were, then our code
# would be unsound whenever -Dwarnings is not enabled).
#
# TODO(#560), TODO(#187): Once we migrate to the ui-test crate, we
# likely won't have to special-case the UI tests like this.
RUSTFLAGS="$RUSTFLAGS -Wwarnings" ./cargo.sh +${{ matrix.toolchain }} test \
--package ${{ matrix.crate }} \
--target ${{ matrix.target }} \
${{ matrix.features }} \
--verbose \
ui
# Only run tests when targetting x86 (32- or 64-bit) - we're executing on
# x86_64, so we can't run tests for any non-x86 target.
#
# TODO(https://github.com/dtolnay/trybuild/issues/184#issuecomment-1269097742):
# Run compile tests when building for other targets.
#
# Only run UI tests for zerocopy-derive, or for zerocopy with the derive
# feature.
#
# Only run UI tests for the 'msrv', 'stable', and 'nightly' toolchains.
# Other toolchains are tested only because zerocopy has behavior which
# differs on those toolchains, but at present, none of that behavior
# affects UI tests. If we were to run UI tests on these toolchains, we
# would need a new set of UI test files per toolchain, which would be a
# maintenance burden not worth the (at present, zero) benefit.
if: |
(contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')) &&
(matrix.crate == 'zerocopy-derive' ||
(matrix.features != '' && matrix.features != '--no-default-features')) &&
(matrix.toolchain == 'msrv' || matrix.toolchain == 'stable' || matrix.toolchain == 'nightly')
- name: Run tests under Miri
run: |
set -eo pipefail
# Work around https://github.com/rust-lang/miri/issues/3125
[ "${{ matrix.target }}" == "aarch64-unknown-linux-gnu" ] && cargo clean
# Spawn twice the number of workers as there are CPU cores.
THREADS=$(echo "$(nproc) * 2" | bc)
echo "Running Miri tests with $THREADS threads" | tee -a $GITHUB_STEP_SUMMARY
cargo install cargo-nextest
# Run under both the stacked borrows model (default) and under the tree
# borrows model to ensure we're compliant with both.
for EXTRA_FLAGS in "" "-Zmiri-tree-borrows"; do
MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" ./cargo.sh +${{ matrix.toolchain }} \
miri nextest run \
--test-threads "$THREADS" \
--package ${{ matrix.crate }} \
--target ${{ matrix.target }} \
${{ matrix.features }}
done
# Only nightly has a working Miri, so we skip installing on all other
# toolchains.
#
# We skip Miri tests on pull request, but run them in the merge queue.
# Miri tests are far and away the most expensive aspect of our CI tests,
# but they rarely surface issues (ie, tests that pass `cargo test` but
# fail `cargo miri test`). Skipping them during PR development
# significantly speeds up our development flow, while still ensuring that
# Miri can catch any errors before a PR is merged into main.
#
# TODO(#22): Re-enable testing on riscv64gc-unknown-linux-gnu and/or
# wasm32-wasi once those work.
if: |
matrix.toolchain == 'nightly' &&
matrix.target != 'riscv64gc-unknown-linux-gnu' &&
matrix.target != 'wasm32-wasi' &&
github.event_name != 'pull_request'
- name: Clippy check
run: ./cargo.sh +${{ matrix.toolchain }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose
# Clippy improves the accuracy of lints over time, and fixes bugs. Only
# running Clippy on nightly allows us to avoid having to write code which
# is compatible with older versions of Clippy, which sometimes requires
# hacks to work around limitations that are fixed in more recent versions.
if: matrix.toolchain == 'nightly'
- name: Cargo doc
# We pass --document-private-items and --document-hidden items to ensure that
# documentation always builds even for these items. This makes future changes to
# make those items public/non-hidden more painless. Note that
# --document-hidden-items is unstable; if a future release breaks or removes it,
# we can just update CI to no longer pass that flag.
run: |
# Include arguments passed during docs.rs deployments to make sure those
# work properly.
METADATA_DOCS_RS_RUSTDOC_ARGS="$(cargo metadata --format-version 1 | \
jq -r ".packages[] | select(.name == \"zerocopy\").metadata.docs.rs.\"rustdoc-args\".[]" | tr '\n' ' ')"
export RUSTDOCFLAGS="${{ matrix.toolchain == 'nightly' && '-Z unstable-options --document-hidden-items' || '' }} $RUSTDOCFLAGS $METADATA_DOCS_RS_RUSTDOC_ARGS"
./cargo.sh +${{ matrix.toolchain }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }}
# Check semver compatibility with the most recently-published version on
# crates.io. We do this in the matrix rather than in its own job so that it
# gets run on different targets. Some of our API is target-specific (e.g.,
# SIMD type impls), and so we need to run on each target.
#
# TODO(https://github.com/obi1kenobi/cargo-semver-checks-action/issues/54):
# Currently we don't actually do anything with `matrix.target`, so we're
# just duplicating work by running this job multiple times, each time
# targetting the host platform.
- name: Check semver compatibility
uses: obi1kenobi/cargo-semver-checks-action@c7306483f698c511eaf7416d1bf2e1958c90140f # v2.4
with:
# Don't semver check zerocopy-derive; as a proc macro, it doesn't have
# an API that cargo-semver-checks can understand.
package: zerocopy
# Don't test nightly features. The reason this doesn't work is that
# cargo-semver-checks works relative to the most recently-published
# stable release on crates.io. The pinned nightly toolchain may be
# incompatible with the unstable features on that published release, as
# nightly features may have changed since it was released. Luckily, we
# don't care about the API stability of these features anyway.
feature-group: only-explicit-features
features: __internal_use_only_features_that_work_on_stable
rust-toolchain: ${{ env.ZC_TOOLCHAIN }}
if: matrix.crate == 'zerocopy' && matrix.features == '--features __internal_use_only_features_that_work_on_stable' && matrix.toolchain == 'nightly'
# TODO(#453): Doing this as a matrix step is a hack that allows us to depend
# on the fact that toolchains have already been installed. We currently only
# run this on a single matrix combination, but doing it outside of the
# matrix would require us to replicate the toolchain resolution and
# installation logic. We should either:
# - Figure out how to factor out the toolchain resolution and installation
# logic (see #1275) for an attempt
# - Support multiple matrix combinations (which we intend to do as part of
# #453 eventually anyway) so that this location is justified
- name: Generate code coverage
run: |
set -eo pipefail
./cargo.sh +nightly install cargo-llvm-cov
./cargo.sh +${{ matrix.toolchain }} llvm-cov \
--package ${{ matrix.crate }} \
--target ${{ matrix.target }} \
${{ matrix.features }} \
--lcov \
--output-path lcov.info \
--verbose \
-- \
--skip ui
if: |
matrix.crate == 'zerocopy' &&
matrix.features == '--all-features' &&
matrix.toolchain == 'nightly' &&
matrix.target == 'x86_64-unknown-linux-gnu'
- name: Upload coverage to Codecov
uses: codecov/codecov-action@v4
with:
token: ${{ secrets.CODECOV_TOKEN }}
files: lcov.info
if: |
matrix.crate == 'zerocopy' &&
matrix.features == '--all-features' &&
matrix.toolchain == 'nightly' &&
matrix.target == 'x86_64-unknown-linux-gnu'
kani:
runs-on: ubuntu-latest
name: 'Run tests under Kani'
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- uses: model-checking/kani-github-action@f838096619a707b0f6b2118cf435eaccfa33e51f # v1.1
with:
# Use `--features __internal_use_only_features_that_work_on_stable`
# because the Kani GitHub Action uses its own pinned nightly
# toolchain. Sometimes, we make changes to our nightly features for
# more recent toolchains, and so our nightly features become
# incompatible with the toolchain that Kani uses. By only testing
# stable features, we ensure that this doesn't cause problems in CI.
#
# TODO(https://github.com/model-checking/kani-github-action/issues/56):
# Go back to testing all features once the Kani GitHub Action supports
# specifying a particular toolchain.
args: "--package zerocopy --features __internal_use_only_features_that_work_on_stable --output-format=terse --randomize-layout --memory-safety-checks --overflow-checks --undefined-function-checks --unwinding-checks"
# This version is automatically rolled by
# `roll-pinned-toolchain-versions.yml`.
kani-version: 0.51.0
check_fmt:
runs-on: ubuntu-latest
name: Check Rust formatting
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Check Rust formatting
run: ./ci/check_fmt.sh
check_readme:
needs: generate_cache
runs-on: ubuntu-latest
name: Check README.md
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Populate cache
uses: ./.github/actions/cache
- name: Check README.md
run: ./ci/check_readme.sh
check_msrv:
needs: generate_cache
runs-on: ubuntu-latest
name: Check MSRVs match
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Populate cache
uses: ./.github/actions/cache
# Make sure that the MSRV in zerocopy's and zerocopy-derive's `Cargo.toml`
# files are the same. In CI, we test with a single MSRV (the one indicated
# in zerocopy's `Cargo.toml`), so it's important that:
# - zerocopy-derive's MSRV is not lower than zerocopy's (we don't test with
# a lower MSRV in CI, so we couldn't guarantee that zerocopy-derive
# actually built and ran on a lower MSRV)
# - zerocopy-derive's MSRV is not higher than zerocopy's (this would mean
# that compiling zerocopy with the `derive` feature enabled would fail
# on its own published MSRV)
- name: Check MSRVs match
run: ./ci/check_msrv.sh
check_versions:
needs: generate_cache
runs-on: ubuntu-latest
name: Check crate versions match
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Populate cache
uses: ./.github/actions/cache
# Make sure that both crates are at the same version, and that zerocopy
# depends exactly upon the current version of zerocopy-derive. See
# `INTERNAL.md` for an explanation of why we do this.
- name: Check crate versions match
run: ./ci/check_versions.sh
generate_cache:
runs-on: ubuntu-latest
name: Generate cache
steps:
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Populate cache
uses: ./.github/actions/cache
- name: Download dependencies
run: |
# Ensure all dependencies are downloaded - both for our crates and for
# tools we use in CI. We don't care about these tools succeeding for
# two reasons: First, this entire job is best-effort since it's just a
# performance optimization. Second, there may be failures due to
# issues other than failing to download dependencies (e.g., `cargo
# metadata` called with a malformed `Cargo.toml`, build failure in our
# own crate or in dependencies, etc). For those reasons, we discard
# stderr and ignore status codes.
#
# For downloading our crates' dependencies in particular, note that
# there is no support for doing this directly [1], so we just check
# all crates using --tests.
#
# We background all jobs and then wait for them so that they can run
# in parallel.
#
# [1] https://stackoverflow.com/a/42139535/836390
cargo check --workspace --tests &> /dev/null &
cargo metadata &> /dev/null &
cargo install cargo-readme --version 3.2.0 &> /dev/null &
cargo install --locked kani-verifier &> /dev/null &
cargo install cargo-nextest &> /dev/null &
cargo kani setup &> /dev/null &
wait
check-all-toolchains-tested:
runs-on: ubuntu-latest
name: Check that all toolchains listed in Cargo.toml are tested in CI
steps:
- name: Install yq (for YAML parsing)
run: go install github.com/mikefarah/yq/v4@latest
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Run check
run: ./ci/check_all_toolchains_tested.sh
check-job-dependencies:
runs-on: ubuntu-latest
name: Check all-jobs-succeeded depends on all jobs
steps:
- name: Install yq (for YAML parsing)
run: go install github.com/mikefarah/yq/v4@latest
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Run dependency check
run: ./ci/check_job_dependencies.sh
run-git-hooks:
runs-on: ubuntu-latest
name: Run Git hooks
steps:
- name: Install yq (for YAML parsing)
run: go install github.com/mikefarah/yq/v4@latest
- uses: actions/checkout@a5ac7e51b41094c92402da3b24376905380afc29 # v4.1.6
- name: Run dependency check
# Ensure that Git hooks execute successfully.
#
# Note that this runs all hooks. As of this writing, the only Git hook
# is `pre-push`. Running all hooks ensures that, if a new hook is added
# which can't run in CI (at least not using this naive setup), we'll
# notice and it will remind us to update this test.
run: set -e; for hook in ./githooks/*; do $hook; done
# Used to signal to branch protections that all other jobs have succeeded.
all-jobs-succeed:
name: All checks succeeded
# On failure, we run and unconditionally exit with a failing status code.
# On success, this job is skipped. Jobs skipped using `if:` are considered
# to have succeeded:
#
# https://docs.github.com/en/pull-requests/collaborating-with-pull-requests/collaborating-on-repositories-with-code-quality-features/troubleshooting-required-status-checks#handling-skipped-but-required-checks
if: failure()
runs-on: ubuntu-latest
needs: [build_test, kani, check_fmt, check_readme, check_msrv, check_versions, generate_cache, check-all-toolchains-tested, check-job-dependencies, run-git-hooks]
steps:
- name: Mark the job as failed
run: exit 1