Skip to content

Commit

Permalink
[cargo.sh] Initial commit (#363)
Browse files Browse the repository at this point in the history
`cargo.sh` is a small utility that supports running Cargo with
human-readable toolchain names that are mapped to the equivalent
versions pinned in CI. It supports "msrv", "stable", and "nightly". It
also supports a meta-toolchain, "all", which instructs it to run a
command once with each of the three pinned toolchains.

In this commit, we replace the toolchain name-to-version resolution
logic in CI with this command. This simplifies our CI configuration, and
maintains the property that the code run in CI is the same code a
developer would run on their local machine.
  • Loading branch information
joshlf authored Sep 10, 2023
1 parent eb2bd15 commit 311e662
Show file tree
Hide file tree
Showing 2 changed files with 112 additions and 47 deletions.
77 changes: 30 additions & 47 deletions .github/workflows/ci.yml
Original file line number Diff line number Diff line change
Expand Up @@ -80,50 +80,26 @@ jobs:
~/.cargo/
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}

# We use toolchain descriptors ("msrv", "stable", and "nightly") in the
# matrix. This step converts the current descriptor to a particular
# toolchain version by looking up the corresponding key in `Cargo.toml`. It
# sets the `ZC_TOOLCHAIN` environment variable for future steps to use.
#
# Note that all metadata is stored in zerocopy's `Cargo.toml` (the one at
# the repository root). zerocopy-derive is tested with the same versions,
# and we have another CI job (see below) that makes sure that the
# `package.rust_version` key in zerocopy-derive's `Cargo.toml` is the same
# as the one in zerocopy's `Cargo.toml`. This key indicates the crate's
# MSRV, and if this check weren't present, it would be possible for
# zerocopy-derive to be published with an earlier MSRV than the one we test
# for in CI - and thus potentially an MSRV that zerocopy-derive isn't
# actually compatible with.
- name: Set toolchain version
- name: Configure environment variables
run: |
set -eo pipefail
function pkg-meta {
cargo metadata --format-version 1 | jq -r ".packages[] | select(.name == \"zerocopy\").$1"
}
case ${{ matrix.toolchain }} in
msrv)
ZC_TOOLCHAIN="$(pkg-meta rust_version)"
;;
stable)
ZC_TOOLCHAIN="$(pkg-meta 'metadata.ci."pinned-stable"')"
;;
nightly)
ZC_TOOLCHAIN="$(pkg-meta 'metadata.ci."pinned-nightly"')"
;;
*)
echo 'Unrecognized toolchain: ${{ matrix.toolchain }}' | tee -a $GITHUB_STEP_SUMMARY >&2
exit 1
;;
esac
# We use toolchain descriptors ("msrv", "stable", and "nightly") in the
# matrix. This step converts the current descriptor to a particular
# toolchain version by looking up the corresponding key in `Cargo.toml`. It
# sets the `ZC_TOOLCHAIN` environment variable for use in the next step
# (toolchain installation) because GitHub variable interpolation doesn't
# support running arbitrary commands. In other words, we can't rewrite:
#
# toolchain: $ {{ env.ZC_TOOLCHAIN }}
#
# ...to:
#
# toolchain: $ {{ ./cargo.sh --version matrix.toolchain }} # hypothetical syntax
ZC_TOOLCHAIN="$(./cargo.sh --version ${{ matrix.toolchain }})"
echo "Found that the '${{ matrix.toolchain }}' toolchain is $ZC_TOOLCHAIN" | tee -a $GITHUB_STEP_SUMMARY
echo "ZC_TOOLCHAIN=$ZC_TOOLCHAIN" >> $GITHUB_ENV
- name: Configure environment variables
run: |
set -eo pipefail
if [[ '${{ matrix.toolchain }}' == 'nightly' ]]; then
RUSTFLAGS="$RUSTFLAGS $ZC_NIGHTLY_RUSTFLAGS"
MIRIFLAGS="$MIRIFLAGS $ZC_NIGHTLY_MIRIFLAGS"
Expand All @@ -134,7 +110,7 @@ jobs:
echo "Using non-nightly toolchain; not modifying RUSTFLAGS='$RUSTFLAGS' or MIRIFLAGS='$MIRIFLAGS'" | tee -a $GITHUB_STEP_SUMMARY
fi
- name: Install Rust with toolchain ${{ env.ZC_TOOLCHAIN }} and target ${{ matrix.target }}
- name: Install Rust with ${{ matrix.toolchain }} toolchain (${{ env.ZC_TOOLCHAIN }}) and target ${{ matrix.target }}
uses: dtolnay/rust-toolchain@00b49be78f40fba4e87296b2ead62868750bdd83 # stable
with:
toolchain: ${{ env.ZC_TOOLCHAIN }}
Expand All @@ -153,10 +129,10 @@ jobs:
key: "${{ matrix.target }}"

- name: Check
run: cargo +${{ env.ZC_TOOLCHAIN }} check --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
run: ./cargo.sh +${{ matrix.toolchain }} check --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose

- name: Build
run: cargo +${{ env.ZC_TOOLCHAIN }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
run: ./cargo.sh +${{ matrix.toolchain }} build --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose

# When building tests for the i686 target, we need certain libraries which
# are not installed by default; `gcc-multilib` includes these libraries.
Expand All @@ -174,7 +150,7 @@ jobs:
if: contains(matrix.target, 'i686')

- name: Run tests
run: cargo +${{ env.ZC_TOOLCHAIN }} test --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
run: ./cargo.sh +${{ matrix.toolchain }} test --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
# Only run tests when targetting x86 (32- or 64-bit) - we're executing on
# x86_64, so we can't run tests for any non-x86 target.
#
Expand All @@ -189,7 +165,7 @@ jobs:
for EXTRA_FLAGS in "" "-Zmiri-tree-borrows"; do
# Skip the `ui` test since it invokes the compiler, which we can't do from
# Miri (and wouldn't want to do anyway).
MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" cargo +${{ env.ZC_TOOLCHAIN }} \
MIRIFLAGS="$MIRIFLAGS $EXTRA_FLAGS" ./cargo.sh +${{ matrix.toolchain }} \
miri test \
--package ${{ matrix.crate }} \
--target ${{ matrix.target }} \
Expand All @@ -209,7 +185,7 @@ jobs:
# make use of derives, and so would fail without the `derive` feature
# enabled. We skip this step for `zerocopy` when the `derive` feature is
# omitted for that reason.
run: cargo +${{ env.ZC_TOOLCHAIN }} test --doc --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
run: ./cargo.sh +${{ matrix.toolchain }} test --doc --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --verbose
# Only run tests when targetting x86 (32- or 64-bit) - we're executing on
# x86_64, so we can't run tests for any non-x86 target.
#
Expand All @@ -218,7 +194,7 @@ jobs:
if: ${{ (contains(matrix.target, 'x86_64') || contains(matrix.target, 'i686')) && !(matrix.crate == 'zerocopy' && !contains(matrix.features, 'derive')) }}

- name: Clippy check
run: cargo +${{ env.ZC_TOOLCHAIN }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose
run: ./cargo.sh +${{ matrix.toolchain }} clippy --package ${{ matrix.crate }} --target ${{ matrix.target }} ${{ matrix.features }} --tests --verbose
# Clippy improves the accuracy of lints over time, and fixes bugs. Only
# running Clippy on nightly allows us to avoid having to write code which
# is compatible with older versions of Clippy, which sometimes requires
Expand All @@ -233,7 +209,7 @@ jobs:
# we can just update CI to no longer pass that flag.
run: |
export RUSTDOCFLAGS="${{ matrix.toolchain == 'nightly' && '-Z unstable-options --document-hidden-items' || '' }} $RUSTDOCFLAGS"
cargo +${{ env.ZC_TOOLCHAIN }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }}
./cargo.sh +${{ matrix.toolchain }} doc --document-private-items --package ${{ matrix.crate }} ${{ matrix.features }}
# When the `alloc` feature is disabled, `cargo doc` fails because we link
# to `alloc::vec::Vec` in a doc comment, and the `alloc` crate is not in
# scope without the `alloc` feature. This isn't a big deal because we care
Expand Down Expand Up @@ -316,7 +292,14 @@ jobs:
key: ${{ runner.os }}-cargo-${{ hashFiles('**/Cargo.toml') }}

# Make sure that the MSRV in zerocopy's and zerocopy-derive's `Cargo.toml`
# files are the same.
# files are the same. In CI, we test with a single MSRV (the one indicated
# in zerocopy's `Cargo.toml`), so it's important that:
# - zerocopy-derive's MSRV is not lower than zerocopy's (we don't test with
# a lower MSRV in CI, so we couldn't guarantee that zerocopy-derive
# actually built and ran on a lower MSRV)
# - zerocopy-derive's MSRV is not higher than zerocopy's (this would mean
# that compiling zerocopy with the `derive` feature enabled would fail
# on its own published MSRV)
- name: Check MSRVs match
run: |
set -eo pipefail
Expand Down
82 changes: 82 additions & 0 deletions cargo.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,82 @@
#!/bin/bash
#
# Copyright 2023 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.

# This script is a thin wrapper around Cargo that provides human-friendly
# toolchain names which are automatically translated to the toolchain versions
# we have pinned in CI.
#
# cargo.sh --version <toolchain-name> # looks up the version for the named toolchain
# cargo.sh +<toolchain-name> [...] # runs cargo commands with the named toolchain
# cargo.sh +all [...] # runs cargo commands with each toolchain
#
# The meta-toolchain "all" instructs this script to run the provided command
# once for each toolchain (msrv, stable, nightly).
#
# A common task that is especially annoying to perform by hand is to update
# trybuild's stderr files. Using this script:
#
# TRYBUILD=overwrite ./cargo.sh +all test --workspace

set -eo pipefail

function print-usage-and-exit {
echo "Usage:" >&2
echo " $0 --version <toolchain-name>" >&2
echo " $0 +<toolchain-name> [...]" >&2
echo " $0 +all [...]" >&2
exit 1
}

[[ $# -gt 0 ]] || print-usage-and-exit

function pkg-meta {
cargo metadata --format-version 1 | jq -r ".packages[] | select(.name == \"zerocopy\").$1"
}

function lookup-version {
VERSION="$1"
case "$VERSION" in
msrv)
pkg-meta rust_version
;;
stable)
pkg-meta 'metadata.ci."pinned-stable"'
;;
nightly)
pkg-meta 'metadata.ci."pinned-nightly"'
;;
*)
echo "Unrecognized toolchain name: '$VERSION' (options are 'msrv', 'stable', 'nightly')" >&2
return 1
;;
esac
}

case "$1" in
# cargo.sh --version <toolchain-name>
--version)
[[ $# -eq 2 ]] || print-usage-and-exit
lookup-version "$2"
;;
# cargo.sh +all [...]
+all)
echo "[cargo.sh] warning: running the same command for each toolchain (msrv, stable, nightly)" >&2
for toolchain in msrv stable nightly; do
echo "[cargo.sh] running with toolchain: $toolchain" >&2
TOOLCHAIN="$(lookup-version $toolchain)"
cargo "+$TOOLCHAIN" ${@:2}
done
exit 0
;;
# cargo.sh +<toolchain-name> [...]
+*)
TOOLCHAIN="$(lookup-version ${1:1})"
cargo "+$TOOLCHAIN" ${@:2}
;;
*)
print-usage-and-exit
;;
esac

0 comments on commit 311e662

Please sign in to comment.