Merge branch 'main' into fix-test-anti-replay

.github/actions/nss/action.yml

@@ -59,6 +59,32 @@ runs:
  echo "System NSS is suitable: $NSS_VERSION"
  echo "BUILD_NSS=0" >> "$GITHUB_ENV"
 
+ - name: Use sccache
+ # Apparently the action can't be installed twice in the same workflow, so check if
+ # it's already installed by checking if the SCCACHE_ENABLED environment variable is set
+ # (which every "use" of this action needs to therefore set)
+ #
+ # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit
+ # is too small for this to be effective there.
+ if: env.SCCACHE_ENABLED != '1' && env.BUILD_NSS == '1' && runner.environment != 'github-hosted'
+ uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4
+
+ - name: Enable sscache
+ if: env.BUILD_NSS == '1' && runner.environment != 'github-hosted'
+ shell: bash
+ run: |
+ echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV"
+ if [ "${{ runner.os }}" != "Windows" ]; then
+ # TODO: Figure out how to make this work on Windows
+ echo "SCCACHE_CC=sccache cc" >> "$GITHUB_ENV"
+ echo "SCCACHE_CXX=sccache c++" >> "$GITHUB_ENV"
+ fi
+ echo "CMAKE_C_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
+ echo "CMAKE_CXX_COMPILER_LAUNCHER=sccache" >> "$GITHUB_ENV"
+ if [ "${{ runner.environment }}" == "github-hosted" ]; then
+ echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
+ fi
+
  - name: Checkout NSS
  if: env.BUILD_NSS == '1'
  uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
@@ -73,6 +99,34 @@ runs:
  repository: nss-dev/nspr
  path: nspr
 
+ - name: Get head revisions
+ if: env.BUILD_NSS == '1'
+ shell: bash
+ run: |
+ NSS_HEAD=$(git -C nss rev-parse HEAD)
+ NSPR_HEAD=$(git -C nspr rev-parse HEAD)
+ echo "NSS_HEAD=$NSS_HEAD" >> "$GITHUB_ENV"
+ echo "NSPR_HEAD=$NSPR_HEAD" >> "$GITHUB_ENV"
+
+ - name: Cache NSS
+ id: cache
+ if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
+ uses: actions/cache@0c45773b623bea8c8e75f6c82b208c3cf94ea4f9 # v4.0.2
+ with:
+ path: dist
+ key: nss-${{ runner.os }}-${{ inputs.type }}-${{ env.NSS_HEAD }}-${{ env.NSPR_HEAD }}
+
+ - name: Check if build is needed
+ if: env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
+ shell: bash
+ run: |
+ if [ "${{ steps.cache.outputs.cache-hit }}" == "true" ]; then
+ echo "Using cached prebuilt NSS"
+ echo "BUILD_NSS=0" >> "$GITHUB_ENV"
+ else
+ echo "Building NSS from source"
+ fi
+
  - name: Install build dependencies (Linux)
  shell: bash
  if: runner.os == 'Linux' && env.BUILD_NSS == '1' && runner.environment == 'github-hosted'
@@ -120,25 +174,31 @@ runs:
  # See https://github.com/ilammy/msvc-dev-cmd#name-conflicts-with-shell-bash
  rm /usr/bin/link.exe || true
 
- - name: Build
+ - name: Set up environment
  shell: bash
- if: env.BUILD_NSS == '1'
  run: |
- if [ "${{ inputs.type }}" != "Debug" ]; then
- # We want to do an optimized build for accurate CPU profiling, but
- # we also want debug symbols and frame pointers for that, which the normal optimized NSS
- # build process doesn't provide.
- OPT="-o"
- [ "${{ runner.os }}" != "Windows" ] && export CFLAGS="-ggdb3 -fno-omit-frame-pointer"
- fi
  NSS_TARGET="${{ inputs.type }}"
  echo "NSS_TARGET=$NSS_TARGET" >> "$GITHUB_ENV"
  NSS_OUT="$NSS_DIR/../dist/$NSS_TARGET"
  echo "LD_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
  echo "DYLD_FALLBACK_LIBRARY_PATH=$NSS_OUT/lib" >> "$GITHUB_ENV"
  echo "$NSS_OUT/lib" >> "$GITHUB_PATH"
  echo "NSS_DIR=$NSS_DIR" >> "$GITHUB_ENV"
- $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static
+ echo "NSS_PREBUILT=1" >> "$GITHUB_ENV"
  env:
  NSS_DIR: ${{ github.workspace }}/nss
  NSPR_DIR: ${{ github.workspace }}/nspr
+
+ - name: Build
+ shell: bash
+ if: env.BUILD_NSS == '1'
+ run: |
+ if [ "${{ inputs.type }}" != "Debug" ]; then
+ # We want to do an optimized build for accurate CPU profiling, but
+ # we also want debug symbols and frame pointers for that, which the normal optimized NSS
+ # build process doesn't provide.
+ OPT="-o"
+ [ "${{ runner.os }}" != "Windows" ] && export CFLAGS="-ggdb3 -fno-omit-frame-pointer"
+ fi
+ [ "$SCCACHE_CC" ] && [ "$SCCACHE_CXX" ] && export CC="$SCCACHE_CC" CXX="$SCCACHE_CXX"
+ $NSS_DIR/build.sh -g -Ddisable_tests=1 $OPT --static

.github/actions/rust/action.yml

@@ -21,18 +21,34 @@ inputs:
 runs:
  using: composite
  steps:
- - name: Upgrade rustup (MacOS)
- shell: bash
- if: runner.os == 'MacOS'
- run: brew update && brew upgrade rustup
-
  - name: Install Rust
  uses: dtolnay/rust-toolchain@21dc36fb71dd22e3317045c0c31a3f4249868b17 # master
  with:
  toolchain: ${{ inputs.version }}
  components: ${{ inputs.components }}
  targets: ${{ inputs.targets }}
 
+ - name: Use sccache
+ # Apparently the action can't be installed twice in the same workflow, so check if
+ # it's already installed by checking if the SCCACHE_ENABLED environment variable is set
+ # (which every "use" of this action needs to therefore set)
+ #
+ # Also, only enable sscache on our self-hosted runner, because the GitHub cache limit
+ # is too small for this to be effective there.
+ if: env.SCCACHE_ENABLED != '1' && runner.environment != 'github-hosted'
+ uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4
+
+ - name: Enable sscache
+ if: runner.environment != 'github-hosted'
+ shell: bash
+ run: |
+ echo "SCCACHE_ENABLED=1" >> "$GITHUB_ENV"
+ echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
+ echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
+ if [ "${{ runner.environment }}" == "github-hosted" ]; then
+ echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
+ fi
+
  - name: Set up MSVC (Windows)
  if: runner.os == 'Windows'
  uses: ilammy/msvc-dev-cmd@v1
@@ -58,28 +74,3 @@ runs:
  env:
  GITHUB_TOKEN: ${{ inputs.token }}
  run: cargo +${{ inputs.version }} quickinstall $(echo ${{ inputs.tools }} | tr -d ",")
-
- # sccache slows CI down, so we leave it disabled.
- # Leaving the steps below commented out, so we can re-evaluate enabling it later.
- # - name: Use sccache
- # uses: mozilla-actions/sccache-action@2e7f9ec7921547d4b46598398ca573513895d0bd # v0.0.4
-
- # - name: Enable sscache
- # shell: bash
- # run: |
- # if [ "${{ runner.os }}" = "Windows" ]; then
- # echo "CC=sccache cl" >> "$GITHUB_ENV"
- # echo "CXX=sccache cl" >> "$GITHUB_ENV"
- # else
- # echo "CC=sccache cc" >> "$GITHUB_ENV"
- # echo "CXX=sccache c++" >> "$GITHUB_ENV"
- # fi
- # echo "SCCACHE_GHA_ENABLED=true" >> "$GITHUB_ENV"
- # echo "RUSTC_WRAPPER=sccache" >> "$GITHUB_ENV"
- # echo "CARGO_INCREMENTAL=0" >> "$GITHUB_ENV"
-
- # Ditto for rust-cache.
- # - name: Use Rust cache
- # uses: Swatinem/rust-cache@23bce251a8cd2ffc3c1075eaa2367cf899916d84 # v2.7.3
- # with:
- # cache-all-crates: "true"

.github/workflows/bench.yml

@@ -10,7 +10,7 @@ env:
  CARGO_PROFILE_RELEASE_DEBUG: true
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
- TOOLCHAIN: nightly-2024-09-01
+ TOOLCHAIN: nightly
  RUSTFLAGS: -C link-arg=-fuse-ld=lld -C link-arg=-Wl,--no-rosegment, -C force-frame-pointers=yes
  PERF_OPT: record -F997 --call-graph fp -g
 
@@ -58,8 +58,8 @@ jobs:
 
  - name: Build neqo
  run: |
- cargo "+$TOOLCHAIN" bench --features bench --no-run
- cargo "+$TOOLCHAIN" build --release
+ cargo "+$TOOLCHAIN" bench --workspace --features bench --no-run
+ cargo "+$TOOLCHAIN" build --release --bin neqo-client --bin neqo-server
 
  - name: Build msquic
  run: |

.github/workflows/check.yml

@@ -7,6 +7,7 @@ on:
  branches: ["main"]
  paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
  merge_group:
+ workflow_dispatch:
 env:
  CARGO_TERM_COLOR: always
  RUST_BACKTRACE: 1
@@ -20,15 +21,14 @@ permissions:
 
 jobs:
  check:
- name: Build & test
  strategy:
  fail-fast: false
  matrix:
  os: [ubuntu-latest, macos-14, windows-latest]
  # Don't increase beyond what Firefox is currently using:
  # https://searchfox.org/mozilla-central/search?q=MINIMUM_RUST_VERSION&path=python/mozboot/mozboot/util.py
  # Keep in sync with Cargo.toml
- rust-toolchain: [1.76.0, stable, nightly-2024-09-01]
+ rust-toolchain: [1.76.0, stable, nightly]
  type: [debug]
  include:
  - os: ubuntu-latest
@@ -42,36 +42,35 @@ jobs:
  shell: bash
 
  steps:
- - name: Checkout
- uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
 
- - name: Install Rust
- uses: ./.github/actions/rust
+ - uses: ./.github/actions/rust
  with:
  version: ${{ matrix.rust-toolchain }}
- components: rustfmt, clippy, llvm-tools-preview
- tools: cargo-llvm-cov, cargo-nextest, cargo-hack, cargo-fuzz, cargo-machete
+ components: ${{ matrix.rust-toolchain == 'stable' && 'llvm-tools-preview' || '' }}
+ tools: ${{ matrix.rust-toolchain == 'stable' && 'cargo-llvm-cov, ' || '' }} cargo-nextest
  token: ${{ secrets.GITHUB_TOKEN }}
 
- - name: Get minimum NSS version
- id: nss-version
+ - id: nss-version
  run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT"
 
- - name: Install NSS
- uses: ./.github/actions/nss
+ - uses: ./.github/actions/nss
  with:
  minimum-version: ${{ steps.nss-version.outputs.minimum }}
 
- - name: Build
+ - name: Check
  run: |
  # shellcheck disable=SC2086
- cargo +${{ matrix.rust-toolchain }} build $BUILD_TYPE --all-targets --features ci
+ cargo +${{ matrix.rust-toolchain }} check $BUILD_TYPE --all-targets --features ci
 
  - name: Run tests and determine coverage
  run: |
  # shellcheck disable=SC2086
- RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
- cargo +${{ matrix.rust-toolchain }} bench --features bench --no-run
+ if [ "${{ matrix.rust-toolchain }}" == "stable" ]; then
+ RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} llvm-cov nextest $BUILD_TYPE --features ci --no-fail-fast --lcov --output-path lcov.info
+ else
+ RUST_LOG=trace cargo +${{ matrix.rust-toolchain }} nextest run $BUILD_TYPE --features ci --no-fail-fast
+ fi
 
  - name: Run client/server transfer
  run: |
@@ -90,41 +89,7 @@ jobs:
  RUST_LOG: warn
  BUILD_DIR: ${{ matrix.type == 'release' && 'release' || 'debug' }}
 
- - name: Check formatting
- run: |
- if [ "${{ startsWith(matrix.rust-toolchain, 'nightly') && 'nightly' }}" != "nightly" ]; then
- CONFIG_PATH="--config-path=$(mktemp)"
- fi
- # shellcheck disable=SC2086
- cargo +${{ matrix.rust-toolchain }} fmt --all -- --check $CONFIG_PATH
- if: success() || failure()
-
- - name: Check for unused dependencies
- run: |
- # --with-metadata has false positives, see https://github.com/bnjbvr/cargo-machete/issues/127
- cargo +${{ matrix.rust-toolchain }} machete
-
- - name: Clippy
- run: |
- # Use cargo-hack to run clippy on each crate individually with its
- # respective default features only. Can reveal warnings otherwise
- # hidden given that a plain cargo clippy combines all features of the
- # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695.
- cargo +${{ matrix.rust-toolchain }} hack clippy --all-targets --feature-powerset --exclude-features gecko -- -D warnings || ${{ matrix.rust-toolchain == 'nightly' }}
- # Check that the fuzz targets also build
- if [ ${{ startsWith(matrix.rust-toolchain, 'nightly') && 'nightly' }} == 'nightly' ]; then
- cargo +${{ matrix.rust-toolchain }} fuzz check
- fi
- if: success() || failure()
-
- - name: Check rustdoc links
- run: cargo +${{ matrix.rust-toolchain }} doc --workspace --no-deps --document-private-items
- env:
- RUSTDOCFLAGS: "--deny rustdoc::broken_intra_doc_links --deny warnings"
- if: success() || failure()
-
- - name: Upload coverage reports to Codecov
- uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
+ - uses: codecov/codecov-action@e28ff129e5465c2c0dcc6f003fc735cb6ae0c673 # v4.5.0
  with:
  file: lcov.info
  fail_ci_if_error: false
@@ -135,6 +100,5 @@ jobs:
  if: matrix.type == 'debug' && matrix.rust-toolchain == 'stable'
 
  bench:
- name: "Benchmark"
  needs: [check]
  uses: ./.github/workflows/bench.yml

.github/workflows/clippy.yml

@@ -0,0 +1,47 @@
+name: Clippy
+on:
+ push:
+ branches: ["main"]
+ paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+ pull_request:
+ branches: ["main"]
+ paths-ignore: ["*.md", "*.png", "*.svg", "LICENSE-*"]
+ merge_group:
+ workflow_dispatch:
+env:
+ CARGO_TERM_COLOR: always
+ RUST_BACKTRACE: 1
+
+concurrency:
+ group: ${{ github.workflow }}-${{ github.ref_name }}
+ cancel-in-progress: true
+
+permissions:
+ contents: read
+
+jobs:
+ clippy:
+ runs-on: ubuntu-latest
+ steps:
+ - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+ - uses: ./.github/actions/rust
+ with:
+ components: clippy
+ tools: cargo-hack
+ token: ${{ secrets.GITHUB_TOKEN }}
+
+ - id: nss-version
+ run: echo "minimum=$(cat neqo-crypto/min_version.txt)" >> "$GITHUB_OUTPUT"
+
+ - uses: ./.github/actions/nss
+ with:
+ minimum-version: ${{ steps.nss-version.outputs.minimum }}
+
+ # Use cargo-hack to run clippy on each crate individually with its
+ # respective default features only. Can reveal warnings otherwise
+ # hidden given that a plain cargo clippy combines all features of the
+ # workspace. See e.g. https://github.com/mozilla/neqo/pull/1695.
+ - run: cargo hack clippy --all-targets --feature-powerset --exclude-features gecko -- -D warnings
+ - run: cargo doc --workspace --no-deps --document-private-items
+ env:
+ RUSTDOCFLAGS: "--deny rustdoc::broken_intra_doc_links --deny warnings"