From e5ba9db644987a01894b25f839d6423270f98722 Mon Sep 17 00:00:00 2001 From: kaczmarczyck <43844792+kaczmarczyck@users.noreply.github.com> Date: Wed, 8 Nov 2023 17:24:15 +0100 Subject: [PATCH] Reworks workflows into script (#661) * Reworks some workflows to run a script instead Advantages are: - Only one set of tests needs maintenance. - Local results match workflows, no surprises. - Reduced reliance on GitHub actions. Fixes #50, #168, #169, #171, #507 * Adds macos to the test matrix --- .github/workflows/boards_build.yml | 38 ---- .github/workflows/cargo_audit.yml | 6 +- .github/workflows/cargo_bloat.yml | 9 +- .github/workflows/cargo_check.yml | 102 ----------- .github/workflows/cargo_clippy.yml | 34 ---- .github/workflows/cargo_fmt.yml | 96 ---------- .github/workflows/cargo_fuzz.yml | 33 ---- .github/workflows/cbor_test.yml | 37 ---- .github/workflows/ci.yml | 33 ++++ .github/workflows/coveralls.yml | 4 +- .github/workflows/crypto_test.yml | 43 ----- .github/workflows/heapviz_test.yml | 34 ---- .github/workflows/opensk_build.yml | 40 ---- .github/workflows/opensk_test.yml | 56 ------ .github/workflows/persistent_store_test.yml | 29 --- .github/workflows/python.yml | 46 ----- .github/workflows/reproducible.yml | 4 +- docs/customization.md | 2 +- .../opensk/src/api/crypto/rust_crypto.rs | 1 - libraries/opensk/src/api/private_key.rs | 4 +- .../persistent_store/fuzz/examples/store.rs | 3 +- libraries/persistent_store/src/driver.rs | 2 +- libraries/persistent_store/src/format.rs | 2 +- libraries/persistent_store/src/fragment.rs | 2 +- libraries/persistent_store/src/linear.rs | 2 +- libraries/persistent_store/src/store.rs | 12 +- requirements.txt | 1 + run_desktop_tests.sh | 173 ++++++++---------- setup.sh | 2 + tools/run_pylint.sh | 21 --- 30 files changed, 128 insertions(+), 743 deletions(-) delete mode 100644 .github/workflows/boards_build.yml delete mode 100644 .github/workflows/cargo_check.yml delete mode 100644 .github/workflows/cargo_clippy.yml delete mode 100644 .github/workflows/cargo_fmt.yml delete mode 100644 .github/workflows/cargo_fuzz.yml delete mode 100644 .github/workflows/cbor_test.yml create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/crypto_test.yml delete mode 100644 .github/workflows/heapviz_test.yml delete mode 100644 .github/workflows/opensk_build.yml delete mode 100644 .github/workflows/opensk_test.yml delete mode 100644 .github/workflows/persistent_store_test.yml delete mode 100644 .github/workflows/python.yml delete mode 100755 tools/run_pylint.sh diff --git a/.github/workflows/boards_build.yml b/.github/workflows/boards_build.yml deleted file mode 100644 index 1588c632..00000000 --- a/.github/workflows/boards_build.yml +++ /dev/null @@ -1,38 +0,0 @@ ---- -name: Build supported boards -on: - push: - paths: - - 'patches/tock/*' - - 'third_party/tock/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - build_boards: - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Building board nrf52840dk_opensk - run: ./deploy.py --board=nrf52840dk_opensk --no-app --programmer=none - - name: Building board nrf52840_dongle_opensk - run: ./deploy.py --board=nrf52840_dongle_opensk --no-app --programmer=none - - name: Building board nrf52840_dongle_dfu - run: ./deploy.py --board=nrf52840_dongle_dfu --no-app --programmer=none - - name: Building board nrf52840_mdk_dfu - run: ./deploy.py --board=nrf52840_mdk_dfu --no-app --programmer=none diff --git a/.github/workflows/cargo_audit.yml b/.github/workflows/cargo_audit.yml index 7bbed87a..ee322723 100644 --- a/.github/workflows/cargo_audit.yml +++ b/.github/workflows/cargo_audit.yml @@ -11,13 +11,9 @@ jobs: - uses: actions/checkout@v2 with: submodules: "true" - - name: Install Rust toolchain - run: rustup show - uses: actions/setup-python@v1 with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel + python-version: "3.10" - name: Set up OpenSK run: ./setup.sh - uses: actions-rs/audit-check@v1 diff --git a/.github/workflows/cargo_bloat.yml b/.github/workflows/cargo_bloat.yml index deac05e2..37229da5 100644 --- a/.github/workflows/cargo_bloat.yml +++ b/.github/workflows/cargo_bloat.yml @@ -8,9 +8,7 @@ jobs: # Setup - uses: actions/setup-python@v1 with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel + python-version: "3.10" - uses: actions-rs/cargo@v1 with: command: install @@ -20,8 +18,6 @@ jobs: - uses: actions/checkout@v2 with: submodules: true - - name: Install Rust toolchain - run: rustup show - name: Set up OpenSK run: ./setup.sh - name: Run bloat on the PR @@ -33,9 +29,6 @@ jobs: submodules: true ref: ${{ github.base_ref }} path: OpenSK_base - - name: Install old Rust toolchain - working-directory: ./OpenSK_base - run: rustup show - name: Set up OpenSK working-directory: ./OpenSK_base run: ./setup.sh diff --git a/.github/workflows/cargo_check.yml b/.github/workflows/cargo_check.yml deleted file mode 100644 index d74e6008..00000000 --- a/.github/workflows/cargo_check.yml +++ /dev/null @@ -1,102 +0,0 @@ -name: Cargo check -on: - push: - paths: - - 'examples/*.rs' - - 'libraries/**/*.rs' - - 'src/**/*.rs' - - 'patches/**' - - '**/Cargo.toml' - - '.cargo/config' - - '!third_party/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - cargo_check: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Check OpenSK w/o features - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release - - - name: Check OpenSK with_ctap1 - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features with_ctap1 - - - name: Check OpenSK vendor_hid - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features vendor_hid - - - name: Check OpenSK ed25519 - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features ed25519 - - - name: Check OpenSK debug_ctap - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features debug_ctap - - - name: Check OpenSK panic_console - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features panic_console - - - name: Check OpenSK debug_allocations - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features debug_allocations - - - name: Check OpenSK verbose - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features verbose - - - name: Check OpenSK debug_ctap,with_ctap1 - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features debug_ctap,with_ctap1 - - - name: Check OpenSK debug_ctap,with_ctap1,vendor_hid,ed25519,panic_console,debug_allocations,verbose - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --features debug_ctap,with_ctap1,vendor_hid,ed25519,panic_console,debug_allocations,verbose - - - name: Check examples - uses: actions-rs/cargo@v1 - with: - command: check - args: --target thumbv7em-none-eabi --release --examples - - - name: Check bootloader - uses: actions-rs/cargo@v1 - with: - command: check - args: --manifest-path bootloader/Cargo.toml --target thumbv7em-none-eabi --release diff --git a/.github/workflows/cargo_clippy.yml b/.github/workflows/cargo_clippy.yml deleted file mode 100644 index c815ec12..00000000 --- a/.github/workflows/cargo_clippy.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Cargo Clippy -on: - push: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - cargo_clippy: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - uses: actions-rs/clippy-check@v1 - with: - token: ${{ secrets.GITHUB_TOKEN }} - args: --features std - - name: Deny Clippy warnings (std) - run: cargo clippy --features std -- -D warnings - - name: Deny Clippy warnings (all) - run: cargo clippy --features std,with_ctap1,ed25519,vendor_hid -- -D warnings - - name: Deny Clippy warnings (all, nfc) - run: cargo clippy --features std,with_ctap1,with_nfc,ed25519,vendor_hid -- -D warnings diff --git a/.github/workflows/cargo_fmt.yml b/.github/workflows/cargo_fmt.yml deleted file mode 100644 index 621b5547..00000000 --- a/.github/workflows/cargo_fmt.yml +++ /dev/null @@ -1,96 +0,0 @@ -name: Cargo format -on: - push: - paths: - - 'examples/*.rs' - - 'libraries/**/*.rs' - - 'src/**/*.rs' - - 'tools/**/*.rs' - - 'patches/**' - - '**/Cargo.toml' - - '.cargo/config' - - '!third_party/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - cargo_format: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - name: Install nightly toolchain - run: rustup toolchain install nightly - - name: Install nightly cargo fmt - run: rustup component add rustfmt --toolchain nightly - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Cargo format src/ - uses: actions-rs/cargo@v1 - with: - command: fmt - args: -- --check - - - name: Cargo format libraries/opensk - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/opensk/Cargo.toml -- --check - - - name: Cargo format libraries/opensk/fuzz - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/opensk/fuzz/Cargo.toml -- --check - - - name: Cargo format libraries/cbor - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/cbor/Cargo.toml -- --check - - - name: Cargo format libraries/cbor/fuzz - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/cbor/fuzz/Cargo.toml -- --check - - - name: Cargo format libraries/crypto - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/crypto/Cargo.toml -- --check - - - name: Cargo format libraries/persistent_store - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path libraries/persistent_store/Cargo.toml -- --check - - - name: Cargo format tools/heapviz - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: fmt - args: --manifest-path tools/heapviz/Cargo.toml -- --check - - - name: Cargo format bootloader - uses: actions-rs/cargo@v1 - with: - command: fmt - args: --manifest-path bootloader/Cargo.toml -- --check diff --git a/.github/workflows/cargo_fuzz.yml b/.github/workflows/cargo_fuzz.yml deleted file mode 100644 index c997ce9b..00000000 --- a/.github/workflows/cargo_fuzz.yml +++ /dev/null @@ -1,33 +0,0 @@ ---- -name: Cargo fuzz build -on: - push: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - build_fuzzing: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - toolchain: nightly - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - name: Set up fuzzing - run: ./fuzzing_setup.sh - - - name: Cargo fuzz build - run: cd libraries/opensk && cargo +nightly fuzz build - - name: Cargo fuzz build (libraries/cbor) - run: cd libraries/cbor && cargo +nightly fuzz build - - name: Cargo fuzz build (libraries/persistent_store) - run: cd libraries/persistent_store && cargo +nightly fuzz build diff --git a/.github/workflows/cbor_test.yml b/.github/workflows/cbor_test.yml deleted file mode 100644 index becd1063..00000000 --- a/.github/workflows/cbor_test.yml +++ /dev/null @@ -1,37 +0,0 @@ ---- -name: CBOR tests -on: - push: - paths: - - 'libraries/cbor/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - cbor_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Unit testing of CBOR library (release mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path libraries/cbor/Cargo.toml --release - - - name: Unit testing of CBOR library (debug mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path libraries/cbor/Cargo.toml diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 00000000..0b5815a0 --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,33 @@ +name: Continuous Integration + +on: + push: + branches: + - develop + pull_request: + branches: + - develop + schedule: + - cron: 30 1 * * 2 # every Tuesday at 1:30 UTC + +concurrency: + group: ci-${{ github.ref }} + cancel-in-progress: ${{ github.event_name == 'pull_request' }} + +jobs: + runtests: + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + permissions: + contents: read + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v1 + with: + python-version: "3.10" + - run: ./setup.sh + - run: rustup component add rustfmt --toolchain nightly + - run: rustup component add clippy --toolchain nightly + - run: ./run_desktop_tests.sh diff --git a/.github/workflows/coveralls.yml b/.github/workflows/coveralls.yml index 7a2988c2..6b1439ac 100644 --- a/.github/workflows/coveralls.yml +++ b/.github/workflows/coveralls.yml @@ -20,9 +20,7 @@ jobs: run: rustup show - uses: actions/setup-python@v1 with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel + python-version: "3.10" - name: Set up OpenSK run: ./setup.sh - name: Install llvm tools diff --git a/.github/workflows/crypto_test.yml b/.github/workflows/crypto_test.yml deleted file mode 100644 index 19724351..00000000 --- a/.github/workflows/crypto_test.yml +++ /dev/null @@ -1,43 +0,0 @@ ---- -name: Crypto library tests -on: - push: - paths: - - 'libraries/crypto/**' - pull_request: - types: [opened, synchronize, reopened] - paths: - - 'libraries/crypto/**' - -jobs: - crypto_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - run: echo "RUSTFLAGS=-C target-feature=+aes" >> $GITHUB_ENV - - - name: Unit testing of crypto library (release mode) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/crypto/Cargo.toml --release --features std - - - name: Unit testing of crypto library (debug mode) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/crypto/Cargo.toml --features std diff --git a/.github/workflows/heapviz_test.yml b/.github/workflows/heapviz_test.yml deleted file mode 100644 index 28e0d5d5..00000000 --- a/.github/workflows/heapviz_test.yml +++ /dev/null @@ -1,34 +0,0 @@ ---- -name: Heapviz tool tests -on: - push: - paths: - - 'tools/heapviz/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - heapviz_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Install ncurses - run: sudo apt-get install libncurses-dev - - - name: Check heapviz tool - uses: actions-rs/cargo@v1 - with: - command: check - args: --manifest-path tools/heapviz/Cargo.toml - - - name: Unit testing of heapviz tool (debug mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path tools/heapviz/Cargo.toml - - - name: Unit testing of heapviz tool (release mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path tools/heapviz/Cargo.toml --release diff --git a/.github/workflows/opensk_build.yml b/.github/workflows/opensk_build.yml deleted file mode 100644 index 349c10e3..00000000 --- a/.github/workflows/opensk_build.yml +++ /dev/null @@ -1,40 +0,0 @@ ---- -name: OpenSK build -on: - push: - pull_request: - types: [opened, synchronize, reopened] - -jobs: - build_ctap2: - strategy: - matrix: - os: [ubuntu-latest, macos-latest] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Building sha256sum tool - uses: actions-rs/cargo@v1 - with: - command: build - args: --manifest-path third_party/tock/tools/sha256sum/Cargo.toml - - - name: Building OpenSK - uses: actions-rs/cargo@v1 - with: - command: build - args: --release --target=thumbv7em-none-eabi --features with_ctap1,vendor_hid - - name: Compute SHA-256 sum - run: ./third_party/tock/tools/sha256sum/target/debug/sha256sum target/thumbv7em-none-eabi/release/ctap2 diff --git a/.github/workflows/opensk_test.yml b/.github/workflows/opensk_test.yml deleted file mode 100644 index 2aae23f9..00000000 --- a/.github/workflows/opensk_test.yml +++ /dev/null @@ -1,56 +0,0 @@ ---- -name: OpenSK tests -on: - push: - paths: - - 'src/**/*.rs' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - ctap2_test: - name: CTAP2 unit tests - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v2 - with: - submodules: "true" - - name: Install Rust toolchain - run: rustup show - - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel - - name: Set up OpenSK - run: ./setup.sh - - - name: Unit testing of CTAP2 (release mode) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/opensk/Cargo.toml --release --features std - - - name: Unit testing of CTAP2 (debug mode) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/opensk/Cargo.toml --features std - - - name: Unit testing of CTAP2 (release mode + all features) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/opensk/Cargo.toml --release --features std,debug_ctap,with_ctap1,vendor_hid,ed25519 - - - name: Unit testing of CTAP2 (debug mode + all features) - uses: actions-rs/cargo@v1 - with: - toolchain: nightly - command: test - args: --manifest-path libraries/opensk/Cargo.toml --features std,debug_ctap,with_ctap1,vendor_hid,ed25519 - diff --git a/.github/workflows/persistent_store_test.yml b/.github/workflows/persistent_store_test.yml deleted file mode 100644 index e6cbe0fe..00000000 --- a/.github/workflows/persistent_store_test.yml +++ /dev/null @@ -1,29 +0,0 @@ ---- -name: Persistent store tests -on: - push: - paths: - - 'libraries/peristent_store/**' - pull_request: - types: [opened, synchronize, reopened] - -jobs: - persistent_store_test: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - - name: Install Rust toolchain - run: rustup show - - - name: Unit testing of Persistent store library (release mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path libraries/persistent_store/Cargo.toml --release --features=std - - - name: Unit testing of Persistent store library (debug mode) - uses: actions-rs/cargo@v1 - with: - command: test - args: --manifest-path libraries/persistent_store/Cargo.toml --features=std diff --git a/.github/workflows/python.yml b/.github/workflows/python.yml deleted file mode 100644 index 8a362c48..00000000 --- a/.github/workflows/python.yml +++ /dev/null @@ -1,46 +0,0 @@ -name: pylint -on: - push: - paths: - - '**/*.py' - - '.pylintrc' - - '!third_party/**' - pull_request: - types: [opened, synchronize, reopened] -jobs: - pylint: - runs-on: ubuntu-latest - strategy: - matrix: - python-version: ['3.7', '3.8', '3.9', '3.10', '3.11'] - steps: - - uses: actions/checkout@v2 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v1 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - pip install 'tockloader==1.5' pylint - - name: Register matcher - run: echo ::add-matcher::./.github/python_matcher.json - - name: Test code with pylint - run: ./tools/run_pylint.sh - - yapf: - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v2 - - name: Set up Python 3.7 - uses: actions/setup-python@v1 - with: - python-version: 3.7 - - name: Install dependencies - run: | - python -m pip install --upgrade pip setuptools wheel - pip install 'yapf>=0.30.0' tockloader - - name: Test code formatting with yapf - run: | - echo ::add-matcher::./.github/python_matcher.json - yapf --style=yapf --recursive --exclude third_party --diff . diff --git a/.github/workflows/reproducible.yml b/.github/workflows/reproducible.yml index 46fd5b82..fbb4adc0 100644 --- a/.github/workflows/reproducible.yml +++ b/.github/workflows/reproducible.yml @@ -20,9 +20,7 @@ jobs: run: rustup show - uses: actions/setup-python@v1 with: - python-version: 3.7 - - name: Install Python dependencies - run: python -m pip install --upgrade pip setuptools wheel + python-version: "3.10" - name: Set up OpenSK run: ./setup.sh diff --git a/docs/customization.md b/docs/customization.md index 6d6b82c9..7ee7c605 100644 --- a/docs/customization.md +++ b/docs/customization.md @@ -85,7 +85,7 @@ OpenSK is fuzzed with the [OSS-Fuzz](https://github.com/google/oss-fuzz) project. You can also run fuzzing locally. First install: ```shell -cargo +stable install cargo-fuzz --version 0.10.2 +./fuzzing_setup.sh ``` Then choose a fuzz target from `fuzz/fuzz_targets/`, e.g.: diff --git a/libraries/opensk/src/api/crypto/rust_crypto.rs b/libraries/opensk/src/api/crypto/rust_crypto.rs index a70199c4..c03932ad 100644 --- a/libraries/opensk/src/api/crypto/rust_crypto.rs +++ b/libraries/opensk/src/api/crypto/rust_crypto.rs @@ -205,7 +205,6 @@ impl ecdsa::Signature for SoftwareEcdsaSignature { Some(SoftwareEcdsaSignature { signature }) } - #[cfg(feature = "std")] fn to_slice(&self, bytes: &mut [u8; EC_SIGNATURE_SIZE]) { bytes.copy_from_slice(&self.signature.to_bytes()); } diff --git a/libraries/opensk/src/api/private_key.rs b/libraries/opensk/src/api/private_key.rs index 39745e88..c476a9a6 100644 --- a/libraries/opensk/src/api/private_key.rs +++ b/libraries/opensk/src/api/private_key.rs @@ -161,10 +161,10 @@ impl PrivateKey { let wrapped_bytes = extract_byte_string(array.pop().unwrap())?; let key_bytes = aes256_cbc_decrypt::(wrap_key, &wrapped_bytes, true)?; match SignatureAlgorithm::try_from(array.pop().unwrap())? { - SignatureAlgorithm::Es256 => PrivateKey::new_ecdsa_from_bytes(&*key_bytes) + SignatureAlgorithm::Es256 => PrivateKey::new_ecdsa_from_bytes(&key_bytes) .ok_or(Ctap2StatusCode::CTAP2_ERR_INVALID_CBOR), #[cfg(feature = "ed25519")] - SignatureAlgorithm::Eddsa => PrivateKey::new_ed25519_from_bytes(&*key_bytes) + SignatureAlgorithm::Eddsa => PrivateKey::new_ed25519_from_bytes(&key_bytes) .ok_or(Ctap2StatusCode::CTAP2_ERR_INVALID_CBOR), _ => Err(Ctap2StatusCode::CTAP2_ERR_INVALID_CBOR), } diff --git a/libraries/persistent_store/fuzz/examples/store.rs b/libraries/persistent_store/fuzz/examples/store.rs index be9240be..f163a9eb 100644 --- a/libraries/persistent_store/fuzz/examples/store.rs +++ b/libraries/persistent_store/fuzz/examples/store.rs @@ -13,8 +13,7 @@ // limitations under the License. use fuzz_store::{fuzz, StatKey, Stats}; -use std::io::Write; -use std::io::{stdout, Read}; +use std::io::{stdout, Read, Write}; use std::path::Path; fn usage(program: &str) { diff --git a/libraries/persistent_store/src/driver.rs b/libraries/persistent_store/src/driver.rs index 143f5f73..0a5a21d6 100644 --- a/libraries/persistent_store/src/driver.rs +++ b/libraries/persistent_store/src/driver.rs @@ -490,7 +490,7 @@ impl StoreDriverOn { /// Checks that the given entries are wiped from the storage. fn check_deleted(&self, deleted: &[StoreHandle]) -> Result<(), StoreInvariant> { for handle in deleted { - let value = self.store.inspect_value(&handle); + let value = self.store.inspect_value(handle); if !value.iter().all(|&x| x == 0x00) { return Err(StoreInvariant::NotWiped { key: handle.get_key(), diff --git a/libraries/persistent_store/src/format.rs b/libraries/persistent_store/src/format.rs index 5690b934..d28d0cd3 100644 --- a/libraries/persistent_store/src/format.rs +++ b/libraries/persistent_store/src/format.rs @@ -187,7 +187,7 @@ impl Format { word_size == WORD_SIZE && page_size % word_size == 0 && (MIN_PAGE_SIZE * word_size <= page_size && page_size <= MAX_PAGE_SIZE) - && (MIN_NUM_PAGES <= num_pages && num_pages <= MAX_PAGE_INDEX + 1) + && (MIN_NUM_PAGES..=MAX_PAGE_INDEX + 1).contains(&num_pages) && max_word_writes >= 2 && max_page_erases <= MAX_ERASE_CYCLE } diff --git a/libraries/persistent_store/src/fragment.rs b/libraries/persistent_store/src/fragment.rs index 661d5dbf..9a1adaf7 100644 --- a/libraries/persistent_store/src/fragment.rs +++ b/libraries/persistent_store/src/fragment.rs @@ -146,7 +146,7 @@ pub fn delete(store: &mut Store, keys: &impl Keys) -> StoreResult< /// The handles are truncated to the keys that are present. fn get_handles(store: &Store, keys: &impl Keys) -> StoreResult> { let keys_len = keys.len(); - let mut handles: Vec> = vec![None; keys_len as usize]; + let mut handles: Vec> = vec![None; keys_len]; for handle in store.iter()? { let handle = handle?; let pos = match keys.pos(handle.get_key()) { diff --git a/libraries/persistent_store/src/linear.rs b/libraries/persistent_store/src/linear.rs index 2d190437..3c40a83b 100644 --- a/libraries/persistent_store/src/linear.rs +++ b/libraries/persistent_store/src/linear.rs @@ -148,7 +148,7 @@ impl Linear { value = &value[len..]; index.byte += len; // Write the unaligned end if needed. - if value.len() > 0 { + if !value.is_empty() { let mut word = self.storage.read_slice(index, word_size)?.into_owned(); word[..value.len()].copy_from_slice(value); self.storage.write_slice(index, &word)?; diff --git a/libraries/persistent_store/src/store.rs b/libraries/persistent_store/src/store.rs index b946878b..4f178034 100644 --- a/libraries/persistent_store/src/store.rs +++ b/libraries/persistent_store/src/store.rs @@ -19,12 +19,10 @@ use crate::format::{ Word, WordState, }; #[cfg(feature = "std")] -pub use crate::model::{StoreModel, StoreOperation}; -use crate::{usize_to_nat, Nat, Storage, StorageError, StorageIndex}; +pub use crate::model::StoreOperation; #[cfg(feature = "std")] -pub use crate::{ - BufferStorage, StoreDriver, StoreDriverOff, StoreDriverOn, StoreInterruption, StoreInvariant, -}; +pub use crate::BufferStorage; +use crate::{usize_to_nat, Nat, Storage, StorageError, StorageIndex}; use alloc::borrow::Cow; use alloc::boxed::Box; use alloc::vec::Vec; @@ -245,7 +243,7 @@ impl Store { } /// Iterates over the entries. - pub fn iter<'a>(&'a self) -> StoreResult> { + pub fn iter(&self) -> StoreResult> { let head = or_invalid(self.head)?; Ok(Box::new(or_invalid(self.entries.as_ref())?.iter().map( move |&offset| { @@ -794,7 +792,7 @@ impl Store { /// Continues a transaction after it has been written. fn transaction_apply(&mut self, sorted_keys: &[Nat], marker: Position) -> StoreResult<()> { - self.delete_keys(&sorted_keys, marker)?; + self.delete_keys(sorted_keys, marker)?; self.set_padding(marker)?; let end = or_invalid(self.head)? + self.format.window_size(); let mut pos = marker + 1; diff --git a/requirements.txt b/requirements.txt index da13ac6d..e1ef7ea6 100644 --- a/requirements.txt +++ b/requirements.txt @@ -11,4 +11,5 @@ fido2 >= 1.0.0 # Tests pylint +yapf hid diff --git a/run_desktop_tests.sh b/run_desktop_tests.sh index dd212291..9f23d9ab 100755 --- a/run_desktop_tests.sh +++ b/run_desktop_tests.sh @@ -15,124 +15,101 @@ set -ex -echo "Checking formatting..." -cargo fmt -- --check -cd libraries/opensk -cargo +nightly fmt -- --check -cd ../.. -cd libraries/cbor -cargo fmt -- --check -cd ../.. -cd libraries/crypto -cargo fmt -- --check -cd ../.. -cd libraries/persistent_store -cargo fmt -- --check -cd ../.. -cd tools/heapviz -cargo fmt -- --check -cd ../.. -cd bootloader -cargo fmt -- --check -cd .. +./fuzzing_setup.sh +# Excludes std +MOST_FEATURES=config_command,debug_allocations,debug_ctap,panic_console,verbose,with_ctap1,vendor_hid,ed25519 -echo "Running Clippy lints..." -cargo clippy --lib --tests --bins --benches --features std -- -D warnings -cargo clippy --lib --tests --bins --benches --features std,with_ctap1,ed25519,vendor_hid -- -D warnings -cargo clippy --lib --tests --bins --benches --features std,with_ctap1,with_nfc,ed25519,vendor_hid -- -D warnings - -echo "Building sha256sum tool..." -cargo build --manifest-path third_party/tock/tools/sha256sum/Cargo.toml -echo "Checking that heapviz tool builds properly..." -cargo build --manifest-path tools/heapviz/Cargo.toml -echo "Testing heapviz tool..." -cargo test --manifest-path tools/heapviz/Cargo.toml - -echo "Checking that CTAP2 builds properly..." +echo "Checking that OpenSK builds properly..." cargo check --release --target=thumbv7em-none-eabi -cargo check --release --target=thumbv7em-none-eabi --features with_ctap1 -cargo check --release --target=thumbv7em-none-eabi --features vendor_hid -cargo check --release --target=thumbv7em-none-eabi --features ed25519 +cargo check --release --target=thumbv7em-none-eabi --features config_command +cargo check --release --target=thumbv7em-none-eabi --features debug_allocations cargo check --release --target=thumbv7em-none-eabi --features debug_ctap cargo check --release --target=thumbv7em-none-eabi --features panic_console -cargo check --release --target=thumbv7em-none-eabi --features debug_allocations cargo check --release --target=thumbv7em-none-eabi --features verbose -cargo check --release --target=thumbv7em-none-eabi --features debug_ctap,with_ctap1 -cargo check --release --target=thumbv7em-none-eabi --features debug_ctap,with_ctap1,vendor_hid,ed25519,panic_console,debug_allocations,verbose - -echo "Checking that examples build properly..." +cargo check --release --target=thumbv7em-none-eabi --features with_ctap1 +cargo check --release --target=thumbv7em-none-eabi --features with_nfc +cargo check --release --target=thumbv7em-none-eabi --features vendor_hid +cargo check --release --target=thumbv7em-none-eabi --features ed25519 +cargo check --release --target=thumbv7em-none-eabi --features rust_crypto +cargo check --release --target=thumbv7em-none-eabi --features "$MOST_FEATURES" cargo check --release --target=thumbv7em-none-eabi --examples cargo check --release --target=thumbv7em-none-eabi --examples --features with_nfc +cargo check --release --target=thumbv7em-none-eabi --manifest-path bootloader/Cargo.toml +cargo check --release --manifest-path tools/heapviz/Cargo.toml -echo "Checking that bootloader builds properly..." -cd bootloader -cargo check --release --target=thumbv7em-none-eabi -cd .. +echo "Checking Rust formatting..." +cargo fmt -- --check +cargo +nightly fmt --manifest-path libraries/opensk/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/opensk/fuzz/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/cbor/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/cbor/fuzz/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/persistent_store/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/persistent_store/fuzz/Cargo.toml -- --check +cargo +nightly fmt --manifest-path libraries/crypto/Cargo.toml -- --check +cargo +nightly fmt --manifest-path tools/heapviz/Cargo.toml -- --check +cargo +nightly fmt --manifest-path bootloader/Cargo.toml -- --check + +echo "Checking Python formatting..." +py_virtual_env/bin/pylint --score=n `git ls-files --deduplicate --exclude-standard --full-name '*.py'` +py_virtual_env/bin/yapf --style=yapf --recursive --exclude py_virtual_env --exclude third_party --diff . -echo "Checking that fuzz targets build properly..." -# Uses nightly since our old toolchain causes problems. -cd libraries/opensk -cargo +nightly fuzz build -cd ../.. -cd libraries/cbor -cargo +nightly fuzz build -cd ../.. -cd libraries/persistent_store -cargo +nightly fuzz build -cd ../.. +echo "Running Clippy lints..." +cargo clippy --lib --tests --bins --benches --features std -- -D warnings +cargo clippy --lib --tests --bins --benches --features std,"$MOST_FEATURES" -- -D warnings +(cd libraries/opensk && cargo +nightly clippy --features std -- -D warnings) +(cd libraries/opensk && cargo +nightly clippy --features std,config_command,debug_ctap,with_ctap1,vendor_hid,ed25519,rust_crypto -- -D warnings) +(cd libraries/cbor && cargo +nightly clippy -- -D warnings) +# Uncomment when persistent store is fixed: +# (cd libraries/persistent_store && cargo +nightly clippy --features std -- -D warnings) +# Probably not worth fixing: +# (cd libraries/crypto && cargo +nightly clippy --features std -- -D warnings) + +echo "Checking that fuzz targets..." +(cd libraries/opensk && cargo +nightly fuzz check) +(cd libraries/cbor && cargo +nightly fuzz check) +(cd libraries/persistent_store && cargo +nightly fuzz check) + +echo "Building sha256sum tool..." +cargo build --manifest-path third_party/tock/tools/sha256sum/Cargo.toml echo "Checking that CTAP2 builds and links properly (1 set of features)..." -cargo build --release --target=thumbv7em-none-eabi --features with_ctap1 +cargo build --release --target=thumbv7em-none-eabi --features config_command,with_ctap1 ./third_party/tock/tools/sha256sum/target/debug/sha256sum target/thumbv7em-none-eabi/release/ctap2 -echo "Checking that supported boards build properly..." +echo "Running OpenSK library unit tests..." +cd libraries/opensk +cargo +nightly test --features std +cargo +nightly test --features std,config_command,with_ctap1 +cargo +nightly test --all-features +cd ../.. + +echo "Running other unit tests..." +cargo test --lib --tests --bins --benches --features std +cargo test --lib --tests --bins --benches --all-features +cargo +nightly test --manifest-path libraries/cbor/Cargo.toml +cargo +nightly test --manifest-path libraries/persistent_store/Cargo.toml --features std +# Running release mode to speed up. This library is legacy anyway. +cargo +nightly test --manifest-path libraries/crypto/Cargo.toml --features std --release +cargo +nightly test --manifest-path tools/heapviz/Cargo.toml + +echo "Checking that boards build properly..." make -C third_party/tock/boards/nordic/nrf52840dk_opensk make -C third_party/tock/boards/nordic/nrf52840_dongle_opensk - -echo "Checking that other boards build properly..." make -C third_party/tock/boards/nordic/nrf52840_dongle_dfu make -C third_party/tock/boards/nordic/nrf52840_mdk_dfu -echo "Checking deployment of supported boards..." +echo "Checking deployment of boards..." ./deploy.py --board=nrf52840dk_opensk --no-app --programmer=none ./deploy.py --board=nrf52840_dongle_opensk --no-app --programmer=none - -echo "Checking deployment of other boards..." ./deploy.py --board=nrf52840_dongle_dfu --no-app --programmer=none ./deploy.py --board=nrf52840_mdk_dfu --no-app --programmer=none -if [ -z "${TRAVIS_OS_NAME}" -o "${TRAVIS_OS_NAME}" = "linux" ] -then - echo "Running unit tests on the desktop (release mode)..." - cargo test --lib --tests --bins --benches --release --features std - cargo test --lib --tests --bins --benches --release --all-features - cd libraries/cbor - cargo test --release - cd ../.. - cd libraries/persistent_store - cargo test --release --features std - cd ../.. - - echo "Running unit tests on the desktop (debug mode)..." - cargo test --lib --tests --bins --benches --features std - cargo test --lib --tests --bins --benches --all-features - cd libraries/cbor - cargo test - cd ../.. - cd libraries/persistent_store - cargo test --features std - cd ../.. - - cd libraries/opensk - echo "Running CTAP library unit tests (release mode)..." - cargo +nightly test --release --features std - echo "Running CTAP library unit tests (release mode + all features)..." - cargo +nightly test --release --features std,debug_ctap,with_ctap1,vendor_hid,ed25519 - echo "Running CTAP library unit tests (release mode + experimental rust crypto)..." - cargo +nightly test --release --features std,debug_ctap,with_ctap1,vendor_hid,ed25519,rust_crypto - - echo "Running CTAP library unit tests (debug mode)..." - cargo +nightly test --features std - echo "Running CTAP library unit tests (debug mode + all features)..." - cargo +nightly test --features std,debug_ctap,with_ctap1,vendor_hid,ed25519 -fi +echo "Check app deployment" +./deploy.py --board=nrf52840dk_opensk --programmer=none --opensk +./deploy.py --board=nrf52840dk_opensk --programmer=none --crypto_bench +./deploy.py --board=nrf52840dk_opensk --programmer=none --store_latency +./deploy.py --board=nrf52840dk_opensk --programmer=none --erase_storage +./deploy.py --board=nrf52840dk_opensk --programmer=none --panic_test +./deploy.py --board=nrf52840dk_opensk --programmer=none --oom_test +./deploy.py --board=nrf52840dk_opensk --programmer=none --console_test +./deploy.py --board=nrf52840dk_opensk --programmer=none --nfct_test --nfc diff --git a/setup.sh b/setup.sh index ba0a7d67..51b5e73d 100755 --- a/setup.sh +++ b/setup.sh @@ -17,6 +17,7 @@ export TERM=${TERM:-vt100} done_text="$(tput bold)DONE.$(tput sgr0)" PY_VENV_NAME=py_virtual_env +PYTHON="$PY_VENV_NAME"/bin/python PIP="$PY_VENV_NAME"/bin/pip set -e @@ -34,6 +35,7 @@ check_command () { } check_command rustup " Follow the steps under https://rustup.rs/ to install it." python3 -m venv "$PY_VENV_NAME" +"$PYTHON" -m pip install --upgrade pip setuptools wheel check_command "$PIP" # Ensure we have certificates, keys, etc. so that the tests can run diff --git a/tools/run_pylint.sh b/tools/run_pylint.sh deleted file mode 100755 index 2055078a..00000000 --- a/tools/run_pylint.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env bash -# Copyright 2019 Google LLC -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - -# Ensure we are at the project root directory -cd $(readlink -f $(dirname $0))/.. - -export PATH="py_virtual_env/bin:$PATH" - -pylint --score=n `git ls-files --deduplicate --exclude-standard --full-name '*.py'`