From 4ea2a40acde452f0a7b0905d458408dc4551d06b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 23 Nov 2023 15:50:41 +0000 Subject: [PATCH 1/6] utils/dependency_cross_matcher.py: Print error info MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Print error information on the dependency mismatch checker instead of doing so in the ci.sh script. This simplifies the logic and reduces maintainability efforts. Signed-off-by: Tomás González --- tests/ci.sh | 5 ----- utils/dependency_cross_matcher.py | 6 ++++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/tests/ci.sh b/tests/ci.sh index 90aac4a..732a251 100755 --- a/tests/ci.sh +++ b/tests/ci.sh @@ -35,11 +35,6 @@ done ######################### if [ "$MISMATCHER" = "True" ]; then python3 $(pwd)/utils/dependency_cross_matcher.py --deps_dir $(pwd) - mismatcher_result=$? - if [ "$mismatcher_result" -ne 0 ]; then - error_msg "Found dependencies version mismatches" - fi - exit 0 fi diff --git a/utils/dependency_cross_matcher.py b/utils/dependency_cross_matcher.py index fabbe72..6bcc236 100644 --- a/utils/dependency_cross_matcher.py +++ b/utils/dependency_cross_matcher.py @@ -54,6 +54,9 @@ def main(argv=[], prog_name=''): mismatches = get_deps_with_more_than_1v(mismatches) + print('---------------------exceptions-----------------------\n\n') + print_deps(exceptions) + print('---------------------mistmatches----------------------\n\n') print_deps(mismatches) @@ -65,8 +68,7 @@ def main(argv=[], prog_name=''): 'yasna': ['v0.4.0', 'v0.5.2'], } - if exceptions != mismatches: - return 1 + assert exceptions == mismatches, "Found dependencies version mismatches in parsec-tool" return 0 From 361a6171187f9d48521edd2032634a4293464d68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 23 Nov 2023 15:08:47 +0000 Subject: [PATCH 2/6] dependency_cross_matcher.py: Compatibilize parsec and parsec-tool MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parsec and parsec-tool share some dependencies. In those shared dependencies, the used versions should be the same. * Modify utils/dependency_cross_matcher.py to cover the case of comparing parsec and parsec-tool dependencies as well as the previous case of checking mismatches in parsec-tool dependencies. Signed-off-by: Tomás González --- tests/ci.sh | 2 ++ utils/dependency_cross_matcher.py | 57 ++++++++++++++++++++++++------- 2 files changed, 46 insertions(+), 13 deletions(-) diff --git a/tests/ci.sh b/tests/ci.sh index 732a251..453c844 100755 --- a/tests/ci.sh +++ b/tests/ci.sh @@ -34,7 +34,9 @@ done # Dependency mismatcher # ######################### if [ "$MISMATCHER" = "True" ]; then + python3 $(pwd)/utils/dependency_cross_matcher.py --deps_dir $(pwd) + exit 0 fi diff --git a/utils/dependency_cross_matcher.py b/utils/dependency_cross_matcher.py index 6bcc236..9aa8de3 100644 --- a/utils/dependency_cross_matcher.py +++ b/utils/dependency_cross_matcher.py @@ -5,8 +5,10 @@ import sys -def run_cargo_tree(path): - cmd = 'cargo tree --all-features -d' +def run_cargo_tree(path, flags=None): + cmd = 'cargo tree' + if flags is not None: + cmd += ' ' + flags prev_dir = os.getcwd() os.chdir(os.path.join(path)) return subprocess.check_output(cmd, shell=True).decode() @@ -43,14 +45,46 @@ def main(argv=[], prog_name=''): parser = argparse.ArgumentParser(prog='DependencyCrossmatcher', description='Checks the version mismatches for dependencies ' 'in Cargo based repositories') + parser.add_argument("-c", "--compare", action='store_true', + help='Check for mismatches between 2 repositories') parser.add_argument('--deps_dir', required=True, - help='Existing directory that contains the Cargo.toml for analyzing' + nargs='+', + help='Existing directories that contain Cargo.toml for analyzing ' 'dependencies') args = parser.parse_args() - mismatches = run_deps_mismatcher(run_cargo_tree(args.deps_dir)) - print_deps(mismatches) + mismatches = dict() + parsec_tool_flags = '--all-features -d' + + if args.compare: + exceptions = { + 'bindgen': ['v0.66.1', 'v0.57.0'], + } + parsec_repo, parsec_tool_repo = args.deps_dir + parsec_flags = '--all-features' + ' ' + parsec_flags = '--features tss-esapi/generate-bindings,cryptoki/generate-bindings -d' + mismatches_parsec = run_deps_mismatcher(run_cargo_tree(parsec_repo, parsec_flags)) + mismatches_parsec_tool = run_deps_mismatcher(run_cargo_tree(parsec_tool_repo, + parsec_tool_flags) + ) + + # Dependencies that are common to both parsec_repo and parsec_tool_repo repos + common_deps = list(set(mismatches_parsec.keys()) & set(mismatches_parsec_tool.keys())) + for dep in common_deps: + # Symmetric difference of repos parsec_repo and parsec_tool_repo + mistmatch = list(set(mismatches_parsec[dep]) ^ set(mismatches_parsec_tool[dep])) + if len(mistmatch) > 0: + mismatches[dep] = mistmatch + else: + exceptions = { + 'base64': ['v0.13.1', 'v0.21.4'], + 'bitflags': ['v1.3.2', 'v2.4.1'], + 'nom': ['v5.1.3', 'v7.1.3'], + 'syn': ['v1.0.109', 'v2.0.38'], + 'yasna': ['v0.4.0', 'v0.5.2'], + } + mismatches = run_deps_mismatcher(run_cargo_tree(args.deps_dir[0], parsec_tool_flags)) mismatches = get_deps_with_more_than_1v(mismatches) @@ -60,15 +94,12 @@ def main(argv=[], prog_name=''): print('---------------------mistmatches----------------------\n\n') print_deps(mismatches) - exceptions = { - 'base64': ['v0.13.1', 'v0.21.4'], - 'bitflags': ['v1.3.2', 'v2.4.1'], - 'nom': ['v5.1.3', 'v7.1.3'], - 'syn': ['v1.0.109', 'v2.0.38'], - 'yasna': ['v0.4.0', 'v0.5.2'], - } + if not args.compare: + errormsg = "Found dependencies version mismatches in parsec-tool" + else: + errormsg = "Found dependencies version mismatches between parsec and parsec-tool" - assert exceptions == mismatches, "Found dependencies version mismatches in parsec-tool" + assert exceptions == mismatches, errormsg return 0 From 270ea5eb56c4a94355574247ee569a3067777091 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 23 Nov 2023 15:54:54 +0000 Subject: [PATCH 3/6] ci.yml,ci/nightly: Move the mismatcher job to ci.yml MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit We could easily discover mismatches in a normal CI run instead of in a nightly run. * Update the mismatcher job so that it's run on each PR instead of on a nightly. Signed-off-by: Tomás González --- .github/workflows/ci.yml | 17 +++++++++++++++++ .github/workflows/nightly.yml | 17 ----------------- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 5a34965..4bcbd52 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -40,3 +40,20 @@ jobs: args: -v -r *.md - name: Fail if there were link errors run: exit ${{ steps.lc.outputs.exit_code }} + + mismatcher: + name: Check for mismatched dependencies (those that have more than one version) + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + with: + ref: "${{ github.event.inputs.rev }}" + - name: Install latest Rust + uses: actions-rust-lang/setup-rust-toolchain@v1 + with: + toolchain: stable + rustflags: "" + - name: Execute CI script + uses: ./.github/actions/ci_script + with: + ci-flags: "mismatcher" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 5a36b45..7b66771 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -42,20 +42,3 @@ jobs: run: cargo install cargo-audit - name: Execute cargo audit run: cargo audit - - mismatcher: - name: Check for mismatched dependencies (those that have more than one version) - runs-on: ubuntu-latest - steps: - - uses: actions/checkout@v3 - with: - ref: "${{ github.event.inputs.rev }}" - - name: Install Rust MSRV - uses: actions-rust-lang/setup-rust-toolchain@v1 - with: - toolchain: stable - rustflags: "" - - name: Execute CI script - uses: ./.github/actions/ci_script - with: - ci-flags: "mismatcher" From 029d98b42a67fce6b008dcab6c8f8ff5eba7dd71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Mon, 27 Nov 2023 13:48:40 +0000 Subject: [PATCH 4/6] ci.yml: Check parsec and parsec-tool dependency mismatches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use the newly added --compare option in utils/dependency_cross_matcher.py to compare dependencies between parsec and parsec-tool to spot common dependencies that are being used with different versions. Signed-off-by: Tomás González --- tests/ci.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/ci.sh b/tests/ci.sh index 453c844..6c12174 100755 --- a/tests/ci.sh +++ b/tests/ci.sh @@ -37,6 +37,8 @@ if [ "$MISMATCHER" = "True" ]; then python3 $(pwd)/utils/dependency_cross_matcher.py --deps_dir $(pwd) + python3 $(pwd)/utils/dependency_cross_matcher.py -c --deps_dir $(pwd)/parsec $(pwd) + exit 0 fi From 52d9b1707799d317e05d10d4f7c3bbb364a78d4d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Thu, 23 Nov 2023 16:10:14 +0000 Subject: [PATCH 5/6] Cargo.lock: Update rustix to its latest version MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Keep rustix updated and avoid a dependency mismatch with parsec. Please check: https://github.com/parallaxsecond/parsec/pull/741 Signed-off-by: Tomás González --- Cargo.lock | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 89af22d..041be59 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -1097,9 +1097,9 @@ dependencies = [ [[package]] name = "rustix" -version = "0.38.19" +version = "0.38.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "745ecfa778e66b2b63c88a61cb36e0eea109e803b0b86bf9879fbc77c70e86ed" +checksum = "2b426b0506e5d50a7d8dafcf2e81471400deb602392c7dd110815afb4eaf02a3" dependencies = [ "bitflags 2.4.1", "errno", From 0d6b9deff594b9f818436e45701f58db1be13a93 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tom=C3=A1s=20Gonz=C3=A1lez?= Date: Fri, 24 Nov 2023 17:29:29 +0000 Subject: [PATCH 6/6] utils/dependency_cross_matcher: Add license and description comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Tomás González --- utils/dependency_cross_matcher.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/utils/dependency_cross_matcher.py b/utils/dependency_cross_matcher.py index 9aa8de3..5dfff22 100644 --- a/utils/dependency_cross_matcher.py +++ b/utils/dependency_cross_matcher.py @@ -1,3 +1,9 @@ +# Copyright 2023 Contributors to the Parsec project. +# SPDX-License-Identifier: Apache-2.0 + +# Checks the version mismatches for dependencies in Cargo based repositories: +# * In parsec-tool itself +# * Between parsec and parsec-tool import argparse import re import os