Merge branch 'main' into 2592-mapper-bridge

thin-edge · Mar 15, 2024 · 9203378 · 9203378
2 parents dcd4329 + dc8fcb5
commit 9203378
Show file tree

Hide file tree

Showing 9 changed files with 729 additions and 36 deletions.
diff --git a/.github/workflows/build-workflow.yml b/.github/workflows/build-workflow.yml
@@ -8,11 +8,6 @@ on:
   merge_group:
   workflow_dispatch:
     inputs:
-      disable_cache:
-        description: Disable rust caching
-        type: boolean
-        required: false
-        default: false
       include:
         description: Only run tests matching tests with the given tags
         type: string
@@ -23,6 +18,12 @@ on:
         type: string
         required: false
         default: "10"
+      run_rust_tests:
+        description: Run Rust tests (generally this is not required as the PR check will run full Rust tests)
+        type: boolean
+        required: false
+        default: false
+
   # Use a manual approval process before PR's are given access to
   # the secrets which are required to run the integration tests.
   # The PR code should be manually approved to see if it can be trusted.
@@ -33,45 +34,95 @@ on:
 
 env:
   CARGO_TERM_COLOR: always
-  RUSTFLAGS: -D warnings
 
 jobs:
   build:
-    name: Build ${{ matrix.job.target }}
-    runs-on: ubuntu-20.04
+    name: Build ${{ matrix.target }}
+    runs-on: ${{ matrix.host_os }}
     strategy:
       fail-fast: false
       matrix:
-        job:
-          - { target: x86_64-unknown-linux-musl, cache: true }
-          - { target: aarch64-unknown-linux-musl, cache: true }
-          - { target: arm-unknown-linux-musleabihf, cache: true }
-          - { target: armv7-unknown-linux-musleabihf, cache: true }
+        # Note: Targets which requires a non-default rust_channel
+        # then they should not be included in the .target list, but
+        # rather in the include section.
+        # This is just how Github processes matrix jobs
+        target:
+          - aarch64-unknown-linux-musl
+          - armv7-unknown-linux-musleabihf
+          - arm-unknown-linux-musleabihf
+          - arm-unknown-linux-musleabi
+          - armv5te-unknown-linux-musleabi
+          - x86_64-unknown-linux-musl
+          - i686-unknown-linux-musl
+          - riscv64gc-unknown-linux-gnu
+          - x86_64-apple-darwin
+        mode:
+          - --release
+
+        rust_channel:
+          - "1.70"
+
+        include:
+          - target: aarch64-unknown-linux-musl
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: armv7-unknown-linux-musleabihf
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: arm-unknown-linux-musleabihf
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: arm-unknown-linux-musleabi
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: armv5te-unknown-linux-musleabi
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: x86_64-unknown-linux-musl
+            host_os: ubuntu-22.04
+
+          - target: i686-unknown-linux-musl
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: riscv64gc-unknown-linux-gnu
+            host_os: ubuntu-22.04
+            cargo_options: --no-run
+
+          - target: aarch64-apple-darwin
+            mode: --release
+            rust_channel: "1.72" # ahash uses "stdsimd" feature which was stabilized in 1.72, https://github.com/tkaitchuck/aHash/issues/195
+            host_os: macos-14
+            cargo_options: --no-run
+
+          - target: x86_64-apple-darwin
+            host_os: macos-14
 
     steps:
+      - if: ${{ contains(matrix.host_os, 'ubuntu') }}
+        run: sudo apt-get update -y
       - name: Checkout
         uses: actions/checkout@v4
         with:
           ref: ${{ github.event_name == 'pull_request_target' && github.event.pull_request.head.sha || '' }}
           fetch-depth: 0
 
-      - name: Retrieve MSRV from workspace Cargo.toml
-        id: rust_version
-        uses: SebRollen/toml-action@v1.2.0
-        with:
-          file: Cargo.toml
-          field: "workspace.package.rust-version"
-
-      - name: Enable toolchain via github action
-        uses: dtolnay/rust-toolchain@master
-        with:
-          toolchain: ${{ steps.rust_version.outputs.value }}
-          targets: ${{ matrix.job.target }}
+      - run: mk/install-build-tools.sh +${{ matrix.rust_channel }} --target=${{ matrix.target }}
+        shell: sh
+      - name: Build
+        run: |
+          . ./ci/build_scripts/version.sh
+          mk/cargo.sh +${{ matrix.rust_channel }} build --target=${{ matrix.target }} ${{ matrix.mode }}
 
-      - name: Enable cache
-        if: ${{ matrix.job.cache && !inputs.disable_cache }}
-        # https://github.com/marketplace/actions/rust-cache
-        uses: Swatinem/rust-cache@v2
+      - if: ${{ inputs.run_rust_tests && !contains(matrix.host_os, 'windows') }}
+        name: Test
+        run: |
+          mk/cargo.sh +${{ matrix.rust_channel }} test -vv --target=${{ matrix.target }} ${{ matrix.cargo_options }} ${{ matrix.mode }} --no-fail-fast --locked --all-features --all-targets
 
       # Install nfpm used to for linux packaging
       - uses: actions/setup-go@v5
@@ -80,16 +131,16 @@ jobs:
           cache: false
       - run: go install github.com/goreleaser/nfpm/v2/cmd/nfpm@latest
 
-      - uses: taiki-e/install-action@just
-      - name: Build for ${{ matrix.job.target }}
-        run: just release ${{ matrix.job.target }}
+      - name: Package
+        run: |
+          ./ci/build_scripts/build.sh ${{ matrix.target }} --skip-build --skip-deprecated-packages
 
       - name: Upload packages as zip
         # https://github.com/marketplace/actions/upload-a-build-artifact
         uses: actions/upload-artifact@v4
         with:
-            name: packages-${{ matrix.job.target }}
-            path: target/${{ matrix.job.target }}/packages/*.*
+            name: packages-${{ matrix.target }}
+            path: target/${{ matrix.target }}/packages/*.*
 
   approve:
     # Note: Use approval as a job so that the downstream jobs are only prompted once (if more than 1 matrix job is defined)
@@ -200,6 +251,13 @@ jobs:
           # Keep arm-unknown-linux-musleabihf in separate repo due to armhf conflict between raspbian and debian
           - { target: arm-unknown-linux-musleabihf,   repo: tedge-main-armv6,    component: main }
           - { target: armv7-unknown-linux-musleabihf, repo: tedge-main,          component: main }
+          - { target: arm-unknown-linux-musleabi,     repo: tedge-main,          component: main }
+          - { target: i686-unknown-linux-musl,        repo: tedge-main,          component: main }
+          # Debian also calls this "armel" (conflict with arm-unknown-linux-musleabi)
+          # - { target: armv5te-unknown-linux-musleabi, repo: tedge-main,          component: main }
+          - { target: riscv64gc-unknown-linux-gnu,    repo: tedge-main,          component: main }
+          - { target: aarch64-apple-darwin,      repo: tedge-main,          component: main }
+          - { target: x86_64-apple-darwin,       repo: tedge-main,          component: main }
 
     steps:
       - name: Checkout

diff --git a/ci/build_scripts/publish_packages.sh b/ci/build_scripts/publish_packages.sh
@@ -183,12 +183,28 @@ get_user_friendly_arch() {
         *arm-unknown-linux-musleabihf*)
             easy_arch=armv6
             ;;
+        *arm-unknown-linux-musleabi*)
+            easy_arch=armv5
+            ;;
+        *armv5te-unknown-linux-*)
+            easy_arch=armv5
+            ;;
+        *i686-unknown-linux-musl*)
+            easy_arch=i386
+            ;;
+        *riscv64gc-unknown-linux-*)
+            easy_arch=riscv64
+            ;;
         *aarch64-apple-darwin*)
             easy_arch=macos-arm64
             ;;
         *x86_64-apple-darwin*)
             easy_arch=macos-amd64
             ;;
+        *)
+            echo "Unknown architecture. $1" >&2
+            exit 1
+            ;;
     esac
     echo "$easy_arch"
 }

diff --git a/crates/common/download/src/download.rs b/crates/common/download/src/download.rs
@@ -559,7 +559,7 @@ mod tests {
     async fn downloader_download_with_content_length_larger_than_usable_disk_space() {
         use nix::sys::statvfs;
         let tmpstats = statvfs::statvfs("/tmp").unwrap();
-        let usable_disk_space = tmpstats.blocks_free() * tmpstats.block_size();
+        let usable_disk_space = (tmpstats.blocks_free() as u64) * (tmpstats.block_size() as u64);
 
         let mut server = mockito::Server::new();
         let _mock1 = server

diff --git a/crates/core/c8y_api/src/json_c8y.rs b/crates/core/c8y_api/src/json_c8y.rs
@@ -605,7 +605,7 @@ mod tests {
 
         assert_eq!(actual_c8y_event.event_type, "empty_event".to_string());
         assert_eq!(actual_c8y_event.text, "empty_event".to_string());
-        assert!(actual_c8y_event.time < OffsetDateTime::now_utc());
+        assert!(actual_c8y_event.time <= OffsetDateTime::now_utc());
         assert_matches!(actual_c8y_event.source, None);
         assert!(actual_c8y_event.extras.is_empty());
 

diff --git a/mk/README.md b/mk/README.md
@@ -0,0 +1,30 @@
+# Notes
+
+The scripts in this folder are taken from the [briansmith/ring](https://github.com/briansmith/ring) repository under the folder. This is done as the build targets of thin-edge.io are coupled with the build targets supported by ring (as it is the crypto library used in the thin-edge.io usage of rustls and some other libraries).
+
+
+## How to update the mk files from ring
+
+1. Navigate to the [mk folder in the briansmith/ring](https://github.com/briansmith/ring/tree/main/mk) repository
+
+2. Copy the following files into the `mk` folder of thin-edge.io
+
+    **Files that don't require patching**
+
+    * mk/llvm-snapshot.gpg.key
+
+    **Files that require minimal patching**
+
+    THe following files require minor patching to add some additional thin-edge.io specific changes, however adjusting should be fairly easy to manage using the git diff:
+
+    * mk/cargo.sh
+    * mk/install-build-tools.sh
+
+    Some of the changes are due to shell check warnings, and there is already an [upstream PR](https://github.com/briansmith/ring/pull/1993) exists to resolved this warnings) so that this step can be skipped once the PR is merged. Alternatively you can take the files from the aforementioned PR instead of the PR itself if you are unsure how to merge the shell check changes.
+
+    **Notes**
+    * The copyright notice at the top of each files originating from **briansmith/ring** *MUST* be preserved
+
+3. Review the changes and resolve any differences
+
+4. Create a PR with the updated changes