test

Signed-off-by: Yadong Ding <ding_yadong@foxmail.com>

.github/workflows/test.yml

@@ -0,0 +1,282 @@
+name: test
+
+on:
+  push:
+  workflow_dispatch:
+env:
+  CARGO_TERM_COLOR: always
+
+jobs:
+  contrib-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64, ppc64le, riscv64]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Golang
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: 'go.work'
+        cache-dependency-path: "**/*.sum"
+    - name: Build Contrib
+      run: |
+        make -e DOCKER=false GOARCH=${{ matrix.arch }} contrib-release
+    - name: Upload Nydusify
+      if: matrix.arch == 'amd64'
+      uses: actions/upload-artifact@v3
+      with:
+        name: nydusify-artifact
+        path: contrib/nydusify/cmd
+
+  nydus-build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64, arm64, ppc64le, riscv64]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - uses: dsherret/rust-toolchain-file@v1
+    - name: Rust Cache
+      uses: Swatinem/rust-cache@v2.7.0
+      with:
+        cache-on-failure: true
+        shared-key: ${{ runner.os }}-cargo-${{ matrix.arch }}
+        save-if: ${{ github.ref == 'refs/heads/master' }}
+    - name: Build Nydus
+      run: |
+        declare -A rust_target_map=( ["amd64"]="x86_64-unknown-linux-musl" ["arm64"]="aarch64-unknown-linux-musl" ["ppc64le"]="powerpc64le-unknown-linux-gnu" ["riscv64"]="riscv64gc-unknown-linux-gnu")
+        RUST_TARGET=${rust_target_map[${{ matrix.arch }}]}
+        cargo install --locked --version 0.2.4 cross
+        make -e RUST_TARGET_STATIC=$RUST_TARGET -e CARGO=cross static-release
+        sudo mv target/$RUST_TARGET/release/nydusd .
+        sudo mv target/$RUST_TARGET/release/nydus-image .
+    - name: Upload Nydus Binaries
+      if: matrix.arch == 'amd64'
+      uses: actions/upload-artifact@v3
+      with:
+        name: nydus-artifact
+        path: |
+          nydus-image
+          nydusd
+
+  nydusd-build-macos:
+    runs-on: macos-11
+    strategy:
+      matrix:
+        arch: [amd64, arm64]
+    steps:
+    - uses: actions/checkout@v3
+    - uses: dsherret/rust-toolchain-file@v1
+    - name: Cache cargo
+      uses: Swatinem/rust-cache@v2.7.0
+      with:
+        cache-on-failure: true
+        shared-key: ${{ runner.os }}-cargo-${{ matrix.arch }}
+        save-if: ${{ github.ref == 'refs/heads/master' }}
+    - name: build
+      run: |
+        if [[ "${{matrix.arch}}" == "amd64" ]]; then
+          RUST_TARGET="x86_64-apple-darwin"
+        else
+          RUST_TARGET="aarch64-apple-darwin"
+        fi
+        cargo install --version 0.2.4 cross
+        rustup target add ${RUST_TARGET}
+        make -e RUST_TARGET_STATIC=$RUST_TARGET -e CARGO=cross static-release
+
+  nydus-integration-test:
+    runs-on: ubuntu-latest
+    needs: [contrib-build, nydus-build]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Docker Cache
+      uses: jpribyl/action-docker-layer-caching@v0.1.0
+      continue-on-error: true
+    - name: Download Nydus
+      uses: actions/download-artifact@v3
+      with:
+        name: nydus-artifact
+        path: |
+          target/release
+    - name: Download Nydusify
+      uses: actions/download-artifact@v3
+      with:
+        name: nydusify-artifact
+        path: contrib/nydusify/cmd
+    - name: Prepare Older Binaries
+      id: prepare-binaries
+      run: |
+        export NYDUS_STABLE_VERSION=$(curl https://api.github.com/repos/Dragonflyoss/nydus/releases/latest | jq -r '.tag_name')
+
+        versions=(v0.1.0 ${NYDUS_STABLE_VERSION})
+        version_archs=(v0.1.0-x86_64 ${NYDUS_STABLE_VERSION}-linux-amd64)
+        for i in ${!versions[@]}; do
+          version=${versions[$i]}
+          version_arch=${version_archs[$i]}
+
+          wget -q https://github.com/dragonflyoss/nydus/releases/download/$version/nydus-static-$version_arch.tgz
+          sudo mkdir nydus-$version /usr/bin/nydus-$version
+          sudo tar xzf nydus-static-$version_arch.tgz -C nydus-$version
+          sudo cp -r nydus-$version/nydus-static/* /usr/bin/nydus-$version/
+        done
+    - name: Setup Golang
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: 'go.work'
+        cache-dependency-path: "**/*.sum"
+    - name: Integration Test
+      run: |
+        sudo mkdir -p /usr/bin/nydus-latest /home/runner/work/workdir
+        sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/bin/nydus-latest
+        sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/bin/nydus-latest
+
+        export NYDUS_STABLE_VERSION=$(curl https://api.github.com/repos/Dragonflyoss/nydus/releases/latest | jq -r '.tag_name')
+        export NYDUS_STABLE_VERSION_EXPORT="${NYDUS_STABLE_VERSION//./_}"
+
+        versions=(v0.1.0 ${NYDUS_STABLE_VERSION} latest)
+        version_exports=(v0_1_0 ${NYDUS_STABLE_VERSION_EXPORT} latest)
+        for i in ${!version_exports[@]}; do
+          version=${versions[$i]}
+          version_export=${version_exports[$i]}
+          export NYDUS_BUILDER_$version_export=/usr/bin/nydus-$version/nydus-image
+          export NYDUS_NYDUSD_$version_export=/usr/bin/nydus-$version/nydusd
+          export NYDUS_NYDUSIFY_$version_export=/usr/bin/nydus-$version/nydusify
+        done
+
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.54.2
+        sudo -E make smoke-only
+
+  nydus-unit-test:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Rust Cache
+      uses: Swatinem/rust-cache@v2.7.0
+      with:
+        cache-on-failure: true
+        shared-key: Linux-cargo-amd64
+        save-if: ${{ github.ref == 'refs/heads/master' }}
+    - name: Install cargo nextest
+      uses: taiki-e/install-action@nextest
+    - name: Fscache Setup
+      run: sudo bash misc/fscache/setup.sh
+    - name: Unit Test
+      run: |
+        CARGO_HOME=${HOME}/.cargo
+        CARGO_BIN=$(which cargo)
+        sudo -E CARGO=${CARGO_BIN} make ut-nextest
+
+  contrib-unit-test-coverage:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Setup Golang
+      uses: actions/setup-go@v4
+      with:
+        go-version-file: 'go.work'
+        cache-dependency-path: "**/*.sum"
+    - name: Unit Test
+      run: |
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.54.2
+        make -e DOCKER=false contrib-test
+    - name: Upload contrib coverage file
+      uses: actions/upload-artifact@v3
+      with:
+        name: contrib-test-coverage-artifact
+        path: |
+          contrib/nydusify/coverage.txt
+
+  nydus-unit-test-coverage:
+    runs-on: ubuntu-latest
+    env:
+      CARGO_TERM_COLOR: always
+    steps:
+      - uses: actions/checkout@v3
+      - name: Rust Cache
+        uses: Swatinem/rust-cache@v2.7.0
+        with:
+          cache-on-failure: true
+          shared-key: Linux-cargo-amd64
+          save-if: ${{ github.ref == 'refs/heads/master' }}
+      - name: Install cargo-llvm-cov
+        uses: taiki-e/install-action@cargo-llvm-cov
+      - name: Fscache Setup
+        run: sudo bash misc/fscache/setup.sh
+      - name: Generate code coverage
+        run: |
+          CARGO_HOME=${HOME}/.cargo
+          CARGO_BIN=$(which cargo)
+          sudo -E CARGO=${CARGO_BIN} make coverage-codecov
+      - name: Upload nydus coverage file
+        uses: actions/upload-artifact@v3
+        with:
+          name: nydus-test-coverage-artifact
+          path: |
+            codecov.json
+
+  nydus-cargo-deny:
+    name: cargo-deny
+    runs-on: ubuntu-latest
+    timeout-minutes: 10
+    steps:
+      - uses: actions/checkout@v3
+      - uses: EmbarkStudios/cargo-deny-action@v1
+
+  performance-test:
+    runs-on: ubuntu-latest
+    needs: [contrib-build, nydus-build]
+    strategy:
+      matrix:
+        include:
+          - mode: fs-version-5
+          - mode: fs-version-6
+          - mode: zran
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Download Nydus
+      uses: actions/download-artifact@v3
+      with:
+        name: nydus-artifact
+        path: target/release
+    - name: Download Nydusify
+      uses: actions/download-artifact@v3
+      with:
+        name: nydusify-artifact
+        path: contrib/nydusify/cmd
+    - name: Prepare Nydus Container Environment
+      run: |
+        sudo bash misc/performance/prepare.sh
+    - name: Performance Test
+      run: |
+        export PERFORMANCE_TEST_MODE=${{ matrix.mode }}
+        sudo -E make smoke-performance
+
+  failover-test:
+    runs-on: ubuntu-latest
+    needs: [contrib-build, nydus-build]
+    steps:
+    - name: Checkout
+      uses: actions/checkout@v3
+    - name: Download Nydus
+      uses: actions/download-artifact@v3
+      with:
+        name: nydus-artifact
+        path: target/release
+    - name: Download Nydusify
+      uses: actions/download-artifact@v3
+      with:
+        name: nydusify-artifact
+        path: contrib/nydusify/cmd
+    - name: Prepare Nydus Container Environment
+      run: |
+        sudo bash misc/failover/prepare.sh
+    - name: Failover Test
+      run: |
+        sudo -E make smoke-failover

api/src/error.rs

@@ -11,7 +11,7 @@ pub fn make_error(
     _file: &str,
     _line: u32,
 ) -> std::io::Error {
-    #[cfg(all(feature = "error-backtrace"))]
+    #[cfg(feature = "error-backtrace")]
     {
         if let Ok(val) = std::env::var("RUST_BACKTRACE") {
             if val.trim() != "0" {

rafs/src/metadata/inode.rs

@@ -583,10 +583,10 @@ impl InodeWrapper {
         if let Self::Ref(i) = self {
             let i = i.clone();
             if self.is_v6() {
-                *self = Self::V6(RafsV6Inode::from(i.deref().deref()));
+                *self = Self::V6(RafsV6Inode::from(i.deref()));
             } else {
                 assert!(self.is_v5());
-                *self = Self::V5(RafsV5Inode::from(i.deref().deref()));
+                *self = Self::V5(RafsV5Inode::from(i.deref()));
             }
         }
     }

rust-toolchain.toml

@@ -1,2 +1,3 @@
 [toolchain]
-channel = "1.68.2"
+channel = "1.74.1"
+components = ["rustfmt", "clippy"]

storage/src/backend/localdisk.rs

@@ -161,7 +161,7 @@ impl LocalDisk {
                 path, e
             ))
         })?;
-        let device_file = OpenOptions::new().read(true).open(&path_buf).map_err(|e| {
+        let device_file = OpenOptions::new().read(true).open(path_buf).map_err(|e| {
             einval!(format!(
                 "localdisk: can not open disk device at {}, {}",
                 path, e

storage/src/backend/localfs.rs

@@ -128,7 +128,7 @@ impl LocalFs {
             // Search blob file in dir and additionally in alt_dirs
             let is_valid = |dir: &PathBuf| -> bool {
                 let blob = Path::new(&dir).join(blob_id);
-                if let Ok(meta) = std::fs::metadata(&blob) {
+                if let Ok(meta) = std::fs::metadata(blob) {
                     meta.len() != 0
                 } else {
                     false

storage/src/backend/oss.rs

@@ -100,7 +100,7 @@ impl ObjectStorageState for OssState {
             .chain_update(data.as_bytes())
             .finalize()
             .into_bytes();
-        let signature = base64::engine::general_purpose::STANDARD.encode(&hmac);
+        let signature = base64::engine::general_purpose::STANDARD.encode(hmac);
 
         let authorization = format!("OSS {}:{}", self.access_key_id, signature);
 

storage/src/backend/registry.rs

@@ -662,7 +662,7 @@ impl RegistryReader {
 
             // The request has expired or has been denied, need to re-request
             if allow_retry
-                && vec![StatusCode::UNAUTHORIZED, StatusCode::FORBIDDEN].contains(&resp.status())
+                && [StatusCode::UNAUTHORIZED, StatusCode::FORBIDDEN].contains(&resp.status())
             {
                 warn!(
                     "The redirected link has expired: {}, will retry read",

storage/src/cache/filecache/mod.rs

@@ -226,7 +226,7 @@ impl FileCacheEntry {
                 .create(false)
                 .write(false)
                 .read(true)
-                .open(&blob_file_path)?;
+                .open(blob_file_path)?;
             let chunk_map =
                 Arc::new(BlobStateMap::from(NoopChunkMap::new(true))) as Arc<dyn ChunkMap>;
             (file, None, chunk_map, true, true, false)
@@ -250,7 +250,7 @@ impl FileCacheEntry {
                 .create(true)
                 .write(true)
                 .read(true)
-                .open(&blob_data_file_path)?;
+                .open(blob_data_file_path)?;
             let file_size = file.metadata()?.len();
             let cached_file_size = if mgr.cache_raw_data {
                 blob_info.compressed_data_size()

storage/src/device.rs

@@ -554,7 +554,7 @@ impl BlobInfo {
             }
             guard.deref().clone()
         } else {
-            hex::encode(&self.blob_meta_digest)
+            hex::encode(self.blob_meta_digest)
         };
         Ok(id)
     }

storage/src/meta/mod.rs

@@ -531,7 +531,7 @@ impl BlobCompressionContextInfo {
                     TocLocation::default()
                 };
                 let toc_list =
-                    TocEntryList::read_from_cache_file(&toc_path, reader.as_ref(), &location)?;
+                    TocEntryList::read_from_cache_file(toc_path, reader.as_ref(), location)?;
                 toc_list.extract_from_blob(reader.clone(), None, Some(&digest_path))?;
             }
             if !digest_path.exists() {