From b1dd568ac33ce02953992901546d3146bcea80e2 Mon Sep 17 00:00:00 2001 From: Yadong Ding Date: Thu, 28 Dec 2023 05:19:57 +0800 Subject: [PATCH] test Signed-off-by: Yadong Ding --- .github/workflows/test.yml | 282 +++++++++++++++++++++++++++++ api/src/error.rs | 2 +- rafs/src/metadata/inode.rs | 4 +- rust-toolchain.toml | 3 +- storage/src/backend/localdisk.rs | 2 +- storage/src/backend/localfs.rs | 2 +- storage/src/backend/oss.rs | 2 +- storage/src/backend/registry.rs | 2 +- storage/src/cache/filecache/mod.rs | 4 +- storage/src/device.rs | 2 +- storage/src/meta/mod.rs | 2 +- 11 files changed, 295 insertions(+), 12 deletions(-) create mode 100644 .github/workflows/test.yml diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000000..c3373f76b8b --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,282 @@ +name: test + +on: + push: + workflow_dispatch: +env: + CARGO_TERM_COLOR: always + +jobs: + contrib-build: + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64, arm64, ppc64le, riscv64] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version-file: 'go.work' + cache-dependency-path: "**/*.sum" + - name: Build Contrib + run: | + make -e DOCKER=false GOARCH=${{ matrix.arch }} contrib-release + - name: Upload Nydusify + if: matrix.arch == 'amd64' + uses: actions/upload-artifact@v3 + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + + nydus-build: + runs-on: ubuntu-latest + strategy: + matrix: + arch: [amd64, arm64, ppc64le, riscv64] + steps: + - name: Checkout + uses: actions/checkout@v3 + - uses: dsherret/rust-toolchain-file@v1 + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.0 + with: + cache-on-failure: true + shared-key: ${{ runner.os }}-cargo-${{ matrix.arch }} + save-if: ${{ github.ref == 'refs/heads/master' }} + - name: Build Nydus + run: | + declare -A rust_target_map=( ["amd64"]="x86_64-unknown-linux-musl" ["arm64"]="aarch64-unknown-linux-musl" ["ppc64le"]="powerpc64le-unknown-linux-gnu" ["riscv64"]="riscv64gc-unknown-linux-gnu") + RUST_TARGET=${rust_target_map[${{ matrix.arch }}]} + cargo install --locked --version 0.2.4 cross + make -e RUST_TARGET_STATIC=$RUST_TARGET -e CARGO=cross static-release + sudo mv target/$RUST_TARGET/release/nydusd . + sudo mv target/$RUST_TARGET/release/nydus-image . + - name: Upload Nydus Binaries + if: matrix.arch == 'amd64' + uses: actions/upload-artifact@v3 + with: + name: nydus-artifact + path: | + nydus-image + nydusd + + nydusd-build-macos: + runs-on: macos-11 + strategy: + matrix: + arch: [amd64, arm64] + steps: + - uses: actions/checkout@v3 + - uses: dsherret/rust-toolchain-file@v1 + - name: Cache cargo + uses: Swatinem/rust-cache@v2.7.0 + with: + cache-on-failure: true + shared-key: ${{ runner.os }}-cargo-${{ matrix.arch }} + save-if: ${{ github.ref == 'refs/heads/master' }} + - name: build + run: | + if [[ "${{matrix.arch}}" == "amd64" ]]; then + RUST_TARGET="x86_64-apple-darwin" + else + RUST_TARGET="aarch64-apple-darwin" + fi + cargo install --version 0.2.4 cross + rustup target add ${RUST_TARGET} + make -e RUST_TARGET_STATIC=$RUST_TARGET -e CARGO=cross static-release + + nydus-integration-test: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Docker Cache + uses: jpribyl/action-docker-layer-caching@v0.1.0 + continue-on-error: true + - name: Download Nydus + uses: actions/download-artifact@v3 + with: + name: nydus-artifact + path: | + target/release + - name: Download Nydusify + uses: actions/download-artifact@v3 + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Older Binaries + id: prepare-binaries + run: | + export NYDUS_STABLE_VERSION=$(curl https://api.github.com/repos/Dragonflyoss/nydus/releases/latest | jq -r '.tag_name') + + versions=(v0.1.0 ${NYDUS_STABLE_VERSION}) + version_archs=(v0.1.0-x86_64 ${NYDUS_STABLE_VERSION}-linux-amd64) + for i in ${!versions[@]}; do + version=${versions[$i]} + version_arch=${version_archs[$i]} + + wget -q https://github.com/dragonflyoss/nydus/releases/download/$version/nydus-static-$version_arch.tgz + sudo mkdir nydus-$version /usr/bin/nydus-$version + sudo tar xzf nydus-static-$version_arch.tgz -C nydus-$version + sudo cp -r nydus-$version/nydus-static/* /usr/bin/nydus-$version/ + done + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version-file: 'go.work' + cache-dependency-path: "**/*.sum" + - name: Integration Test + run: | + sudo mkdir -p /usr/bin/nydus-latest /home/runner/work/workdir + sudo install -D -m 755 contrib/nydusify/cmd/nydusify /usr/bin/nydus-latest + sudo install -D -m 755 target/release/nydusd target/release/nydus-image /usr/bin/nydus-latest + + export NYDUS_STABLE_VERSION=$(curl https://api.github.com/repos/Dragonflyoss/nydus/releases/latest | jq -r '.tag_name') + export NYDUS_STABLE_VERSION_EXPORT="${NYDUS_STABLE_VERSION//./_}" + + versions=(v0.1.0 ${NYDUS_STABLE_VERSION} latest) + version_exports=(v0_1_0 ${NYDUS_STABLE_VERSION_EXPORT} latest) + for i in ${!version_exports[@]}; do + version=${versions[$i]} + version_export=${version_exports[$i]} + export NYDUS_BUILDER_$version_export=/usr/bin/nydus-$version/nydus-image + export NYDUS_NYDUSD_$version_export=/usr/bin/nydus-$version/nydusd + export NYDUS_NYDUSIFY_$version_export=/usr/bin/nydus-$version/nydusify + done + + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.54.2 + sudo -E make smoke-only + + nydus-unit-test: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.0 + with: + cache-on-failure: true + shared-key: Linux-cargo-amd64 + save-if: ${{ github.ref == 'refs/heads/master' }} + - name: Install cargo nextest + uses: taiki-e/install-action@nextest + - name: Fscache Setup + run: sudo bash misc/fscache/setup.sh + - name: Unit Test + run: | + CARGO_HOME=${HOME}/.cargo + CARGO_BIN=$(which cargo) + sudo -E CARGO=${CARGO_BIN} make ut-nextest + + contrib-unit-test-coverage: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Setup Golang + uses: actions/setup-go@v4 + with: + go-version-file: 'go.work' + cache-dependency-path: "**/*.sum" + - name: Unit Test + run: | + curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b /usr/bin v1.54.2 + make -e DOCKER=false contrib-test + - name: Upload contrib coverage file + uses: actions/upload-artifact@v3 + with: + name: contrib-test-coverage-artifact + path: | + contrib/nydusify/coverage.txt + + nydus-unit-test-coverage: + runs-on: ubuntu-latest + env: + CARGO_TERM_COLOR: always + steps: + - uses: actions/checkout@v3 + - name: Rust Cache + uses: Swatinem/rust-cache@v2.7.0 + with: + cache-on-failure: true + shared-key: Linux-cargo-amd64 + save-if: ${{ github.ref == 'refs/heads/master' }} + - name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + - name: Fscache Setup + run: sudo bash misc/fscache/setup.sh + - name: Generate code coverage + run: | + CARGO_HOME=${HOME}/.cargo + CARGO_BIN=$(which cargo) + sudo -E CARGO=${CARGO_BIN} make coverage-codecov + - name: Upload nydus coverage file + uses: actions/upload-artifact@v3 + with: + name: nydus-test-coverage-artifact + path: | + codecov.json + + nydus-cargo-deny: + name: cargo-deny + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v3 + - uses: EmbarkStudios/cargo-deny-action@v1 + + performance-test: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + strategy: + matrix: + include: + - mode: fs-version-5 + - mode: fs-version-6 + - mode: zran + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@v3 + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@v3 + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Container Environment + run: | + sudo bash misc/performance/prepare.sh + - name: Performance Test + run: | + export PERFORMANCE_TEST_MODE=${{ matrix.mode }} + sudo -E make smoke-performance + + failover-test: + runs-on: ubuntu-latest + needs: [contrib-build, nydus-build] + steps: + - name: Checkout + uses: actions/checkout@v3 + - name: Download Nydus + uses: actions/download-artifact@v3 + with: + name: nydus-artifact + path: target/release + - name: Download Nydusify + uses: actions/download-artifact@v3 + with: + name: nydusify-artifact + path: contrib/nydusify/cmd + - name: Prepare Nydus Container Environment + run: | + sudo bash misc/failover/prepare.sh + - name: Failover Test + run: | + sudo -E make smoke-failover diff --git a/api/src/error.rs b/api/src/error.rs index ad7c391ac02..277c0cc00ba 100644 --- a/api/src/error.rs +++ b/api/src/error.rs @@ -11,7 +11,7 @@ pub fn make_error( _file: &str, _line: u32, ) -> std::io::Error { - #[cfg(all(feature = "error-backtrace"))] + #[cfg(feature = "error-backtrace")] { if let Ok(val) = std::env::var("RUST_BACKTRACE") { if val.trim() != "0" { diff --git a/rafs/src/metadata/inode.rs b/rafs/src/metadata/inode.rs index 19332284818..938b058ae6d 100644 --- a/rafs/src/metadata/inode.rs +++ b/rafs/src/metadata/inode.rs @@ -583,10 +583,10 @@ impl InodeWrapper { if let Self::Ref(i) = self { let i = i.clone(); if self.is_v6() { - *self = Self::V6(RafsV6Inode::from(i.deref().deref())); + *self = Self::V6(RafsV6Inode::from(i.deref())); } else { assert!(self.is_v5()); - *self = Self::V5(RafsV5Inode::from(i.deref().deref())); + *self = Self::V5(RafsV5Inode::from(i.deref())); } } } diff --git a/rust-toolchain.toml b/rust-toolchain.toml index a735428f1d1..4bd1ede5bae 100644 --- a/rust-toolchain.toml +++ b/rust-toolchain.toml @@ -1,2 +1,3 @@ [toolchain] -channel = "1.68.2" \ No newline at end of file +channel = "1.74.1" +components = ["rustfmt", "clippy"] diff --git a/storage/src/backend/localdisk.rs b/storage/src/backend/localdisk.rs index 0c83bfba9db..4475797d5ad 100644 --- a/storage/src/backend/localdisk.rs +++ b/storage/src/backend/localdisk.rs @@ -161,7 +161,7 @@ impl LocalDisk { path, e )) })?; - let device_file = OpenOptions::new().read(true).open(&path_buf).map_err(|e| { + let device_file = OpenOptions::new().read(true).open(path_buf).map_err(|e| { einval!(format!( "localdisk: can not open disk device at {}, {}", path, e diff --git a/storage/src/backend/localfs.rs b/storage/src/backend/localfs.rs index 08df265f12f..6168a1903da 100644 --- a/storage/src/backend/localfs.rs +++ b/storage/src/backend/localfs.rs @@ -128,7 +128,7 @@ impl LocalFs { // Search blob file in dir and additionally in alt_dirs let is_valid = |dir: &PathBuf| -> bool { let blob = Path::new(&dir).join(blob_id); - if let Ok(meta) = std::fs::metadata(&blob) { + if let Ok(meta) = std::fs::metadata(blob) { meta.len() != 0 } else { false diff --git a/storage/src/backend/oss.rs b/storage/src/backend/oss.rs index 592a556f131..9ec08cfd71c 100644 --- a/storage/src/backend/oss.rs +++ b/storage/src/backend/oss.rs @@ -100,7 +100,7 @@ impl ObjectStorageState for OssState { .chain_update(data.as_bytes()) .finalize() .into_bytes(); - let signature = base64::engine::general_purpose::STANDARD.encode(&hmac); + let signature = base64::engine::general_purpose::STANDARD.encode(hmac); let authorization = format!("OSS {}:{}", self.access_key_id, signature); diff --git a/storage/src/backend/registry.rs b/storage/src/backend/registry.rs index 295e7ab1f95..a9947f49efb 100644 --- a/storage/src/backend/registry.rs +++ b/storage/src/backend/registry.rs @@ -662,7 +662,7 @@ impl RegistryReader { // The request has expired or has been denied, need to re-request if allow_retry - && vec![StatusCode::UNAUTHORIZED, StatusCode::FORBIDDEN].contains(&resp.status()) + && [StatusCode::UNAUTHORIZED, StatusCode::FORBIDDEN].contains(&resp.status()) { warn!( "The redirected link has expired: {}, will retry read", diff --git a/storage/src/cache/filecache/mod.rs b/storage/src/cache/filecache/mod.rs index d42ad388fcf..1e38f3b3072 100644 --- a/storage/src/cache/filecache/mod.rs +++ b/storage/src/cache/filecache/mod.rs @@ -226,7 +226,7 @@ impl FileCacheEntry { .create(false) .write(false) .read(true) - .open(&blob_file_path)?; + .open(blob_file_path)?; let chunk_map = Arc::new(BlobStateMap::from(NoopChunkMap::new(true))) as Arc; (file, None, chunk_map, true, true, false) @@ -250,7 +250,7 @@ impl FileCacheEntry { .create(true) .write(true) .read(true) - .open(&blob_data_file_path)?; + .open(blob_data_file_path)?; let file_size = file.metadata()?.len(); let cached_file_size = if mgr.cache_raw_data { blob_info.compressed_data_size() diff --git a/storage/src/device.rs b/storage/src/device.rs index 5151169e058..08c3ee5409a 100644 --- a/storage/src/device.rs +++ b/storage/src/device.rs @@ -554,7 +554,7 @@ impl BlobInfo { } guard.deref().clone() } else { - hex::encode(&self.blob_meta_digest) + hex::encode(self.blob_meta_digest) }; Ok(id) } diff --git a/storage/src/meta/mod.rs b/storage/src/meta/mod.rs index eff935cda17..dafc5bf1890 100644 --- a/storage/src/meta/mod.rs +++ b/storage/src/meta/mod.rs @@ -531,7 +531,7 @@ impl BlobCompressionContextInfo { TocLocation::default() }; let toc_list = - TocEntryList::read_from_cache_file(&toc_path, reader.as_ref(), &location)?; + TocEntryList::read_from_cache_file(toc_path, reader.as_ref(), location)?; toc_list.extract_from_blob(reader.clone(), None, Some(&digest_path))?; } if !digest_path.exists() {