Merge pull request #535 from yawqi/master

github actions: add integration tests for nydus

.github/workflows/it.yml

@@ -0,0 +1,111 @@
+name: Nydus Ingegration Test
+
+on:
+  schedule:
+    # Do conversion every day at 00:03 clock UTC
+    - cron: "3 0 * * *"
+  workflow_dispatch:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        arch: [amd64]
+    steps:          
+    - name: Set up Go
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    - name: Setup pytest
+      run: |
+        sudo apt install --no-install-recommends -y attr libattr1-dev fio pkg-config libssl-dev python3
+        sudo python3 -m pip install --upgrade pip
+        sudo pip3 install pytest xattr requests psutil requests_unixsocket libconf py-splice fallocate pytest-repeat PyYAML six docker toml
+    - name: containerd runc and crictl
+      run: |
+        sudo wget https://github.com/kubernetes-sigs/cri-tools/releases/download/v1.17.0/crictl-v1.17.0-linux-amd64.tar.gz
+        sudo tar zxvf ./crictl-v1.17.0-linux-amd64.tar.gz -C /usr/local/bin
+        sudo wget https://github.com/containerd/containerd/releases/download/v1.4.3/containerd-1.4.3-linux-amd64.tar.gz
+        mkdir containerd
+        sudo tar -zxf ./containerd-1.4.3-linux-amd64.tar.gz -C ./containerd
+        sudo mv ./containerd/bin/* /usr/bin/
+        sudo wget https://github.com/opencontainers/runc/releases/download/v1.1.2/runc.amd64 -O /usr/bin/runc
+        sudo chmod +x /usr/bin/runc
+    - name: Set up ossutils
+      run: |
+        sudo wget https://gosspublic.alicdn.com/ossutil/1.7.13/ossutil64 -O /usr/bin/ossutil64
+        sudo chmod +x /usr/bin/ossutil64
+    - uses: actions/checkout@v3
+    - name: Cache cargo
+      uses: Swatinem/rust-cache@v1
+      with:
+        target-dir: |
+          ./target-fusedev
+          ./target-virtiofs
+        cache-on-failure: true
+        key: ${{ runner.os }}-cargo-${{ matrix.arch }}
+
+    - name: Build nydus-rs
+      run: |
+        declare -A rust_arch_map=( ["amd64"]="x86_64" ["arm64"]="aarch64")
+        arch=${rust_arch_map[${{ matrix.arch }}]}
+        cargo install --version 0.2.1 cross
+        rustup component add rustfmt clippy
+        make -e ARCH=$arch -e CARGO=cross static-release
+        #sudo mv target-fusedev/$arch-unknown-linux-musl/release/nydusd nydusd-fusedev
+        #sudo mv target-fusedev/$arch-unknown-linux-musl/release/nydus-cached .
+        #sudo mv target-fusedev/$arch-unknown-linux-musl/release/nydus-image .
+        #sudo mv target-fusedev/$arch-unknown-linux-musl/release/nydusctl .
+        #sudo mv target-virtiofs/$arch-unknown-linux-musl/release/nydusd nydusd-virtiofs
+        sudo cp -r misc/configs .
+        sudo chown -R $(id -un):$(id -gn) . ~/.cargo/
+        pwd
+        ls -lh target-virtiofs/$arch-unknown-linux-musl/release
+    - name: Set up anchor file
+      env:
+        OSS_AK_ID: ${{ secrets.OSS_TEST_AK_ID }}
+        OSS_AK_SEC: ${{ secrets.OSS_TEST_AK_SECRET }}
+      run: |
+        sudo mkdir -p /home/runner/nydus-test-workspace
+        sudo cat > /home/runner/work/image-service/image-service/contrib/nydus-test/anchor_conf.json << EOF
+        {
+            "workspace": "/home/runner/nydus-test-workspace",
+            "nydus_project": "/home/runner/work/image-service/image-service",
+            "nydus_runtime_conf": {
+                "profile": "release",
+                "log_level": "info"
+            },
+            "registry": {
+                "registry_url": "localhost:5000",
+                "registry_namespace": "",
+                "registry_auth": "Y2h1YW5sYW5nLmdjdzoxcWF6MndzeA=="
+            },
+            "oss": {
+                "endpoint": "oss-cn-beijing.aliyuncs.com",
+                "ak_id": "$OSS_AK_ID",
+                "ak_secret": "$OSS_AK_SEC",
+                "bucket": "nydus-ci"
+            },
+            "images": {
+                "images_array": [
+                    "busybox:latest"
+                ]
+            },
+            "artifacts": {
+                "containerd": "/usr/bin/containerd",
+                "ossutil_bin": "/usr/bin/ossutil64"
+            },
+            "logging_file": "stderr",
+            "target": "musl"
+        }
+        EOF
+    - name: run test_api
+      run: |
+        cd /home/runner/work/image-service/image-service/contrib/nydus-test
+        sudo mkdir -p /blobdir
+        sudo python3 nydus_test_config.py --dist fs_structure.yaml
+        sudo pytest -vs --durations=0 --pdb functional-test/test_api.py::test_detect_io_hang \
+                                  functional-test/test_api.py::test_access_pattern \
+                                  functional-test/test_api.py::test_api_mount_with_prefetch \
+                                  functional-test/test_api.py::test_daemon_info

contrib/nydus-test/README.md

@@ -0,0 +1,199 @@
+
+# Nydus Functional Test
+
+## Introduction
+
+Nydus functional test suit is built on top of [pytest](https://docs.pytest.org/en/stable/).
+
+It basically includes two parts:
+
+* Specific test cases
+* Test framework
+
+Test framework includes several key components which are likely to be used by each test case.
+
+* RafsMount
+* RafsImage
+* TestAnchor
+* Distributor
+* NydusClient
+* WorkloadGenerator
+* Supervisor
+
+Cases are categorized into several files.
+
+* test_api.py
+* test_fault.py
+* test_image_compatibility.py
+* test_layered_image.py
+* test_nydus.py
+* test_nydusify.py
+* test_private.py
+* test_snapshotter.py
+* test_stargz.py
+* test_stress.py
+
+## Getting Started
+
+
+### Generate target rootfs
+
+```bash
+python3 nydus_test_config.py -D fs_structure.yaml
+```
+
+### Configure framework
+
+```json
+{
+    "workspace": "/path/where/temporary/files/placed",
+    "mount_point": "/mnt",
+    "blobcache_dir": "/path/where/saves/blobcache/file",
+    "binary_dir": "/path/to/nydusd/image-builder/binary",
+    "build_dir": "/nydus/source/",
+    "ossutil_bin": "/binary/ossutil",
+    "overlayfs": "/path/where/mount/overlayfs",
+    "log_level": "info",
+    "rootfs": "/target/rootfs",
+    "parent_rootfs": "/target/parent/rootfs",
+    "target_registry": "localhost:5000",
+    "nydusify_work_dir": "/nydusify/workdir",
+    "images_array": "/images/will/be/tested/by/nydusify"
+}
+```
+
+### Build container
+
+```bash
+cd /path/to/this-test-suit
+docker build --tag ${IMAGE_NAME} - < Dockerfile
+```
+
+### Compile Nydus
+
+### Set up test environment
+
+Ensure you have an OSS backend which is accessible from local node running this test suit.
+The test framework will obtain OSS configuration from environment variable
+
+#### Configure OSS backend
+
+1. OSS endpoint from `NYDUS_ENDPOINT`
+
+2. OSS access ID from `NYDUS_AK_ID`
+
+3. OSS access secret from `NYDUS_AK_SECRET`
+
+4. OSS bucket from `NYDUS_OSS_BUCKET`
+
+Before running test, please ensure you have fulfilled those environment variables.
+
+#### Generate your own original rootfs
+
+The framework provides a tool to generate rootfs which will be the test target.
+
+```bash
+ python3 nydus_test_config.py
+```
+
+### Run test
+
+#### From from docker
+
+e.g.
+
+```bash
+# Copy binary for this test
+mkdir -p $PWD/target-fusedev/release
+sudo cp $HOME/.cargo/target-fusedev/release/nydusd $PWD/target-fusedev/release/.
+sudo cp $HOME/.cargo/target-fusedev/release/nydus-image $PWD/target-fusedev/release/.
+
+# On host
+NYDUS_TEST_ROOT=$PWD/nydus-test
+NYDUS_RS_ROOT=$PWD/target-fusedev
+NYDUS_TEST_ANCHOR_CONF=/nydus/anchor_conf.json
+SNAPSHOTTER_IMAGES_ARRAY=/nydus/snapshotter_images_array.txt
+NYDUS_SOURCE=$PWD/nydus-rs
+
+# Inside container
+NYDUS_TEST_WORKDIR=/nydus-test
+
+mkdir workspace
+
+sudo docker run \
+  --rm \
+  -v $NYDUS_TEST_ROOT:/nydus-test \
+  -v $NYDUS_RS_ROOT:/nydus-rs/target-fusedev \
+  -v $NYDUS_TEST_ANCHOR_CONF:$NYDUS_TEST_WORKDIR/anchor_conf.json \
+  -v $SNAPSHOTTER_IMAGES_ARRAY:$NYDUS_TEST_WORKDIR/snapshotter_images_array.txt \
+  -v /sys/fs/fuse:/sys/fs/fuse \
+  -v $PWD/workspace:/workspace \
+  -v $NYDUS_SOURCE:/nydus-source \
+  --env-file /nydus/oss_env \
+  --env PREFERRED_MODE=direct \
+  --env ANCHOR_PATH=$NYDUS_TEST_WORKDIR \
+  --env LOG_FILE=/workspace/pyteset.direct.log \
+  --env GOPROXY=https://goproxy.io \
+  --workdir $NYDUS_TEST_WORKDIR \
+  --net=host \
+  --privileged \
+  nydus-test:0118
+```
+
+Rafs has two kinds of metadata mode - `cached` and `direct`. Framework will try to read the preferred metadata mode from environment variable `PREFERRED_MODE`. If `PREFERRED_MODE` is never assigned framework will select `direct` as the metadata mode.
+
+#### Run all test cases
+
+The whole nydus functional test suit works on top of pytest.
+
+#### Run a specific test case
+
+e.g.
+
+```bash
+pytest -sv functional-test/test_nydus.py::test_basic
+```
+
+#### Run a set of test cases
+
+e.g.
+
+```bash
+pytest -sv functional-test/test_nydus.py
+```
+
+#### Stop once a case fails
+
+e.g.
+
+```bash
+pytest -sv functional-test/test_nydus.py::test_basic --pdb
+```
+
+#### Run case step by step
+
+e.g.
+
+```bash
+pytest -sv functional-test/test_nydus.py::test_basic --trace
+```
+
+### Tune test framework
+
+The entire test is controlled by a configuration file named `anchor_conf.json`.
+When test container starts, it already includes an anchor file located at `/etc/anchor_conf.json`.
+You can also override it by mounting a local `anchor_conf.json` into test container.
+
+#### 1. Log level
+
+Log level can be changed for both nydus image builder `nydus-image` and `nydusd` by `log_level` name/value pair in `anchor_conf.json`
+
+#### 2. Specify nydusd execution location
+
+The framework can find executions from a specified path from `binary_dir` in `anchor_conf.json` file. This is useful when you want to change to different compiled targets, like from release version to debug version of the binary.
+
+#### 3. Specify original rootfs directory
+
+## Future work
+
+* Specify a pattern how to generate tree structure.