From 42ec78f14f930d8884d8cef7deaf8b6997525a83 Mon Sep 17 00:00:00 2001 From: hijackthe2 <2948278083@qq.com> Date: Fri, 27 Oct 2023 09:58:40 +0800 Subject: [PATCH 1/3] tests: add unit test case for blob_cache.rs, block_device.rs, fs_cache.rs, singleton.rs under service/src 1. In blob_cache.rs, two simple lines of code have been added to cover previously missed cases. 2. In block_device.rs, some test cases are added to cover function export(), block_size(), blocks_to_size(), and size_to_blocks(). 3. In fs_cache.rs, some test cases are added to cover function try_from() for struct FsCacheMsgOpen and FsCacheMsgRead. 4. In singletion.rs, some test cases are added to cover function initialize_blob_cache() and initialize_fscache_service(). In addition, fscache must be correctly enabled firstly as the device file `/dev/cachefiles` will used by function initialize_fscache_service(). Signed-off-by: hijackthe2 <2948278083@qq.com> --- service/src/blob_cache.rs | 4 + service/src/block_device.rs | 195 ++++++++++++++++++++++++++++-------- service/src/fs_cache.rs | 67 +++++++++++++ service/src/singleton.rs | 140 ++++++++++++++++++++++++++ 4 files changed, 367 insertions(+), 39 deletions(-) diff --git a/service/src/blob_cache.rs b/service/src/blob_cache.rs index 8d3d332b45e..1683657c12b 100644 --- a/service/src/blob_cache.rs +++ b/service/src/blob_cache.rs @@ -622,6 +622,7 @@ mod tests { is_tarfs_mode: false, }; assert_eq!(blob.path(), &path); + assert_eq!(blob.blob_id(), "123456789-123"); } #[test] @@ -734,6 +735,9 @@ mod tests { let blob_id = generate_blob_key(&entry.domain_id, &entry.blob_id); assert!(mgr.get_config(&blob_id).is_some()); + // add the same entry will trigger an error + assert!(mgr.add_blob_entry(&entry).is_err()); + // Check existence of data blob referenced by the bootstrap. let key = generate_blob_key( &entry.domain_id, diff --git a/service/src/block_device.rs b/service/src/block_device.rs index 58635210a95..52c901808db 100644 --- a/service/src/block_device.rs +++ b/service/src/block_device.rs @@ -539,50 +539,16 @@ mod tests { use super::*; use crate::blob_cache::generate_blob_key; use nydus_api::BlobCacheEntry; - use std::fs; + use nydus_utils::digest::{DigestHasher, RafsDigest}; + use std::fs::{self, File}; + use std::io::{BufReader, Read}; use std::path::PathBuf; use vmm_sys_util::tempdir::TempDir; #[test] fn test_block_device() { - let tmpdir = TempDir::new().unwrap(); - let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); - let mut source_path = PathBuf::from(root_dir); - source_path.push("../tests/texture/blobs/be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); - let mut dest_path = tmpdir.as_path().to_path_buf(); - dest_path.push("be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); - fs::copy(&source_path, &dest_path).unwrap(); - - let mut source_path = PathBuf::from(root_dir); - source_path.push("../tests/texture/bootstrap/rafs-v6-2.2.boot"); - let config = r#" - { - "type": "bootstrap", - "id": "rafs-v6", - "domain_id": "domain2", - "config_v2": { - "version": 2, - "id": "factory1", - "backend": { - "type": "localfs", - "localfs": { - "dir": "/tmp/nydus" - } - }, - "cache": { - "type": "filecache", - "filecache": { - "work_dir": "/tmp/nydus" - } - }, - "metadata_path": "RAFS_V5" - } - }"#; - let content = config - .replace("/tmp/nydus", tmpdir.as_path().to_str().unwrap()) - .replace("RAFS_V5", &source_path.display().to_string()); - let mut entry: BlobCacheEntry = serde_json::from_str(&content).unwrap(); - assert!(entry.prepare_configuration_info()); + let tmp_dir = TempDir::new().unwrap(); + let entry = create_bootstrap_entry(&tmp_dir); let mgr = BlobCacheMgr::new(); mgr.add_blob_entry(&entry).unwrap(); @@ -597,6 +563,8 @@ mod tests { assert!(mgr.get_config(&key).is_some()); let mgr = Arc::new(mgr); + // assert with wrong blob_id + assert!(BlockDevice::new_with_cache_manager(String::from("blob_id"), mgr.clone()).is_err()); let device = BlockDevice::new_with_cache_manager(blob_id, mgr).unwrap(); assert_eq!(device.blocks(), 0x209); @@ -642,4 +610,153 @@ mod tests { assert!(res.is_err()); }); } + + fn create_bootstrap_entry(tmp_dir: &TempDir) -> BlobCacheEntry { + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/blobs/be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); + let mut dest_path = tmp_dir.as_path().to_path_buf(); + dest_path.push("be7d77eeb719f70884758d1aa800ed0fb09d701aaec469964e9d54325f0d5fef"); + fs::copy(&source_path, &dest_path).unwrap(); + + let mut source_path = PathBuf::from(root_dir); + source_path.push("../tests/texture/bootstrap/rafs-v6-2.2.boot"); + let config = r#" + { + "type": "bootstrap", + "id": "rafs-v6", + "domain_id": "domain2", + "config_v2": { + "version": 2, + "id": "factory1", + "backend": { + "type": "localfs", + "localfs": { + "dir": "/tmp/nydus" + } + }, + "cache": { + "type": "filecache", + "filecache": { + "work_dir": "/tmp/nydus" + } + }, + "metadata_path": "RAFS_V5" + } + }"#; + + // config with non-existing path + let entry: BlobCacheEntry = serde_json::from_str(&config).unwrap(); + assert!(BlockDevice::new(entry).is_err()); + + // config with correct path + let content = config + .replace("/tmp/nydus", tmp_dir.as_path().to_str().unwrap()) + .replace("RAFS_V5", &source_path.display().to_string()); + let mut entry: BlobCacheEntry = serde_json::from_str(&content).unwrap(); + assert!(entry.prepare_configuration_info()); + entry + } + + fn create_block_device() -> BlockDevice { + let tmp_dir = TempDir::new().unwrap(); + let entry = create_bootstrap_entry(&tmp_dir); + + let device = BlockDevice::new(entry); + assert!(device.is_ok()); + let device = device.unwrap(); + assert_eq!(device.blocks(), 0x209); + + device + } + + #[test] + fn test_block_size() { + let mut device = create_block_device(); + + assert!(!device.is_tarfs_mode); + assert_eq!(device.block_size(), EROFS_BLOCK_SIZE_4096); + assert_ne!(device.block_size(), EROFS_BLOCK_SIZE_512); + + device.is_tarfs_mode = true; + assert_ne!(device.block_size(), EROFS_BLOCK_SIZE_4096); + assert_eq!(device.block_size(), EROFS_BLOCK_SIZE_512); + } + + #[test] + fn test_size_to_blocks() { + let mut device = create_block_device(); + + assert!(!device.is_tarfs_mode); + assert_eq!(device.size_to_blocks(0), 0); + assert_eq!(device.size_to_blocks(4096), 1); + assert_ne!(device.size_to_blocks(4096), 4096); + assert_ne!(device.size_to_blocks(4096), 8); + + device.is_tarfs_mode = true; + assert_eq!(device.size_to_blocks(0), 0); + assert_eq!(device.size_to_blocks(512), 1); + assert_ne!(device.size_to_blocks(512), 512); + assert_ne!(device.size_to_blocks(4096), 1); + } + + #[test] + fn test_blocks_to_size() { + let mut device = create_block_device(); + + assert!(!device.is_tarfs_mode); + assert_eq!(device.blocks_to_size(0), 0); + assert_eq!(device.blocks_to_size(1), 4096); + assert_ne!(device.blocks_to_size(4096), 4096); + assert_ne!(device.blocks_to_size(8), 4096); + + device.is_tarfs_mode = true; + assert_eq!(device.blocks_to_size(0), 0); + assert_eq!(device.blocks_to_size(1), 512); + assert_ne!(device.blocks_to_size(512), 512); + assert_ne!(device.blocks_to_size(1), 4096); + } + + fn sha256_digest(mut reader: R) -> Result { + let mut hasher = RafsDigest::hasher(digest::Algorithm::Sha256); + let mut buffer = [0; 1024]; + + loop { + let count = reader.read(&mut buffer)?; + if count == 0 { + break; + } + hasher.digest_update(&buffer[..count]); + } + + Ok(hasher.digest_finalize().into()) + } + + fn test_export_arg_thread(thread: u32) -> Result<()> { + let entry_tmp_dir = TempDir::new()?; + let entry = create_bootstrap_entry(&entry_tmp_dir); + + let tmp_dir = TempDir::new().unwrap(); + let data_dir = Some(String::from(tmp_dir.as_path().to_str().unwrap())); + + assert!(BlockDevice::export(entry, None, data_dir, thread, true).is_ok()); + + let mut disk_path = PathBuf::from(tmp_dir.as_path()); + disk_path.push("rafs-v6-2.2.boot.disk"); + let input = File::open(disk_path)?; + let reader = BufReader::new(input); + let sha256 = sha256_digest(reader)?; + assert_eq!( + sha256, + String::from("5684c330c622350c12d633d0773201f862b9955375d806670e1aaf36ef038b31") + ); + + Ok(()) + } + + #[test] + fn test_export() { + assert!(test_export_arg_thread(1).is_ok()); + assert!(test_export_arg_thread(2).is_ok()); + } } diff --git a/service/src/fs_cache.rs b/service/src/fs_cache.rs index a98fb1c5cf3..31812aa09ac 100644 --- a/service/src/fs_cache.rs +++ b/service/src/fs_cache.rs @@ -976,4 +976,71 @@ mod tests { FsCacheMsgHeader::try_from(vec![0u8, 0, 0, 1, 0, 0, 0, 2, 0, 0].as_slice()).unwrap_err(); FsCacheMsgHeader::try_from(vec![].as_slice()).unwrap_err(); } + + #[test] + fn test_fs_cache_msg_open_try_from() { + // request message size too small + assert!(FsCacheMsgOpen::try_from( + vec![1u8, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 2u8, 0, 0].as_slice() + ) + .is_err()); + + // volume key size or cookie key size too large + assert!(FsCacheMsgOpen::try_from( + vec![255u8, 127, 127, 127, 255, 127, 127, 255, 17, 0, 0, 0, 2u8, 0, 0, 0, 4u8, 0, 0, 0] + .as_slice() + ) + .is_err()); + assert!(FsCacheMsgOpen::try_from( + vec![ + 255u8, 127, 127, 127, 241u8, 127, 128, 128, 17, 0, 0, 0, 2u8, 0, 0, 0, 4u8, 0, 0, + 0, + ] + .as_slice() + ) + .is_err()); + + // value size too small + assert!(FsCacheMsgOpen::try_from( + vec![1u8, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 2u8, 0, 0, 0, 0].as_slice() + ) + .is_err()); + + let res = FsCacheMsgOpen::try_from( + vec![ + 1u8, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 2u8, 0, 0, 0, 4u8, 0, 0, 0, + ] + .as_slice(), + ); + assert!(res.is_ok()); + assert_eq!( + res.unwrap(), + FsCacheMsgOpen { + volume_key: String::from("\u{4}"), + cookie_key: String::from("\0\0"), + fd: 17, + flags: 2 + } + ); + } + + #[test] + fn test_fs_cache_msg_read_try_from() { + assert!(FsCacheMsgRead::try_from( + vec![1u8, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 2u8, 0, 0].as_slice() + ) + .is_err()); + + let res = FsCacheMsgRead::try_from( + vec![1u8, 0, 0, 0, 2, 0, 0, 0, 17, 0, 0, 0, 2u8, 0, 0, 0].as_slice(), + ); + assert!(res.is_ok()); + assert_eq!( + res.unwrap(), + FsCacheMsgRead { + off: 8589934593, + len: 8589934609, + } + ); + } } diff --git a/service/src/singleton.rs b/service/src/singleton.rs index 05129b5eea8..09addf09c9e 100644 --- a/service/src/singleton.rs +++ b/service/src/singleton.rs @@ -283,3 +283,143 @@ pub fn create_daemon( Ok(daemon) } + +#[cfg(test)] +mod tests { + use crate::blob_cache::generate_blob_key; + + use super::*; + use mio::{Poll, Token}; + use vmm_sys_util::tempdir::TempDir; + + fn create_service_controller() -> ServiceController { + let bti = BuildTimeInfo { + package_ver: String::from("package_ver"), + git_commit: String::from("git_commit"), + build_time: String::from("build_time"), + profile: String::from("profile"), + rustc: String::from("rustc"), + }; + + let (to_sm, _) = channel::(); + let (_, from_sm) = channel::>(); + + let poller = Poll::new().expect("Failed to create poller"); + let waker = Waker::new(poller.registry(), Token(1)).expect("Failed to create waker"); + + ServiceController { + bti, + id: Some(String::from("id")), + request_sender: Arc::new(Mutex::new(to_sm)), + result_receiver: Mutex::new(from_sm), + state: Default::default(), + supervisor: Some(String::from("supervisor")), + waker: Arc::new(waker), + blob_cache_mgr: Arc::new(BlobCacheMgr::new()), + fscache_enabled: AtomicBool::new(false), + #[cfg(target_os = "linux")] + fscache: Mutex::new(None), + } + } + + #[test] + #[cfg(target_os = "linux")] + fn test_initialize_fscache_service() { + let service_controller = create_service_controller(); + + assert!(service_controller + .initialize_fscache_service(None, None, "some path") + .is_err()); + + let mut p = std::env::current_dir().unwrap(); + p.push("Cargo.toml"); + assert!(service_controller + .initialize_fscache_service(None, None, p.to_str().unwrap()) + .is_err()); + + let tmp_dir = TempDir::new().unwrap(); + let dir = tmp_dir.as_path().to_str().unwrap(); + assert!(service_controller + .initialize_fscache_service(None, Some("1"), dir) + .is_ok()); + + assert_eq!(service_controller.id(), Some(String::from("id"))); + assert_eq!( + service_controller.version().build_time, + String::from("build_time") + ); + assert_eq!( + service_controller.supervisor(), + Some(String::from("supervisor")) + ); + } + + fn create_factory_config() -> String { + let config = r#"{ + "blobs": [{ + "type": "bootstrap", + "id": "rafs-v6", + "domain_id": "domain2", + "config_v2": { + "version": 2, + "id": "factory1", + "backend": { + "type": "localfs", + "localfs": { + "dir": "/tmp/nydus" + } + }, + "cache": { + "type": "fscache", + "fscache": { + "work_dir": "/tmp/nydus" + } + }, + "metadata_path": "RAFS_V5" + } + }] + }"#; + config.to_string() + } + + #[test] + fn test_initialize_blob_cache() { + let service_controller = create_service_controller(); + let blob_cache_mgr = service_controller.get_blob_cache_mgr().unwrap(); + let content = create_factory_config(); + let key = generate_blob_key("domain2", "rafs-v6"); + + // test first if + assert!(service_controller.initialize_blob_cache(&None).is_ok()); + assert!(blob_cache_mgr.get_config(&key).is_none()); + + //test second if + let config = serde_json::Value::Null; + assert!(service_controller + .initialize_blob_cache(&Some(config)) + .is_ok()); + assert!(blob_cache_mgr.get_config(&key).is_none()); + + // test third if + let cfg = content.replace("blobs", "blob"); + let config: serde_json::Value = serde_json::from_str(&cfg).unwrap(); + assert!(service_controller + .initialize_blob_cache(&Some(config)) + .is_ok()); + assert!(blob_cache_mgr.get_config(&key).is_none()); + + //test fourth if + let tmp_dir = TempDir::new().unwrap(); + let root_dir = &std::env::var("CARGO_MANIFEST_DIR").expect("$CARGO_MANIFEST_DIR"); + let mut source_path = std::path::PathBuf::from(root_dir); + source_path.push("../tests/texture/bootstrap/rafs-v6-2.2.boot"); + let cfg = content + .replace("/tmp/nydus", tmp_dir.as_path().to_str().unwrap()) + .replace("RAFS_V5", &source_path.display().to_string()); + let config: serde_json::Value = serde_json::from_str(&cfg).unwrap(); + assert!(service_controller + .initialize_blob_cache(&Some(config)) + .is_ok()); + assert!(blob_cache_mgr.get_config(&key).is_some()); + } +} From efa80a01887bc921955604a1a6d82f816ee0fe56 Mon Sep 17 00:00:00 2001 From: hijackthe2 <2948278083@qq.com> Date: Fri, 27 Oct 2023 13:54:19 +0800 Subject: [PATCH 2/3] docs: add fscache configuation Signed-off-by: hijackthe2 <2948278083@qq.com> --- docs/nydus-fscache.md | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/docs/nydus-fscache.md b/docs/nydus-fscache.md index b5dcf98adcb..08dccc08d44 100644 --- a/docs/nydus-fscache.md +++ b/docs/nydus-fscache.md @@ -58,6 +58,26 @@ $ sudo yum update kernel --enablerepo Plus $ sudo reboot ``` +## Enable fscache + +1. ``[ -c /dev/cachefiles ] && echo ok`` to test fscache is enable or not. If your result shows `ok`, then fscache has been already enabled; otherwise, please follow the following steps to enable fscache. + +2. Download cachefilesd package. For centos users, the command is: +``` +sudo yum install cachefilesd +``` + +3. Start cachefilesd deamon. +``` +sudo systemctl start cachefilesd +sudo systemctl status cachefilesd +``` + +4. Ensure the device file `/dev/cachefiles` is not occupied. If your result is not empty, please kill all processes the result shows. +``` +sudo lsof /dev/cachefiles +``` + ## Get ctr-remote and the fscache-supported nydusd 1. Make sure you have installed _rust 1.52.1_ version and golang. From 62adc285a90919b479cb1148a42d7343d582d9ef Mon Sep 17 00:00:00 2001 From: hijackthe2 <2948278083@qq.com> Date: Fri, 27 Oct 2023 15:18:45 +0800 Subject: [PATCH 3/3] ci: add configurations to setup fscache Since using `/dev/cachefiles` requires sudo mode, so some environment variables are defined and we use `sudo -E` to pass these environment variables to sudo operations. The script file for enabling fscache is misc/fscache/setup.sh Signed-off-by: hijackthe2 <2948278083@qq.com> --- .github/workflows/smoke.yml | 13 +++++++++++-- misc/fscache/setup.sh | 17 +++++++++++++++++ 2 files changed, 28 insertions(+), 2 deletions(-) create mode 100644 misc/fscache/setup.sh diff --git a/.github/workflows/smoke.yml b/.github/workflows/smoke.yml index 5a0cf852060..7522029f1d2 100644 --- a/.github/workflows/smoke.yml +++ b/.github/workflows/smoke.yml @@ -610,9 +610,13 @@ jobs: shared-key: nydus-build - name: Install cargo nextest uses: taiki-e/install-action@nextest + - name: Fscache Setup + run: sudo bash misc/fscache/setup.sh - name: Unit Test run: | - make ut-nextest + CARGO_HOME=${HOME}/.cargo + CARGO_BIN=$(which cargo) + sudo -E CARGO=${CARGO_BIN} make ut-nextest nydus-unit-test-coverage: runs-on: ubuntu-latest @@ -626,8 +630,13 @@ jobs: cache-on-failure: true - name: Install cargo-llvm-cov uses: taiki-e/install-action@cargo-llvm-cov + - name: Fscache Setup + run: sudo bash misc/fscache/setup.sh - name: Generate code coverage - run: make coverage-codecov + run: | + CARGO_HOME=${HOME}/.cargo + CARGO_BIN=$(which cargo) + sudo -E CARGO=${CARGO_BIN} make coverage-codecov - name: Upload coverage to Codecov uses: codecov/codecov-action@v3 with: diff --git a/misc/fscache/setup.sh b/misc/fscache/setup.sh new file mode 100644 index 00000000000..da312b53ece --- /dev/null +++ b/misc/fscache/setup.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# This script should be executed in root mode! + +apt update +apt install -y cachefilesd +apt list --installed | grep cachefilesd +chmod a+w /etc/default/cachefilesd +sed -i 's/#RUN=yes/RUN=yes/' /etc/default/cachefilesd +cat /etc/default/cachefilesd +/sbin/modprobe -qab cachefiles +/sbin/cachefilesd -f /etc/cachefilesd.conf +systemctl status cachefilesd +[ -c /dev/cachefiles ] && echo "cachefilesd is successfully enabled" +pid=$(lsof /dev/cachefiles | awk '{if (NR>1) {print $2}}') +kill -9 $pid +echo "/dev/cachefiles is available now"