From 4c17538fba505f07975415a634d32bfed3a9e6ab Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 16 Jan 2024 10:29:00 +0800 Subject: [PATCH 1/5] qcow2-rs: add Qcow2IoBuf page_aligned_vec!() is like one hacky, and it may trigger STATUS_HEAP_CORRUPTION when running 'cargo test'. The reason is that windows with 'cargo test' requires Vec buffer to be freed with same alloc layout(alignment, size). Obviously, page_aligned_vec!() can be implemented in that way. Signed-off-by: Ming Lei --- src/helpers.rs | 89 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 89 insertions(+) diff --git a/src/helpers.rs b/src/helpers.rs index 5523574..4b3128b 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -4,6 +4,7 @@ #![allow(dead_code)] use crate::error::Qcow2Result; use core::future::Future; +use std::ops::{Deref, DerefMut}; use std::pin::Pin; #[macro_export] @@ -106,6 +107,94 @@ macro_rules! page_aligned_vec { }}; } +/// Slice like buffer, which address is aligned with 4096. +/// +pub struct Qcow2IoBuf { + ptr: *mut T, + size: usize, +} + +impl<'a, T> Qcow2IoBuf { + pub fn new(size: usize) -> Self { + let layout = std::alloc::Layout::from_size_align(size, 4096).unwrap(); + let ptr = unsafe { std::alloc::alloc(layout) } as *mut T; + + assert!(size != 0); + + Qcow2IoBuf { ptr, size } + } + + /// how many elements in this buffer + pub fn len(&self) -> usize { + let elem_size = core::mem::size_of::(); + self.size / elem_size + } + + /// Return raw address of this buffer + pub fn as_ptr(&self) -> *const T { + self.ptr + } + + /// Return mutable raw address of this buffer + pub fn as_mut_ptr(&self) -> *mut T { + self.ptr + } + + /// slice with u8 element, only for RefBlock + pub(crate) fn as_u8_slice(&self) -> &[u8] { + unsafe { std::slice::from_raw_parts(self.ptr as *const u8, self.size) } + } + + /// mutable slice with u8 element, only for RefBlock + pub(crate) fn as_u8_slice_mut(&mut self) -> &mut [u8] { + unsafe { std::slice::from_raw_parts_mut(self.ptr as *mut u8, self.size) } + } + + /// fill zero for every bits of this buffer + pub fn zero_buf(&mut self) { + unsafe { + std::ptr::write_bytes(self.as_mut_ptr(), 0, self.len()); + } + } +} + +impl std::fmt::Debug for Qcow2IoBuf { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!( + f, + "ptr {:?} size {} element type {}", + self.ptr, + self.size, + qcow2_type_of(unsafe { &*self.ptr }) + ) + } +} + +/// Slice reference of this buffer +impl Deref for Qcow2IoBuf { + type Target = [T]; + fn deref(&self) -> &[T] { + let elem_size = core::mem::size_of::(); + unsafe { std::slice::from_raw_parts(self.ptr, self.size / elem_size) } + } +} + +/// Mutable slice reference of this buffer +impl DerefMut for Qcow2IoBuf { + fn deref_mut(&mut self) -> &mut [T] { + let elem_size = core::mem::size_of::(); + unsafe { std::slice::from_raw_parts_mut(self.ptr, self.size / elem_size) } + } +} + +/// Free buffer with same alloc layout +impl Drop for Qcow2IoBuf { + fn drop(&mut self) { + let layout = std::alloc::Layout::from_size_align(self.size, 4096).unwrap(); + unsafe { std::alloc::dealloc(self.ptr as *mut u8, layout) }; + } +} + /// It is user's responsibility to not free buffer of the slice pub fn slice_to_vec(s: &[T]) -> Vec { // Get a pointer to the data and its length from the existing slice From ea5e36a64638db99badd92ee8faf9d2eb663835e Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 16 Jan 2024 17:40:02 +0800 Subject: [PATCH 2/5] qcow2-rs: convert meta data handling into Qcow2IoBuf Meta handling is the most tricky part wrt. using page_aligned_vec!(). Move Qcow2IoBuf object into Meta instance(L1, L2, RefTable, RefBlock) directly, and retrieve everything from this buffer. Turns out this way is simpler than before. Signed-off-by: Ming Lei --- src/helpers.rs | 4 ++ src/meta.rs | 109 +++++++++++++++++++++++-------------------------- 2 files changed, 56 insertions(+), 57 deletions(-) diff --git a/src/helpers.rs b/src/helpers.rs index 4b3128b..e630165 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -114,6 +114,10 @@ pub struct Qcow2IoBuf { size: usize, } +// Users of Qcow2IoBuf has to deal with Send & Sync +unsafe impl Send for Qcow2IoBuf {} +unsafe impl Sync for Qcow2IoBuf {} + impl<'a, T> Qcow2IoBuf { pub fn new(size: usize) -> Self { let layout = std::alloc::Layout::from_size_align(size, 4096).unwrap(); diff --git a/src/meta.rs b/src/meta.rs index 4450f69..26548f8 100644 --- a/src/meta.rs +++ b/src/meta.rs @@ -5,9 +5,8 @@ use crate::dev::Qcow2Info; use crate::error::Qcow2Result; use crate::helpers::IntAlignment; +use crate::helpers::Qcow2IoBuf; use crate::numerical_enum; -use crate::page_aligned_vec; -use crate::zero_buf; use bincode::Options; use serde::{Deserialize, Serialize}; use std::cell::RefCell; @@ -962,13 +961,13 @@ impl TableEntry for L1Entry { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug)] pub struct L1Table { header_entries: u32, dirty_blocks: RefCell>, bs_bits: u8, offset: Option, - data: Box<[L1Entry]>, + data: Qcow2IoBuf, } impl L1Table { @@ -989,12 +988,12 @@ impl L1Table { pub fn clone_and_grow(&self, at_least_index: usize, cluster_size: usize) -> Self { let new_size = std::cmp::max(at_least_index + 1, self.data.len()); let new_size = new_size.align_up(cluster_size).unwrap(); - let mut new_data = page_aligned_vec!(L1Entry, new_size); + let mut new_data = Qcow2IoBuf::::new(new_size); new_data[..self.data.len()].copy_from_slice(&self.data); Self { offset: None, - data: new_data.into_boxed_slice(), + data: new_data, bs_bits: self.bs_bits, header_entries: self.data.len() as u32, dirty_blocks: RefCell::new(self.dirty_blocks.borrow().clone()), @@ -1015,13 +1014,14 @@ impl L1Table { impl_top_table_traits!(L1Table, L1Entry, data); -impl From> for L1Table { - fn from(data: Box<[L1Entry]>) -> Self { +impl From> for L1Table { + fn from(data: Qcow2IoBuf) -> Self { Self { + bs_bits: 0, + header_entries: 0, offset: None, data, dirty_blocks: RefCell::new(VecDeque::new()), - ..Default::default() } } } @@ -1356,11 +1356,11 @@ impl TableEntry for L2Entry { // return cluster_offset + (offset % cluster_size) // // [*] this changes if Extended L2 Entries are enabled, see next section -#[derive(Debug, Clone)] +#[derive(Debug)] pub struct L2Table { offset: Option, cluster_bits: u32, - data: Box<[L2Entry]>, + data: Qcow2IoBuf, } impl L2Table { @@ -1428,8 +1428,8 @@ impl L2Table { } } -impl From> for L2Table { - fn from(data: Box<[L2Entry]>) -> Self { +impl From> for L2Table { + fn from(data: Qcow2IoBuf) -> Self { Self { offset: None, cluster_bits: 0, @@ -1496,12 +1496,12 @@ impl TableEntry for RefTableEntry { } } -#[derive(Debug, Clone, Default)] +#[derive(Debug)] pub struct RefTable { dirty_blocks: RefCell>, bs_bits: u8, offset: Option, - data: Box<[RefTableEntry]>, + data: Qcow2IoBuf, } impl RefTable { @@ -1525,13 +1525,13 @@ impl RefTable { (clusters * cluster_size + bs, None) }; - let mut new_data = page_aligned_vec!(RefTableEntry, new_size); - zero_buf!(new_data); + let mut new_data = Qcow2IoBuf::::new(new_size); + new_data.zero_buf(); new_data[..self.data.len()].copy_from_slice(&self.data); Self { offset: new_off, - data: new_data.into_boxed_slice(), + data: new_data, dirty_blocks: RefCell::new(self.dirty_blocks.borrow().clone()), bs_bits: self.bs_bits, } @@ -1550,12 +1550,13 @@ impl RefTable { } } -impl From> for RefTable { - fn from(data: Box<[RefTableEntry]>) -> Self { +impl From> for RefTable { + fn from(data: Qcow2IoBuf) -> Self { Self { data, dirty_blocks: RefCell::new(VecDeque::new()), - ..Default::default() + bs_bits: 0, + offset: None, } } } @@ -1586,7 +1587,7 @@ impl TableEntry for RefBlockEntry { #[derive(Debug)] pub struct RefBlock { offset: Option, - raw_data: Box<[u8]>, + raw_data: Qcow2IoBuf, refcount_order: u8, } @@ -1604,35 +1605,35 @@ impl RefBlock { #[inline(always)] fn __get(&self, index: usize) -> u64 { + let raw_data = &self.raw_data.as_u8_slice(); match self.refcount_order { // refcount_bits == 1 - 0 => ((self.raw_data[index / 8] >> (index % 8)) & 0b0000_0001) as u64, + 0 => ((raw_data[index / 8] >> (index % 8)) & 0b0000_0001) as u64, // refcount_bits == 2 - 1 => ((self.raw_data[index / 4] >> (index % 4)) & 0b0000_0011) as u64, + 1 => ((raw_data[index / 4] >> (index % 4)) & 0b0000_0011) as u64, // refcount_bits == 4 - 2 => ((self.raw_data[index / 2] >> (index % 2)) & 0b0000_1111) as u64, + 2 => ((raw_data[index / 2] >> (index % 2)) & 0b0000_1111) as u64, // refcount_bits == 8 - 3 => self.raw_data[index] as u64, + 3 => raw_data[index] as u64, // refcount_bits == 16 - 4 => u16::from_be_bytes(self.raw_data[index * 2..index * 2 + 2].try_into().unwrap()) - as u64, + 4 => u16::from_be_bytes(raw_data[index * 2..index * 2 + 2].try_into().unwrap()) as u64, // refcount_bits == 32 - 5 => u32::from_be_bytes(self.raw_data[index * 4..index * 4 + 4].try_into().unwrap()) - as u64, + 5 => u32::from_be_bytes(raw_data[index * 4..index * 4 + 4].try_into().unwrap()) as u64, // refcount_bits == 64 - 6 => u64::from_be_bytes(self.raw_data[index * 8..index * 8 + 8].try_into().unwrap()), + 6 => u64::from_be_bytes(raw_data[index * 8..index * 8 + 8].try_into().unwrap()), _ => unreachable!(), } } fn __set(&mut self, index: usize, value: u64) -> Qcow2Result<()> { + let raw_data = &mut self.raw_data.as_u8_slice_mut(); match self.refcount_order { // refcount_bits == 1 0 => { @@ -1643,8 +1644,7 @@ impl RefBlock { ) .into()); } - self.raw_data[index / 8] = (self.raw_data[index / 8] - & !(0b0000_0001 << (index % 8))) + raw_data[index / 8] = (raw_data[index / 8] & !(0b0000_0001 << (index % 8))) | ((value as u8) << (index % 8)); } @@ -1657,8 +1657,7 @@ impl RefBlock { ) .into()); } - self.raw_data[index / 4] = (self.raw_data[index / 4] - & !(0b0000_0011 << (index % 4))) + raw_data[index / 4] = (raw_data[index / 4] & !(0b0000_0011 << (index % 4))) | ((value as u8) << (index % 4)); } @@ -1671,8 +1670,7 @@ impl RefBlock { ) .into()); } - self.raw_data[index / 2] = (self.raw_data[index / 2] - & !(0b0000_1111 << (index % 2))) + raw_data[index / 2] = (raw_data[index / 2] & !(0b0000_1111 << (index % 2))) | ((value as u8) << (index % 2)); } @@ -1685,7 +1683,7 @@ impl RefBlock { ) .into()); } - self.raw_data[index] = value as u8; + raw_data[index] = value as u8; } // refcount_bits == 16 @@ -1697,8 +1695,8 @@ impl RefBlock { ) .into()); } - self.raw_data[index * 2] = (value >> 8) as u8; - self.raw_data[index * 2 + 1] = value as u8; + raw_data[index * 2] = (value >> 8) as u8; + raw_data[index * 2 + 1] = value as u8; } // refcount_bits == 32 @@ -1710,15 +1708,15 @@ impl RefBlock { ) .into()); } - self.raw_data[index * 4] = (value >> 24) as u8; - self.raw_data[index * 4 + 1] = (value >> 16) as u8; - self.raw_data[index * 4 + 2] = (value >> 8) as u8; - self.raw_data[index * 4 + 3] = value as u8; + raw_data[index * 4] = (value >> 24) as u8; + raw_data[index * 4 + 1] = (value >> 16) as u8; + raw_data[index * 4 + 2] = (value >> 8) as u8; + raw_data[index * 4 + 3] = value as u8; } // refcount_bits == 64 6 => { - let array: &mut [u8; 8] = (&mut self.raw_data[index * 8..index * 8 + 8]) + let array: &mut [u8; 8] = (&mut raw_data[index * 8..index * 8 + 8]) .try_into() .unwrap(); *array = value.to_be_bytes(); @@ -1811,7 +1809,7 @@ impl Table for RefBlock { impl_table_gen_funcs!(raw_data); fn entries(&self) -> usize { - self.raw_data.len() * 8 / (1 << self.refcount_order) + self.byte_size() * 8 / (1 << self.refcount_order) } fn get(&self, index: usize) -> Self::Entry { @@ -1826,21 +1824,18 @@ impl Table for RefBlock { self.__set(index, value.into_plain()) } + /// RefBlock is special, since RefBlockEntry is defined as u64 fn byte_size(&self) -> usize { - self.raw_data.len() + self.raw_data.len() * 8 } } -impl From> for RefBlock { - fn from(data: Box<[RefBlockEntry]>) -> Self { - let len = data.len() * size_of::(); - let ptr = Box::into_raw(data); - let _data: Box<[u8]> = - unsafe { Box::from_raw(std::slice::from_raw_parts_mut(ptr as *mut u8, len)) }; +impl From> for RefBlock { + fn from(data: Qcow2IoBuf) -> Self { Self { offset: None, refcount_order: 0, - raw_data: _data, + raw_data: data, } } } @@ -1859,7 +1854,7 @@ where } } -pub trait Table: From> { +pub trait Table: From> { type Entry: TableEntry; fn entries(&self) -> usize; @@ -1889,11 +1884,11 @@ pub trait Table: From> { } fn new_empty(offset: Option, size: usize) -> Self { - let mut table = page_aligned_vec!(Self::Entry, size); + let table = Qcow2IoBuf::::new(size); unsafe { std::ptr::write_bytes(table.as_mut_ptr(), 0, table.len()); } - let mut table: Self = table.into_boxed_slice().into(); + let mut table: Self = table.into(); table.set_offset(offset); table From 83d17df6cdac26528eb68b3d0992563dcd6e35e2 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 16 Jan 2024 11:43:18 +0000 Subject: [PATCH 3/5] qcow2-rs: convert library page_aliged_vec!() into Qcow2IoBuf All conversions are straightforward. Signed-off-by: Ming Lei --- src/dev.rs | 19 ++++++++++--------- src/meta.rs | 2 +- src/sync_io.rs | 4 ++-- src/tokio_io.rs | 4 ++-- src/utils.rs | 6 +++--- 5 files changed, 18 insertions(+), 17 deletions(-) diff --git a/src/dev.rs b/src/dev.rs index 59ae48a..502eccd 100644 --- a/src/dev.rs +++ b/src/dev.rs @@ -1,7 +1,7 @@ use crate::cache::AsyncLruCache; use crate::cache::AsyncLruCacheEntry; use crate::error::Qcow2Result; -use crate::helpers::qcow2_type_of; +use crate::helpers::{qcow2_type_of, Qcow2IoBuf}; use crate::meta::{ L1Entry, L1Table, L2Entry, L2Table, Mapping, MappingSource, Qcow2Header, RefBlock, RefTable, RefTableEntry, SplitGuestOffset, Table, TableEntry, @@ -500,10 +500,10 @@ impl Qcow2Dev { let res = self.file.fallocate(offset, len, flags).await; match res { Err(_) => { - let mut zero_data = crate::page_aligned_vec!(u8, len); - zero_buf!(zero_data); + let mut zero_data = Qcow2IoBuf::::new(len); log::trace!("discard fallback off {:x} len {}", offset, len); + zero_data.zero_buf(); self.call_write(offset, &zero_data).await } Ok(_) => Ok(()), @@ -1537,7 +1537,7 @@ impl Qcow2Dev { let pad = (compressed_offset - aligned_off) as usize; let aligned_len = (pad + compressed_length + bs - 1) & (bs_mask as usize); - let mut _compressed_data = crate::page_aligned_vec!(u8, aligned_len); + let mut _compressed_data = Qcow2IoBuf::::new(aligned_len); let res = self.call_read(aligned_off, &mut _compressed_data).await?; if res != aligned_len { return Err("do_read_compressed: short read compressed data".into()); @@ -1880,7 +1880,7 @@ impl Qcow2Dev { host_off: u64, compressed_mapping: &Mapping, ) -> Qcow2Result<()> { - let mut cbuf = crate::page_aligned_vec!(u8, self.info.cluster_size()); + let mut cbuf = Qcow2IoBuf::::new(self.info.cluster_size()); // copy & write self.do_read_compressed(compressed_mapping.clone(), 0, &mut cbuf) @@ -1899,7 +1899,7 @@ impl Qcow2Dev { ) -> Qcow2Result<()> { match self.backing_file.as_ref() { Some(backing) => { - let mut cbuf = crate::page_aligned_vec!(u8, self.info.cluster_size()); + let mut cbuf = Qcow2IoBuf::::new(self.info.cluster_size()); // copy & write backing @@ -2781,9 +2781,10 @@ impl Qcow2Dev { #[cfg(test)] mod tests { use crate::dev::*; + use crate::helpers::Qcow2IoBuf; + use crate::qcow2_default_params; use crate::tokio_io::Qcow2IoTokio; use crate::utils::qcow2_setup_dev_tokio; - use crate::{page_aligned_vec, qcow2_default_params}; use std::path::PathBuf; use tokio::runtime::Runtime; use utilities::*; @@ -2909,7 +2910,7 @@ mod tests { let size = 64_u64 << 20; let img_file = make_temp_qcow2_img(size, 16, 4); let io = Qcow2IoTokio::new(&img_file.path().to_path_buf(), true, false).await; - let mut buf = page_aligned_vec!(u8, 4096); + let mut buf = Qcow2IoBuf::::new(4096); let _ = io.read_to(0, &mut buf).await; let header = Qcow2Header::from_buf(&buf).unwrap(); @@ -2929,7 +2930,7 @@ mod tests { let params = qcow2_default_params!(true, false); let dev = qcow2_setup_dev_tokio(&path, ¶ms).await.unwrap(); - let mut buf = page_aligned_vec!(u8, 1 << cluster_bits); + let mut buf = Qcow2IoBuf::::new(1 << cluster_bits); dev.read_at(&mut buf, 0).await.unwrap(); }); } diff --git a/src/meta.rs b/src/meta.rs index 26548f8..5074961 100644 --- a/src/meta.rs +++ b/src/meta.rs @@ -420,7 +420,7 @@ impl Qcow2Header { pub const MAX_L1_SIZE: u32 = 32_u32 << 20; pub const MAX_REFCOUNT_TABLE_SIZE: u32 = 8_u32 << 20; - pub fn from_buf(header_buf: &Vec) -> Qcow2Result { + pub fn from_buf(header_buf: &[u8]) -> Qcow2Result { let bincode = bincode::DefaultOptions::new() .with_fixint_encoding() .with_big_endian(); diff --git a/src/sync_io.rs b/src/sync_io.rs index 8bfc062..c4f30e8 100644 --- a/src/sync_io.rs +++ b/src/sync_io.rs @@ -120,9 +120,9 @@ impl Qcow2IoOps for Qcow2IoSync { } #[cfg(not(target_os = "linux"))] async fn fallocate(&self, offset: u64, len: usize, _flags: u32) -> Qcow2Result<()> { - let mut data = crate::page_aligned_vec!(u8, len); - crate::zero_buf!(data); + let mut data = crate::helpers::Qcow2IoBuf::::new(len); + data.zero_buf(); self.write_at(offset, &data).await } diff --git a/src/tokio_io.rs b/src/tokio_io.rs index c0cf2c3..4ea6d52 100644 --- a/src/tokio_io.rs +++ b/src/tokio_io.rs @@ -95,9 +95,9 @@ impl Qcow2IoOps for Qcow2IoTokio { } #[cfg(not(target_os = "linux"))] async fn fallocate(&self, offset: u64, len: usize, _flags: u32) -> Qcow2Result<()> { - let mut data = crate::page_aligned_vec!(u8, len); - crate::zero_buf!(data); + let mut data = crate::helpers::Qcow2IoBuf::::new(len); + data.zero_buf(); self.write_at(offset, &data).await } diff --git a/src/utils.rs b/src/utils.rs index 762b2f6..7493f00 100644 --- a/src/utils.rs +++ b/src/utils.rs @@ -1,8 +1,8 @@ use crate::dev::{Qcow2Dev, Qcow2DevParams}; use crate::error::Qcow2Result; +use crate::helpers::Qcow2IoBuf; use crate::meta::Qcow2Header; use crate::ops::*; -use crate::page_aligned_vec; #[cfg(not(target_os = "windows"))] use crate::sync_io::Qcow2IoSync; #[cfg(target_os = "linux")] @@ -22,7 +22,7 @@ pub fn qcow2_alloc_dev_sync( io: T, params: &Qcow2DevParams, ) -> Qcow2Result<(Qcow2Dev, Option)> { - let mut buf = page_aligned_vec!(u8, 4096); + let mut buf = Qcow2IoBuf::::new(4096); { use std::io::Read; let mut file = std::fs::File::open(path).unwrap(); @@ -45,7 +45,7 @@ pub async fn qcow2_alloc_dev( io: T, params: &Qcow2DevParams, ) -> Qcow2Result<(Qcow2Dev, Option)> { - let mut buf = page_aligned_vec!(u8, 4096); + let mut buf = Qcow2IoBuf::::new(4096); let _ = io.read_to(0, &mut buf).await; let header = Qcow2Header::from_buf(&buf)?; let back_path = match header.backing_filename() { From 0ca0e8b59cf52b79736e5293486c4d405ab3c831 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Tue, 16 Jan 2024 12:04:50 +0000 Subject: [PATCH 4/5] qcow2-rs: tests: convert all remained page_aliged_vec!() into Qcow2IoBuf Signed-off-by: Ming Lei --- README.md | 2 +- src/helpers.rs | 18 ------------------ src/main.rs | 6 +++--- tests/basic.rs | 23 ++++++++++++----------- tests/sync_io.rs | 4 ++-- utilities/src/lib.rs | 13 +++++++------ 6 files changed, 25 insertions(+), 41 deletions(-) diff --git a/README.md b/README.md index 26d3217..9519109 100644 --- a/README.md +++ b/README.md @@ -43,7 +43,7 @@ host cluster leak, format qcow2 image, .... .await .unwrap(); - let mut buf = qcow2_rs::page_aligned_vec!(u8, 4096); + let mut buf = qcow2_rs::helpers::Qcow2IoBuf::::new(4096); // read 4096 bytes to `buf` from virt offset 0 of `test.qcow2` let _ = dev.read_at(&mut buf, 0).await.unwrap(); diff --git a/src/helpers.rs b/src/helpers.rs index e630165..2333cc3 100644 --- a/src/helpers.rs +++ b/src/helpers.rs @@ -89,24 +89,6 @@ impl_int_alignment_for_primitive!(usize); pub type BoxedFuture<'a, T> = Pin + 'a>>; pub type Qcow2FutureResult<'a, T> = BoxedFuture<'a, Qcow2Result>; -#[macro_export] -macro_rules! page_aligned_vec { - ($type:ty, $size:expr) => {{ - #[repr(C, align(4096))] - #[derive(Clone)] - struct PageAlignedBuf([u8; 512]); - - let sz = ($size + 511) & !511; - let nr = sz / 512; - let buf = Vec::::with_capacity(nr); - unsafe { - let mut a: Vec<$type> = std::mem::transmute(buf); - a.set_len(sz / core::mem::size_of::<$type>()); - a - } - }}; -} - /// Slice like buffer, which address is aligned with 4096. /// pub struct Qcow2IoBuf { diff --git a/src/main.rs b/src/main.rs index 9af5441..0ad41a9 100755 --- a/src/main.rs +++ b/src/main.rs @@ -2,10 +2,10 @@ use clap::{Args, Parser, Subcommand}; use clap_num::maybe_hex; use qcow2_rs::dev::{Qcow2DevParams, Qcow2Info}; use qcow2_rs::error::Qcow2Result; +use qcow2_rs::helpers::Qcow2IoBuf; use qcow2_rs::meta::{ L1Table, L2Table, Qcow2FeatureType, Qcow2Header, RefBlock, RefTable, Table, TableEntry, }; -use qcow2_rs::page_aligned_vec; use qcow2_rs::utils::qcow2_setup_dev_tokio; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::PathBuf; @@ -620,7 +620,7 @@ fn read_qcow2(args: ReadArgs) -> Qcow2Result<()> { eprintln!("unaligned offset {:x} or len {:x}", args.start, len); } - let mut buf = page_aligned_vec!(u8, len); + let mut buf = Qcow2IoBuf::::new(len); dev.read_at(&mut buf, args.start) .await @@ -648,7 +648,7 @@ fn write_qcow2(args: WriteArgs) -> Qcow2Result<()> { eprintln!("unaligned offset {:x} or len {:x}", args.start, len); } - let mut buf = page_aligned_vec!(u8, len); + let mut buf = Qcow2IoBuf::::new(len); let pattern = (0..=15).cycle().take(buf.len()); for (elem, pattern_element) in buf.iter_mut().zip(pattern) { diff --git a/tests/basic.rs b/tests/basic.rs index dc50f96..ea2e26c 100644 --- a/tests/basic.rs +++ b/tests/basic.rs @@ -4,8 +4,9 @@ extern crate utilities; mod integretion { use crypto_hash::{hex_digest, Algorithm}; use qcow2_rs::dev::*; + use qcow2_rs::helpers::Qcow2IoBuf; + use qcow2_rs::qcow2_default_params; use qcow2_rs::utils::*; - use qcow2_rs::{page_aligned_vec, qcow2_default_params}; use rand::Rng; use std::io::Read; use std::path::PathBuf; @@ -42,8 +43,8 @@ mod integretion { let img_file = make_temp_qcow2_img(size, cluster_bits, 4); let path = PathBuf::from(img_file.path()); let params = qcow2_default_params!(false, false); - let mut buf = page_aligned_vec!(u8, size as usize); - let mut buf2 = page_aligned_vec!(u8, size as usize); + let mut buf = Qcow2IoBuf::::new(size as usize); + let mut buf2 = Qcow2IoBuf::::new(size as usize); let bsize = size as usize; let boff = 8192; @@ -99,7 +100,7 @@ mod integretion { let bsize = 512_u64 << 10; // build data source from /dev/random - let mut buf = page_aligned_vec!(u8, bsize as usize); + let mut buf = Qcow2IoBuf::::new(bsize as usize); let raw_f = make_rand_raw_img(bsize, cluster_bits); let raw_path = raw_f.path().to_str().unwrap(); let mut file = std::fs::File::open(raw_path).unwrap(); @@ -140,7 +141,7 @@ mod integretion { for off in (0..size).step_by(rsize) { let start = Instant::now(); - let mut buf = page_aligned_vec!(u8, rsize); + let mut buf = Qcow2IoBuf::::new(rsize); let r = dev.read_at(&mut buf, off).await.unwrap(); assert!(r == buf.len()); duration += start.elapsed(); @@ -171,7 +172,7 @@ mod integretion { let params = qcow2_default_params!(false, false); // build data source from /dev/random - let mut buf = page_aligned_vec!(u8, size as usize); + let mut buf = Qcow2IoBuf::::new(size as usize); let raw_f = make_rand_raw_img(size, cluster_bits); let raw_path = raw_f.path().to_str().unwrap(); let mut file = std::fs::File::open(raw_path).unwrap(); @@ -180,7 +181,7 @@ mod integretion { let dev = qcow2_setup_dev_tokio(&path, ¶ms).await.unwrap(); let write = dev.write_at(&buf, 0); - let mut rbuf = page_aligned_vec!(u8, size as usize); + let mut rbuf = Qcow2IoBuf::::new(size as usize); let read = dev.read_at(&mut rbuf, 0); //run write and read concurrently @@ -256,7 +257,7 @@ mod integretion { let off = rng.gen_range(0..blocks) * bs; let bsize = rng.gen_range(min_bs..=max_bs) * bs; - let mut wbuf = page_aligned_vec!(u8, bsize as usize); + let mut wbuf = Qcow2IoBuf::::new(bsize as usize); rng.fill(&mut wbuf[..]); println!("randwrite: off {:x} len {}", off, bsize); @@ -271,7 +272,7 @@ mod integretion { let off = rng.gen_range(0..blocks) * bs; let bsize = rng.gen_range(min_bs..=max_bs) * bs; - let mut rbuf = page_aligned_vec!(u8, bsize as usize); + let mut rbuf = Qcow2IoBuf::::new(bsize as usize); println!("randread: off {:x} len {}", off, bsize); d.read_at(&mut rbuf, off).await.unwrap(); @@ -309,7 +310,7 @@ mod integretion { // write over resized device backing by external image let buf_len = 4 << cluster_bits; let mut rng = rand::thread_rng(); - let mut wbuf = page_aligned_vec!(u8, buf_len); + let mut wbuf = Qcow2IoBuf::::new(buf_len); rng.fill(&mut wbuf[..]); let off = (1 << cluster_bits) / 2; //cross cluster write let wbuf_md5 = hex_digest(Algorithm::MD5, &wbuf); @@ -319,7 +320,7 @@ mod integretion { let dev = qcow2_setup_dev_tokio(&path, ¶ms).await.unwrap(); dev.write_at(&wbuf, off as u64).await.unwrap(); - let mut rbuf = page_aligned_vec!(u8, buf_len); + let mut rbuf = Qcow2IoBuf::::new(buf_len); dev.read_at(&mut rbuf, off as u64).await.unwrap(); let rbuf_md5 = hex_digest(Algorithm::MD5, &rbuf); diff --git a/tests/sync_io.rs b/tests/sync_io.rs index a01d15c..d84e6ec 100644 --- a/tests/sync_io.rs +++ b/tests/sync_io.rs @@ -5,8 +5,8 @@ extern crate utilities; mod sync_io_integretion { use crypto_hash::{hex_digest, Algorithm}; use qcow2_rs::dev::*; + use qcow2_rs::qcow2_default_params; use qcow2_rs::utils::*; - use qcow2_rs::{page_aligned_vec, qcow2_default_params}; use std::path::PathBuf; use tokio::runtime::Runtime; use utilities::*; @@ -18,7 +18,7 @@ mod sync_io_integretion { dev.qcow2_prep_io().await.unwrap(); - let mut buf = page_aligned_vec!(u8, size as usize); + let mut buf = qcow2_rs::helpers::Qcow2IoBuf::::new(size as usize); dev.read_at(&mut buf, 0).await.unwrap(); hex_digest(Algorithm::MD5, &buf) diff --git a/utilities/src/lib.rs b/utilities/src/lib.rs index 3dd9821..657ca30 100644 --- a/utilities/src/lib.rs +++ b/utilities/src/lib.rs @@ -1,9 +1,10 @@ use crypto_hash::{hex_digest, Algorithm}; use qcow2_rs::dev::{Qcow2Dev, Qcow2DevParams}; +use qcow2_rs::helpers::Qcow2IoBuf; use qcow2_rs::meta::Qcow2Header; use qcow2_rs::ops::Qcow2IoOps; +use qcow2_rs::qcow2_default_params; use qcow2_rs::utils::qcow2_setup_dev_tokio; -use qcow2_rs::{page_aligned_vec, qcow2_default_params}; use rand::Rng; use std::io::{Read, Seek, SeekFrom, Write}; use std::path::PathBuf; @@ -159,10 +160,9 @@ pub async fn calculate_qcow2_data_md5( let path = PathBuf::from(qcow2f.path()); let params = qcow2_default_params!(true, false); let dev = qcow2_setup_dev_tokio(&path, ¶ms).await.unwrap(); + let mut buf = Qcow2IoBuf::::new(size as usize); - let mut buf = page_aligned_vec!(u8, size as usize); dev.read_at(&mut buf, off).await.unwrap(); - hex_digest(Algorithm::MD5, &buf) } @@ -177,7 +177,7 @@ pub async fn test_qcow2_dev_write_verify( for (off, len) in input { let d = dev.clone(); fv.push(local.spawn_local(async move { - let mut wbuf = page_aligned_vec!(u8, len as usize); + let mut wbuf = Qcow2IoBuf::::new(len as usize); let mut rng = rand::thread_rng(); rng.fill(&mut wbuf[..]); @@ -186,7 +186,8 @@ pub async fn test_qcow2_dev_write_verify( d.write_at(&wbuf, off).await.unwrap(); println!("write at {:x}/{}..done", off, len); - let mut rbuf = page_aligned_vec!(u8, len as usize); + let mut rbuf = Qcow2IoBuf::::new(len as usize); + d.read_at(&mut rbuf, off).await.unwrap(); let w_sum = hex_digest(Algorithm::MD5, &wbuf); @@ -222,7 +223,7 @@ pub async fn test_cow_write( let size = dev.info.virtual_size(); let buf_size = 2 << cluster_bits; let off = (buf_size / 4) as u64; - let mut buf = page_aligned_vec!(u8, buf_size); + let mut buf = Qcow2IoBuf::::new(buf_size); let mut rng = rand::thread_rng(); rng.fill(&mut buf[..]); let buf_md5 = hex_digest(Algorithm::MD5, &buf); From 779b5f4dce223f16ca55bce4b782f5bfa9314945 Mon Sep 17 00:00:00 2001 From: Ming Lei Date: Mon, 15 Jan 2024 08:46:54 +0000 Subject: [PATCH 5/5] qcow2-rs: ci: add windows ci Now it is ready to setup windows CI since Qcow2IoBuf is used, and same alloc layout is used for both allocation and deallocation. Signed-off-by: Ming Lei --- .github/workflows/ci.yml | 31 ++++++++++++++++++++++++++++++- 1 file changed, 30 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 1f5c4f3..7d01332 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ env: CARGO_TERM_COLOR: always jobs: - build: + build_linux: name: Rust project - latest runs-on: ubuntu-latest strategy: @@ -23,3 +23,32 @@ jobs: - run: cargo build --verbose - run: cargo test -- --nocapture - run: cargo test -r + + build_win: + name: Rust project - latest + runs-on: windows-latest + strategy: + matrix: + toolchain: + - stable + steps: + - name: Checkout repository + uses: actions/checkout@v4 + - name: Install Chocolatey + run: | + $qemuZipPath = Join-Path $env:USERPROFILE 'qemu.zip' + Invoke-WebRequest -Uri 'https://cloudbase.it/downloads/qemu-img-win-x64-2_3_0.zip' -OutFile $qemuZipPath + Expand-Archive -Path $qemuZipPath -DestinationPath $env:USERPROFILE + - run: | + $env:PATH = "$env:USERPROFILE;$env:PATH" + qemu-img --help + - run: rustup update ${{ matrix.toolchain }} && rustup default ${{ matrix.toolchain }} + - run: cargo build --verbose + - name: Rust test(debug) + run: | + $env:PATH = "$env:USERPROFILE;$env:PATH" + cargo test -- --nocapture + - name: Rust test(release) + run: | + $env:PATH = "$env:USERPROFILE;$env:PATH" + cargo test -r