diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 04b3bb8..eaf7f0b 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -76,7 +76,7 @@ jobs: if: ${{ matrix.os == 'windows-latest' }} uses: samypr100/setup-dev-drive@v3 with: - drive-size: 1GB + drive-size: 16GB drive-format: ReFS drive-type: Dynamic drive-path: "${{ runner.temp }}/dev-drives/refs2.vhdx" @@ -94,11 +94,15 @@ jobs: - name: Test if: "! matrix.use-cross" - run: cargo test --target ${{ matrix.target }} -- --ignored + run: cargo test --target ${{ matrix.target }} -- --include-ignored --show-output + env: + RUST_BACKTRACE: 1 - name: Test using cross if: "matrix.use-cross" - run: cross test --target ${{ matrix.target }} -- --ignored + run: cross test --target ${{ matrix.target }} -- --include-ignored --show-output + env: + RUST_BACKTRACE: 1 cross-check: strategy: diff --git a/examples/reflink_block.rs b/examples/reflink_block.rs new file mode 100644 index 0000000..e6c5669 --- /dev/null +++ b/examples/reflink_block.rs @@ -0,0 +1,35 @@ +use std::fs::File; +use std::num::NonZeroU64; + +// cargo run --example reflink_block V:/file.bin V:/file_cow.bin 4096 + +fn main() -> std::io::Result<()> { + let args: Vec<_> = std::env::args().collect(); + + let [_, src_file, tgt_file, cluster_size] = &args[..] else { + eprintln!( + "Usage: {} ", + args[0] + ); + return Ok(()); + }; + let cluster_size: NonZeroU64 = cluster_size.parse().expect("cannot parse cluster size"); + + let from_file = File::open(src_file)?; + let len = from_file.metadata()?.len(); + let to_file = File::create(tgt_file)?; + to_file.set_len(len)?; + + let mut offset = 0u64; + while offset < len as u64 { + println!("reflink {offset}, {cluster_size}"); + reflink_copy::ReflinkBlockBuilder::new(&from_file, &to_file, cluster_size) + .from_offset(offset) + .to_offset(offset) + .cluster_size(cluster_size) + .reflink_block()?; + + offset += cluster_size.get(); + } + Ok(()) +} diff --git a/src/lib.rs b/src/lib.rs index ffc1544..ef9fed9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -14,6 +14,7 @@ //! //! As soon as other OSes support the functionality, support will be added. +mod reflink_block; mod sys; use std::fs; @@ -187,3 +188,5 @@ pub enum ReflinkSupport { /// Reflink support is unconfirmed. Unknown, } + +pub use reflink_block::ReflinkBlockBuilder; diff --git a/src/reflink_block.rs b/src/reflink_block.rs new file mode 100644 index 0000000..d0faaa5 --- /dev/null +++ b/src/reflink_block.rs @@ -0,0 +1,134 @@ +use crate::sys; +use std::fs::File; +use std::io; +use std::num::NonZeroU64; + +/// Creates a reflink of a specified block from one file to another. +/// +/// This functionality is designed to be highly performant and does not perform any extra API calls. +/// It is expected that the user takes care of necessary preliminary checks and preparations. +/// +/// If you need to clone an entire file, consider using the [`reflink`] or [`reflink_or_copy`] +/// functions instead. +/// +/// > Note: Currently the function works only for windows. It returns `Err` for any other platform. +/// +/// # General restrictions +/// - The source and destination regions must begin and end at a cluster boundary. +/// - If the source and destination regions are in the same file, they must not overlap. (The +/// application may able to proceed by splitting up the block clone operation into multiple block +/// clones that no longer overlap.) +/// +/// # Windows specific restrictions and remarks +/// - The destination region must not extend past the end of file. If the application wishes to +/// extend the destination with cloned data, it must first call +/// [`File::set_len`](fn@std::fs::File::set_len). +/// - The source and destination files must be on the same ReFS volume. +/// - The source and destination files must have the same Integrity Streams setting (that is, +/// Integrity Streams must be enabled in both files, or disabled in both files). +/// - If the source file is sparse, the destination file must also be sparse. +/// - The block clone operation will break Shared Opportunistic Locks (also known as Level 2 +/// Opportunistic Locks). +/// - The ReFS volume must have been formatted with Windows Server 2016, and if Windows Failover +/// Clustering is in use, the Clustering Functional Level must have been Windows Server 2016 or +/// later at format time. +/// +/// > Note: In order to handle blocks larger than 4GB, +/// [`ReflinkBlockBuilder::reflink_block`] splits these big blocks into smaller ones. +/// Each smaller block is 4GB minus the cluster size. This means there might be more than one API +/// call needed for the larger blocks. +/// +/// More information about block cloning on Windows can be found by the +/// [link](https://learn.microsoft.com/en-us/windows/win32/fileio/block-cloning). +/// +/// # Examples +/// +/// The example below demonstrates how to create a new file reusing blocks from another file. +/// ```no_run +/// use std::fs::File; +/// use std::num::NonZeroU64; +/// +/// fn shuffle() -> std::io::Result<()> { +/// let from_file = File::open("source.bin")?; +/// let to_file = File::create("destination.bin")?; +/// let cluster_size = NonZeroU64::new(4096).unwrap(); +/// let len = cluster_size.get() * 2; +/// +/// to_file.set_len(len)?; +/// +/// reflink_copy::ReflinkBlockBuilder::new(&from_file, &to_file, cluster_size) +/// .from_offset(0) +/// .to_offset(cluster_size.get()) +/// .reflink_block()?; +/// +/// reflink_copy::ReflinkBlockBuilder::new(&from_file, &to_file, cluster_size) +/// .from_offset(cluster_size.get()) +/// .to_offset(0) +/// .reflink_block()?; +/// +/// Ok(()) +/// } +/// ``` +/// [`reflink`]: crate::reflink +/// [`reflink_or_copy`]: crate::reflink_or_copy +#[derive(Debug)] +pub struct ReflinkBlockBuilder<'from, 'to> { + from: &'from File, + from_offset: u64, + to: &'to File, + to_offset: u64, + src_length: u64, + cluster_size: Option, +} + +impl<'from, 'to> ReflinkBlockBuilder<'from, 'to> { + /// Creates a new instance of [`ReflinkBlockBuilder`]. + pub fn new(from: &'from File, to: &'to File, src_length: NonZeroU64) -> Self { + Self { + from, + from_offset: 0, + to, + to_offset: 0, + src_length: src_length.get(), + cluster_size: None, + } + } + + /// Sets the offset within the source file. + #[must_use] + pub fn from_offset(mut self, from_offset: u64) -> Self { + self.from_offset = from_offset; + self + } + + /// Sets the offset within the destination file. + #[must_use] + pub fn to_offset(mut self, to_offset: u64) -> Self { + self.to_offset = to_offset; + self + } + + /// Sets the cluster size. It is used to calculate the max block size of a single reflink call + /// on Windows. + #[must_use] + pub fn cluster_size(mut self, cluster_size: NonZeroU64) -> Self { + self.cluster_size = Some(cluster_size); + self + } + + /// Performs reflink operation for the specified block of data. + #[cfg_attr(not(windows), allow(unused_variables))] + pub fn reflink_block(self) -> io::Result<()> { + #[cfg(windows)] + return sys::reflink_block( + self.from, + self.from_offset, + self.to, + self.to_offset, + self.src_length, + self.cluster_size, + ); + #[cfg(not(windows))] + Err(io::Error::other("Not implemented")) + } +} diff --git a/src/sys/mod.rs b/src/sys/mod.rs index 527e4e5..c21abff 100644 --- a/src/sys/mod.rs +++ b/src/sys/mod.rs @@ -12,6 +12,7 @@ cfg_if! { mod windows_impl; pub use self::windows_impl::reflink; pub use self::windows_impl::check_reflink_support; + pub(crate) use self::windows_impl::reflink_block; } else { pub use self::reflink_not_supported as reflink; } diff --git a/src/sys/windows_impl.rs b/src/sys/windows_impl.rs index ab97717..fc48186 100644 --- a/src/sys/windows_impl.rs +++ b/src/sys/windows_impl.rs @@ -1,5 +1,6 @@ use super::utility::AutoRemovedFile; use crate::ReflinkSupport; +use std::num::NonZeroU64; use std::{ convert::TryInto, @@ -90,44 +91,19 @@ pub fn reflink(from: &Path, to: &Path) -> io::Result<()> { } }; - let mut bytes_copied = 0; - // Must be smaller than 4GB; This is always a multiple of ClusterSize - let max_copy_len: i64 = if cluster_size == 0 { - total_copy_len + let cluster_size = if cluster_size != 0 { + Some(NonZeroU64::new(cluster_size as u64).unwrap()) } else { - (4 * 1024 * 1024 * 1024) - cluster_size + None }; - while bytes_copied < total_copy_len { - let bytes_to_copy = total_copy_len.min(max_copy_len); - if cluster_size != 0 { - debug_assert_eq!(bytes_to_copy % cluster_size, 0); - debug_assert_eq!(bytes_copied % cluster_size, 0); - } - - let mut dup_extent = DUPLICATE_EXTENTS_DATA { - FileHandle: src.as_handle(), - - SourceFileOffset: bytes_copied, - TargetFileOffset: bytes_copied, - ByteCount: bytes_to_copy, - }; - - let mut bytes_returned = 0u32; - unsafe { - DeviceIoControl( - dest.as_handle(), - FSCTL_DUPLICATE_EXTENTS_TO_FILE, - Some(&mut dup_extent as *mut _ as *mut c_void), - mem::size_of::().try_into().unwrap(), - None, - 0, - Some(&mut bytes_returned as *mut _), - None, - ) - }?; - bytes_copied += bytes_to_copy; - } - + reflink_block( + &src, + 0, + dest.as_inner_file(), + 0, + total_copy_len as u64, + cluster_size, + )?; if !src_is_sparse { dest.unset_sparse()?; } @@ -378,10 +354,103 @@ fn get_volume_flags(volume_path_w: &[u16]) -> io::Result { Ok(file_system_flags) } +pub(crate) fn reflink_block( + from: &File, + from_offset: u64, + to: &File, + to_offset: u64, + src_length: u64, + cluster_size: Option, +) -> io::Result<()> { + const GB: u64 = 1024u64 * 1024 * 1024; + const MAX_REFS_CLUSTER_SIZE: u64 = 64 * 1024; + + // Must be smaller than 4GB; This is always a multiple of ClusterSize + let max_io_size = 4u64 * GB + - cluster_size + .map(NonZeroU64::get) + .unwrap_or(MAX_REFS_CLUSTER_SIZE); + + let mut bytes_copied = 0; + while bytes_copied < src_length { + let bytes_to_copy = max_io_size.min(src_length - bytes_copied); + if let Some(cluster_size) = cluster_size { + debug_assert_eq!(bytes_to_copy % cluster_size, 0); + debug_assert_eq!(bytes_copied % cluster_size, 0); + } + + duplicate_extent_to_file( + from, + from_offset + bytes_copied, + to, + to_offset + bytes_copied, + bytes_to_copy, + )?; + + bytes_copied += bytes_to_copy; + } + + Ok(()) +} + +fn duplicate_extent_to_file( + from: &File, + from_offset: u64, + to: &File, + to_offset: u64, + src_length: u64, +) -> io::Result<()> { + let mut dup_extent = DUPLICATE_EXTENTS_DATA { + FileHandle: from.as_handle(), + SourceFileOffset: from_offset as i64, + TargetFileOffset: to_offset as i64, + ByteCount: src_length as i64, + }; + + let mut bytes_returned = 0u32; + unsafe { + DeviceIoControl( + to.as_handle(), + FSCTL_DUPLICATE_EXTENTS_TO_FILE, + Some(&mut dup_extent as *mut _ as *mut c_void), + size_of::().try_into().unwrap(), + None, + 0, + Some(&mut bytes_returned as *mut _), + None, + ) + }?; + Ok(()) +} + #[cfg(test)] mod test { use super::*; + #[test] + fn test_round_up() { + assert_eq!(round_up(0, 2), 0); + assert_eq!(round_up(1, 2), 2); + assert_eq!(round_up(2, 2), 2); + + assert_eq!(round_up(15, 8), 16); + assert_eq!(round_up(17, 8), 24); + + assert_eq!(round_up(100000, 4096), 102400); + assert_eq!(round_up(100000, 65536), 131072); + } + + #[test] + #[should_panic] + fn test_invalid_multiple_zero() { + round_up(10, 0); + } + #[test] + #[should_panic] + fn test_invalid_multiple_non_power_of_two() { + round_up(10, 3); + } + #[test] fn test_get_volume_path_is_same() -> io::Result<()> { let src_volume_path = get_volume_path("./src")?; diff --git a/tests/reflink_win.rs b/tests/reflink_win.rs index c47b006..613f76a 100644 --- a/tests/reflink_win.rs +++ b/tests/reflink_win.rs @@ -1,11 +1,16 @@ #![cfg(windows)] -use reflink_copy::{check_reflink_support, reflink, reflink_or_copy, ReflinkSupport}; -use std::io::Write; +use reflink_copy::{ + check_reflink_support, reflink, reflink_or_copy, ReflinkBlockBuilder, ReflinkSupport, +}; +use std::fs::File; +use std::io::{Read, Write}; +use std::num::NonZeroU64; use std::path::{Path, PathBuf}; const FILE_SIZE: usize = 256 * 1024; const FILENAME: &str = "test_file.dat"; +const CLUSTER_SIZE: usize = 4 * 1024; // paths are defined in build.yml @@ -33,7 +38,7 @@ fn create_test_file(path: &Path) -> std::io::Result<()> { std::fs::create_dir_all(folder)?; } - let mut file = std::fs::File::create(path)?; + let mut file = File::create(path)?; file.write_all(&vec![0u8; FILE_SIZE])?; Ok(()) } @@ -136,3 +141,234 @@ fn test_reflink_or_copy_on_unsupported_config() -> std::io::Result<()> { assert_eq!(result, Some(FILE_SIZE as u64)); Ok(()) } + +fn compare_files_eq(file1: &Path, file2: &Path) -> std::io::Result<()> { + let mut f1 = File::open(file1)?; + let mut f2 = File::open(file2)?; + let block_size = f1.metadata()?.len().min(1024 * 1024) as usize; + + let mut buffer1 = vec![0; block_size]; + let mut buffer2 = vec![0; block_size]; + + loop { + let bytes_read1 = f1.read(&mut buffer1)?; + let bytes_read2 = f2.read(&mut buffer2)?; + assert_eq!(bytes_read1, bytes_read2); + + if bytes_read1 == 0 { + break; + } + assert_eq!(&buffer1[..bytes_read1], &buffer2[..bytes_read1]); + } + + Ok(()) +} + +#[test] +#[ignore] +fn test_reflink_block_whole_file() -> std::io::Result<()> { + let num_clusters = 3; + let data_size = CLUSTER_SIZE * num_clusters; + + let from = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + let to = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + + let mut source_file = File::create_new(&from)?; + + let data: Vec = (1..=num_clusters) + .flat_map(|i| vec![i as u8; CLUSTER_SIZE]) + .collect(); + source_file.write_all(&data)?; + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size as u64); + + let mut dest_file = File::create_new(&to)?; + + dest_file.set_len(data_size as u64)?; + ReflinkBlockBuilder::new( + &source_file, + &dest_file, + NonZeroU64::new(data_size as u64).unwrap(), + ) + .reflink_block()?; + + dest_file.flush()?; + drop(source_file); + drop(dest_file); + + compare_files_eq(&from, &to)?; + Ok(()) +} + +#[test] +#[ignore] +fn test_reflink_block_6gb() -> std::io::Result<()> { + let data_size = 6u64 * 1024 * 1024 * 1024; + let from = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + let to = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + + let mut source_file = File::create_new(&from)?; + source_file.set_len(data_size as u64)?; + // to make test faster, we don't write anything to the file + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size as u64); + + let mut dest_file = File::create_new(&to)?; + + dest_file.set_len(data_size as u64)?; + ReflinkBlockBuilder::new( + &source_file, + &dest_file, + NonZeroU64::new(data_size as u64).unwrap(), + ) + .reflink_block()?; + + dest_file.flush()?; + drop(source_file); + drop(dest_file); + + compare_files_eq(&from, &to)?; + Ok(()) +} + +#[test] +#[ignore] +fn test_reflink_unaligned_file() -> std::io::Result<()> { + let num_clusters = 3; + let data_size = (CLUSTER_SIZE * num_clusters + 1) as u64; + let aligned_data_size = (CLUSTER_SIZE * num_clusters + CLUSTER_SIZE) as u64; + + let from = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + let to = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + + let mut source_file = File::create_new(&from)?; + + let data: Vec = (1..=num_clusters) + .flat_map(|i| vec![i as u8; CLUSTER_SIZE]) + .collect(); + source_file.write_all(&data)?; + source_file.write("+".as_bytes())?; + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size); + + let mut dest_file = File::create_new(&to)?; + dest_file.set_len(data_size)?; + println!( + "reflink {}:0 -> {}:0, block {data_size}", + from.display(), + to.display() + ); + + ReflinkBlockBuilder::new( + &source_file, + &dest_file, + NonZeroU64::new(aligned_data_size as u64).unwrap(), + ) + .reflink_block()?; + + dest_file.flush()?; + drop(source_file); + drop(dest_file); + + compare_files_eq(&from, &to)?; + Ok(()) +} + +#[test] +#[ignore] +fn test_reflink_source_file() -> std::io::Result<()> { + let num_clusters = 3; + let data_size = (CLUSTER_SIZE * num_clusters) as u64; + + let from = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + let mut source_file = File::create_new(&from)?; + + let data: Vec = (1..=num_clusters) + .flat_map(|i| vec![i as u8; CLUSTER_SIZE]) + .collect(); + source_file.write_all(&data)?; + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size); + + source_file.set_len(data_size * 2)?; + println!( + "reflink {}:0 -> {}:{data_size}, block {data_size}", + from.display(), + from.display() + ); + ReflinkBlockBuilder::new( + &source_file, + &source_file, + NonZeroU64::new(data_size as u64).unwrap(), + ) + .to_offset(data_size as u64) + .reflink_block()?; + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size * 2); + drop(source_file); + + let mut file = File::open(from)?; + let mut buffer1 = vec![0u8; data_size as usize]; + let mut buffer2 = vec![0u8; data_size as usize]; + file.read_exact(buffer1.as_mut_slice())?; + file.read_exact(buffer2.as_mut_slice())?; + assert_eq!(buffer1, buffer2); + Ok(()) +} + +#[test] +#[ignore] +fn test_reflink_block_reverse() -> std::io::Result<()> { + let num_clusters = 3; + let data_size = CLUSTER_SIZE * num_clusters; + + let from = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + let to = make_subfolder(&refs2_dir(), line!())?.join(FILENAME); + + let mut source_file = File::create_new(&from)?; + + let data: Vec> = (1..=num_clusters) + .map(|i| vec![i as u8; CLUSTER_SIZE]) + .collect(); + for cluster in &data { + source_file.write_all(&cluster)?; + } + source_file.flush()?; + assert_eq!(source_file.metadata()?.len(), data_size as u64); + + let mut dest_file = File::create_new(&to)?; + + dest_file.set_len(data_size as u64)?; + + for i in 0..num_clusters { + let r = num_clusters - 1 - i; + let from_offset = i * CLUSTER_SIZE; + let to_offset = r * CLUSTER_SIZE; + println!( + "reflink {}:{from_offset} -> {}:{to_offset}, block {CLUSTER_SIZE}", + from.display(), + to.display() + ); + ReflinkBlockBuilder::new( + &source_file, + &dest_file, + NonZeroU64::new(CLUSTER_SIZE as u64).unwrap(), + ) + .from_offset(from_offset as u64) + .to_offset(to_offset as u64) + .reflink_block()?; + } + dest_file.flush()?; + drop(source_file); + drop(dest_file); + + let mut dest_file = std::fs::OpenOptions::new().read(true).open(&to)?; + + let mut buf = vec![0; CLUSTER_SIZE]; + for i in num_clusters - 1..=0 { + dest_file.read(buf.as_mut_slice())?; + assert_eq!(buf, data[i as usize]); + } + + Ok(()) +}