Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Less syscalls for the copy_file_range probe #122079

Merged
merged 3 commits into from
May 26, 2024
Merged

Less syscalls for the copy_file_range probe #122079

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
214c498
Less syscalls for the `copy_file_range` probe
tbu- Mar 6, 2024
5073475
Be stricter with `copy_file_range` probe results
tbu- Mar 6, 2024
7d67ee5
Lift the probe code of `copy_file_range` into a function
tbu- Apr 27, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
67 changes: 48 additions & 19 deletions library/std/src/sys/pal/unix/kernel_copy.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -560,6 +560,12 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
// We store the availability in a global to avoid unnecessary syscalls
static HAS_COPY_FILE_RANGE: AtomicU8 = AtomicU8::new(NOT_PROBED);

let mut have_probed = match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => false,
UNAVAILABLE => return CopyResult::Fallback(0),
_ => true,
};

syscall! {
fn copy_file_range(
fd_in: libc::c_int,
Expand All @@ -571,25 +577,22 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
) -> libc::ssize_t
}

match HAS_COPY_FILE_RANGE.load(Ordering::Relaxed) {
NOT_PROBED => {
// EPERM can indicate seccomp filters or an immutable file.
// To distinguish these cases we probe with invalid file descriptors which should result in EBADF if the syscall is supported
// and some other error (ENOSYS or EPERM) if it's not available
let result = unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
};

if matches!(result.map_err(|e| e.raw_os_error()), Err(Some(EBADF))) {
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
} else {
HAS_COPY_FILE_RANGE.store(UNAVAILABLE, Ordering::Relaxed);
return CopyResult::Fallback(0);
}
fn probe_copy_file_range_support() -> u8 {
// In some cases, we cannot determine availability from the first
// `copy_file_range` call. In this case, we probe with an invalid file
// descriptor so that the results are easily interpretable.
match unsafe {
cvt(copy_file_range(INVALID_FD, ptr::null_mut(), INVALID_FD, ptr::null_mut(), 1, 0))
.map_err(|e| e.raw_os_error())
} {
Err(Some(EPERM | ENOSYS)) => UNAVAILABLE,
Err(Some(EBADF)) => AVAILABLE,
Ok(_) => panic!("unexpected copy_file_range probe success"),
// Treat other errors as the syscall
// being unavailable.
Err(_) => UNAVAILABLE,
}
UNAVAILABLE => return CopyResult::Fallback(0),
_ => {}
};
}

let mut written = 0u64;
while written < max_len {
Expand All @@ -604,6 +607,11 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
cvt(copy_file_range(reader, ptr::null_mut(), writer, ptr::null_mut(), bytes_to_copy, 0))
};

if !have_probed && copy_result.is_ok() {
have_probed = true;
HAS_COPY_FILE_RANGE.store(AVAILABLE, Ordering::Relaxed);
}

match copy_result {
Ok(0) if written == 0 => {
// fallback to work around several kernel bugs where copy_file_range will fail to
Expand All @@ -619,7 +627,28 @@ pub(super) fn copy_regular_files(reader: RawFd, writer: RawFd, max_len: u64) ->
return match err.raw_os_error() {
// when file offset + max_length > u64::MAX
Some(EOVERFLOW) => CopyResult::Fallback(written),
Some(ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF) if written == 0 => {
Some(raw_os_error @ (ENOSYS | EXDEV | EINVAL | EPERM | EOPNOTSUPP | EBADF))
if written == 0 =>
{
if !have_probed {
let available = if matches!(raw_os_error, ENOSYS | EOPNOTSUPP | EPERM) {
// EPERM can indicate seccomp filters or an
// immutable file. To distinguish these
// cases we probe with invalid file
// descriptors which should result in EBADF
// if the syscall is supported and EPERM or
// ENOSYS if it's not available.
//
// For EOPNOTSUPP, see below. In the case of
// ENOSYS, we try to cover for faulty FUSE
// drivers.
probe_copy_file_range_support()
} else {
AVAILABLE
};
HAS_COPY_FILE_RANGE.store(available, Ordering::Relaxed);
}

// Try fallback io::copy if either:
// - Kernel version is < 4.5 (ENOSYS¹)
// - Files are mounted on different fs (EXDEV)
Expand Down
Loading