Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Convert UNIX std::fs::remove_dir_all() from recursive to looping #93473

Closed
wants to merge 3 commits into from
Closed

Convert UNIX std::fs::remove_dir_all() from recursive to looping #93473

Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
61bd4e9
WIP: looping version of unix remove_dir_all
cuviper Jan 19, 2022
7cf62c7
Temporarily remove special-cased Macos x86-64 implementation.
hkratz Jan 30, 2022
d2cfba1
Readd Macos x86-64 remove_dir_all() special case using dynamic symbols.
hkratz Jan 30, 2022
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
256 changes: 108 additions & 148 deletions library/std/src/sys/unix/fs.rs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1474,140 +1474,49 @@ mod remove_dir_impl {
pub use crate::sys_common::fs::remove_dir_all;
}

// Dynamically choose implementation Macos x86-64: modern for 10.10+, fallback for older versions
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
// Modern implementation using openat(), unlinkat() and fdopendir()
#[cfg(not(any(target_os = "redox", target_os = "espidf")))]
mod remove_dir_impl {
use super::{cstr, lstat, Dir, InnerReadDir, ReadDir};
use super::{cstr, lstat, Dir, DirEntry, InnerReadDir, ReadDir};
use crate::ffi::CStr;
use crate::io;
use crate::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
use crate::os::unix::prelude::{OwnedFd, RawFd};
use crate::path::{Path, PathBuf};
use crate::sync::Arc;
use crate::sys::weak::weak;
use crate::sys::{cvt, cvt_r};
use libc::{c_char, c_int, DIR};

pub fn openat_nofollow_dironly(parent_fd: Option<RawFd>, p: &CStr) -> io::Result<OwnedFd> {
weak!(fn openat(c_int, *const c_char, c_int) -> c_int);
let fd = cvt_r(|| unsafe {
openat.get().unwrap()(
parent_fd.unwrap_or(libc::AT_FDCWD),
p.as_ptr(),
libc::O_CLOEXEC | libc::O_RDONLY | libc::O_NOFOLLOW | libc::O_DIRECTORY,
)
})?;
Ok(unsafe { OwnedFd::from_raw_fd(fd) })
}

fn fdreaddir(dir_fd: OwnedFd) -> io::Result<(ReadDir, RawFd)> {
weak!(fn fdopendir(c_int) -> *mut DIR, "fdopendir$INODE64");
let ptr = unsafe { fdopendir.get().unwrap()(dir_fd.as_raw_fd()) };
if ptr.is_null() {
return Err(io::Error::last_os_error());
}
let dirp = Dir(ptr);
// file descriptor is automatically closed by libc::closedir() now, so give up ownership
let new_parent_fd = dir_fd.into_raw_fd();
// a valid root is not needed because we do not call any functions involving the full path
// of the DirEntrys.
let dummy_root = PathBuf::new();
Ok((
ReadDir {
inner: Arc::new(InnerReadDir { dirp, root: dummy_root }),
end_of_stream: false,
},
new_parent_fd,
))
}

fn remove_dir_all_recursive(parent_fd: Option<RawFd>, p: &Path) -> io::Result<()> {
weak!(fn unlinkat(c_int, *const c_char, c_int) -> c_int);

let pcstr = cstr(p)?;
#[cfg(not(all(target_os = "macos", target_arch = "x86_64"),))]
use libc::{fdopendir, openat, unlinkat};

// entry is expected to be a directory, open as such
let fd = openat_nofollow_dironly(parent_fd, &pcstr)?;
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
mod macos_weak {
use crate::sys::weak::weak;
use libc::{c_char, c_int, DIR};

// open the directory passing ownership of the fd
let (dir, fd) = fdreaddir(fd)?;
for child in dir {
let child = child?;
match child.entry.d_type {
libc::DT_DIR => {
remove_dir_all_recursive(Some(fd), Path::new(&child.file_name()))?;
}
libc::DT_UNKNOWN => {
match cvt(unsafe { unlinkat.get().unwrap()(fd, child.name_cstr().as_ptr(), 0) })
{
// type unknown - try to unlink
Err(err) if err.raw_os_error() == Some(libc::EPERM) => {
// if the file is a directory unlink fails with EPERM
remove_dir_all_recursive(Some(fd), Path::new(&child.file_name()))?;
}
result => {
result?;
}
}
}
_ => {
// not a directory -> unlink
cvt(unsafe { unlinkat.get().unwrap()(fd, child.name_cstr().as_ptr(), 0) })?;
}
}
pub unsafe fn openat(dirfd: c_int, pathname: *const c_char, flags: c_int) -> c_int {
weak!(fn openat(c_int, *const c_char, c_int) -> c_int);
openat.get().unwrap()(dirfd, pathname, flags)
}

// unlink the directory after removing its contents
cvt(unsafe {
unlinkat.get().unwrap()(
parent_fd.unwrap_or(libc::AT_FDCWD),
pcstr.as_ptr(),
libc::AT_REMOVEDIR,
)
})?;
Ok(())
}
pub unsafe fn fdopendir(fd: c_int) -> *mut DIR {
weak!(fn fdopendir(c_int) -> *mut DIR, "fdopendir$INODE64");
fdopendir.get().unwrap()(fd)
}

fn remove_dir_all_modern(p: &Path) -> io::Result<()> {
// We cannot just call remove_dir_all_recursive() here because that would not delete a passed
// symlink. No need to worry about races, because remove_dir_all_recursive() does not recurse
// into symlinks.
let attr = lstat(p)?;
if attr.file_type().is_symlink() {
crate::fs::remove_file(p)
} else {
remove_dir_all_recursive(None, p)
pub unsafe fn unlinkat(dirfd: c_int, pathname: *const c_char, flags: c_int) -> c_int {
weak!(fn unlinkat(c_int, *const c_char, c_int) -> c_int);
unlinkat.get().unwrap()(dirfd, pathname, flags)
}
}

pub fn remove_dir_all(p: &Path) -> io::Result<()> {
weak!(fn openat(c_int, *const c_char, c_int) -> c_int);
if openat.get().is_some() {
// openat() is available with macOS 10.10+, just like unlinkat() and fdopendir()
remove_dir_all_modern(p)
} else {
// fall back to classic implementation
crate::sys_common::fs::remove_dir_all(p)
pub fn has_openat() -> bool {
weak!(fn openat(c_int, *const c_char, c_int) -> c_int);
openat.get().is_some()
}
}
}

// Modern implementation using openat(), unlinkat() and fdopendir()
#[cfg(not(any(
all(target_os = "macos", target_arch = "x86_64"),
target_os = "redox",
target_os = "espidf"
)))]
mod remove_dir_impl {
use super::{cstr, lstat, Dir, DirEntry, InnerReadDir, ReadDir};
use crate::ffi::CStr;
use crate::io;
use crate::os::unix::io::{AsRawFd, FromRawFd, IntoRawFd};
use crate::os::unix::prelude::{OwnedFd, RawFd};
use crate::path::{Path, PathBuf};
use crate::sync::Arc;
use crate::sys::{cvt, cvt_r};
use libc::{fdopendir, openat, unlinkat};
#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
use macos_weak::{fdopendir, openat, unlinkat};

pub fn openat_nofollow_dironly(parent_fd: Option<RawFd>, p: &CStr) -> io::Result<OwnedFd> {
let fd = cvt_r(|| unsafe {
Expand Down Expand Up @@ -1672,55 +1581,106 @@ mod remove_dir_impl {
}
}

fn remove_dir_all_recursive(parent_fd: Option<RawFd>, p: &Path) -> io::Result<()> {
let pcstr = cstr(p)?;

// entry is expected to be a directory, open as such
let fd = openat_nofollow_dironly(parent_fd, &pcstr)?;

// open the directory passing ownership of the fd
let (dir, fd) = fdreaddir(fd)?;
for child in dir {
let child = child?;
match is_dir(&child) {
Some(true) => {
remove_dir_all_recursive(Some(fd), Path::new(&child.file_name()))?;
}
Some(false) => {
cvt(unsafe { unlinkat(fd, child.name_cstr().as_ptr(), 0) })?;
}
None => match cvt(unsafe { unlinkat(fd, child.name_cstr().as_ptr(), 0) }) {
fn unlink_direntry(ent: &DirEntry, parent_fd: RawFd) -> io::Result<bool> {
match is_dir(&ent) {
Some(true) => Ok(false),
Some(false) => {
cvt(unsafe { unlinkat(parent_fd, ent.name_cstr().as_ptr(), 0) })?;
Ok(true)
}
None => {
match cvt(unsafe { unlinkat(parent_fd, ent.name_cstr().as_ptr(), 0) }) {
// type unknown - try to unlink
Err(err)
if err.raw_os_error() == Some(libc::EISDIR)
|| err.raw_os_error() == Some(libc::EPERM) =>
{
// if the file is a directory unlink fails with EISDIR on Linux and EPERM everyhwere else
remove_dir_all_recursive(Some(fd), Path::new(&child.file_name()))?;
// if the file is a directory unlink fails with EISDIR on Linux
// and EPERM everyhwere else
Ok(false)
}
result => {
result?;
}
},
result => result.map(|_| true),
}
}
}
}

// unlink the directory after removing its contents
cvt(unsafe {
unlinkat(parent_fd.unwrap_or(libc::AT_FDCWD), pcstr.as_ptr(), libc::AT_REMOVEDIR)
})?;
Ok(())
fn remove_dir_all_loop(p: &Path) -> io::Result<()> {
use crate::ffi::CString;

struct State {
dir: ReadDir,
fd: RawFd,
parent_fd: Option<RawFd>,
pcstr: CString,
}

impl State {
fn new(parent_fd: Option<RawFd>, pcstr: CString) -> io::Result<Self> {
// entry is expected to be a directory, open as such
let fd = openat_nofollow_dironly(parent_fd, &pcstr)?;

// open the directory passing ownership of the fd
let (dir, fd) = fdreaddir(fd)?;

Ok(Self { dir, fd, parent_fd, pcstr })
}
}

let mut parents = Vec::<State>::new();
let mut current = State::new(None, cstr(p)?)?;
loop {
while let Some(child) = current.dir.next() {
let child = child?;
if !unlink_direntry(&child, current.fd)? {
// Descend into this child directory
let parent = current;
current = State::new(Some(parent.fd), child.name_cstr().into())?;
parents.push(parent);
}
}

// unlink the directory after removing its contents
cvt(unsafe {
unlinkat(
current.parent_fd.unwrap_or(libc::AT_FDCWD),
current.pcstr.as_ptr(),
libc::AT_REMOVEDIR,
)
})?;

match parents.pop() {
Some(parent) => current = parent,
None => return Ok(()),
}
}
}

pub fn remove_dir_all(p: &Path) -> io::Result<()> {
// We cannot just call remove_dir_all_recursive() here because that would not delete a passed
// symlink. No need to worry about races, because remove_dir_all_recursive() does not recurse
fn remove_dir_all_modern(p: &Path) -> io::Result<()> {
// We cannot just call remove_dir_all_loop() here because that would not delete a passed
// symlink. No need to worry about races, because remove_dir_all_loop() does not descend
// into symlinks.
let attr = lstat(p)?;
if attr.file_type().is_symlink() {
crate::fs::remove_file(p)
} else {
remove_dir_all_recursive(None, p)
remove_dir_all_loop(p)
}
}

#[cfg(not(all(target_os = "macos", target_arch = "x86_64")))]
pub fn remove_dir_all(p: &Path) -> io::Result<()> {
remove_dir_all_modern(p)
}

#[cfg(all(target_os = "macos", target_arch = "x86_64"))]
pub fn remove_dir_all(p: &Path) -> io::Result<()> {
if macos_weak::has_openat() {
// openat() is available with macOS 10.10+, just like unlinkat() and fdopendir()
remove_dir_all_modern(p)
} else {
// fall back to classic implementation
crate::sys_common::fs::remove_dir_all(p)
}
}
}