Automatically convert paths to verbatim

ChrisDenton · ChrisDenton · commit 3e2d60624162 · 2021-10-03T19:49:26.000+01:00
This allows using longer paths for filesystem operations without the user needing to `canonicalize` or manually prefix paths.

If the path is already verbatim than this has no effect.
diff --git a/library/std/src/sys/windows/c.rs b/library/std/src/sys/windows/c.rs
@@ -990,6 +990,12 @@ extern "system" {
         cchCount2: c_int,
         bIgnoreCase: BOOL,
     ) -> c_int;
+    pub fn GetFullPathNameW(
+        lpFileName: LPCWSTR,
+        nBufferLength: DWORD,
+        lpBuffer: LPWSTR,
+        lpFilePart: *mut LPWSTR,
+    ) -> DWORD;
 }
 
 #[link(name = "ws2_32")]
diff --git a/library/std/src/sys/windows/fs.rs b/library/std/src/sys/windows/fs.rs
@@ -14,6 +14,7 @@ use crate::sys::time::SystemTime;
 use crate::sys::{c, cvt};
 use crate::sys_common::{AsInner, FromInner, IntoInner};
 
+use super::path::maybe_verbatim;
 use super::to_u16s;
 
 pub struct File {
@@ -281,7 +282,7 @@ impl OpenOptions {
 
 impl File {
     pub fn open(path: &Path, opts: &OpenOptions) -> io::Result<File> {
-        let path = to_u16s(path)?;
+        let path = maybe_verbatim(path)?;
         let handle = unsafe {
             c::CreateFileW(
                 path.as_ptr(),
@@ -706,7 +707,7 @@ impl DirBuilder {
     }
 
     pub fn mkdir(&self, p: &Path) -> io::Result<()> {
-        let p = to_u16s(p)?;
+        let p = maybe_verbatim(p)?;
         cvt(unsafe { c::CreateDirectoryW(p.as_ptr(), ptr::null_mut()) })?;
         Ok(())
     }
@@ -715,7 +716,7 @@ impl DirBuilder {
 pub fn readdir(p: &Path) -> io::Result<ReadDir> {
     let root = p.to_path_buf();
     let star = p.join("*");
-    let path = to_u16s(&star)?;
+    let path = maybe_verbatim(&star)?;
 
     unsafe {
         let mut wfd = mem::zeroed();
@@ -733,20 +734,20 @@ pub fn readdir(p: &Path) -> io::Result<ReadDir> {
 }
 
 pub fn unlink(p: &Path) -> io::Result<()> {
-    let p_u16s = to_u16s(p)?;
+    let p_u16s = maybe_verbatim(p)?;
     cvt(unsafe { c::DeleteFileW(p_u16s.as_ptr()) })?;
     Ok(())
 }
 
 pub fn rename(old: &Path, new: &Path) -> io::Result<()> {
-    let old = to_u16s(old)?;
-    let new = to_u16s(new)?;
+    let old = maybe_verbatim(old)?;
+    let new = maybe_verbatim(new)?;
     cvt(unsafe { c::MoveFileExW(old.as_ptr(), new.as_ptr(), c::MOVEFILE_REPLACE_EXISTING) })?;
     Ok(())
 }
 
 pub fn rmdir(p: &Path) -> io::Result<()> {
-    let p = to_u16s(p)?;
+    let p = maybe_verbatim(p)?;
     cvt(unsafe { c::RemoveDirectoryW(p.as_ptr()) })?;
     Ok(())
 }
@@ -794,7 +795,7 @@ pub fn symlink(original: &Path, link: &Path) -> io::Result<()> {
 
 pub fn symlink_inner(original: &Path, link: &Path, dir: bool) -> io::Result<()> {
     let original = to_u16s(original)?;
-    let link = to_u16s(link)?;
+    let link = maybe_verbatim(link)?;
     let flags = if dir { c::SYMBOLIC_LINK_FLAG_DIRECTORY } else { 0 };
     // Formerly, symlink creation required the SeCreateSymbolicLink privilege. For the Windows 10
     // Creators Update, Microsoft loosened this to allow unprivileged symlink creation if the
@@ -823,8 +824,8 @@ pub fn symlink_inner(original: &Path, link: &Path, dir: bool) -> io::Result<()>
 
 #[cfg(not(target_vendor = "uwp"))]
 pub fn link(original: &Path, link: &Path) -> io::Result<()> {
-    let original = to_u16s(original)?;
-    let link = to_u16s(link)?;
+    let original = maybe_verbatim(original)?;
+    let link = maybe_verbatim(link)?;
     cvt(unsafe { c::CreateHardLinkW(link.as_ptr(), original.as_ptr(), ptr::null_mut()) })?;
     Ok(())
 }
@@ -857,7 +858,7 @@ pub fn lstat(path: &Path) -> io::Result<FileAttr> {
 }
 
 pub fn set_perm(p: &Path, perm: FilePermissions) -> io::Result<()> {
-    let p = to_u16s(p)?;
+    let p = maybe_verbatim(p)?;
     unsafe {
         cvt(c::SetFileAttributesW(p.as_ptr(), perm.attrs))?;
         Ok(())
@@ -900,8 +901,8 @@ pub fn copy(from: &Path, to: &Path) -> io::Result<u64> {
         }
         c::PROGRESS_CONTINUE
     }
-    let pfrom = to_u16s(from)?;
-    let pto = to_u16s(to)?;
+    let pfrom = maybe_verbatim(from)?;
+    let pto = maybe_verbatim(to)?;
     let mut size = 0i64;
     cvt(unsafe {
         c::CopyFileExW(
diff --git a/library/std/src/sys/windows/path.rs b/library/std/src/sys/windows/path.rs
@@ -1,6 +1,10 @@
+use super::{c, fill_utf16_buf, to_u16s};
 use crate::ffi::OsStr;
+use crate::io;
 use crate::mem;
+use crate::path::Path;
 use crate::path::Prefix;
+use crate::ptr;
 
 #[cfg(test)]
 mod tests;
@@ -141,3 +145,96 @@ fn parse_next_component(path: &OsStr, verbatim: bool) -> (&OsStr, &OsStr) {
         None => (path, OsStr::new("")),
     }
 }
+
+/// Returns a UTF-16 encoded path capable of bypassing the legacy `MAX_PATH` limits.
+///
+/// This path may or may not have a verbatim prefix.
+pub(crate) fn maybe_verbatim(path: &Path) -> io::Result<Vec<u16>> {
+    const LEGACY_MAX_PATH: usize = 260;
+    // UTF-16 encoded code points, used in parsing and building UTF-16 paths.
+    // All of these are in the ASCII range so they can be cast directly to `u16`.
+    const SEP: u16 = b'\\' as _;
+    const ALT_SEP: u16 = b'/' as _;
+    const QUERY: u16 = b'?' as _;
+    const COLON: u16 = b':' as _;
+    const DOT: u16 = b'.' as _;
+    const U: u16 = b'U' as _;
+    const N: u16 = b'N' as _;
+    const C: u16 = b'C' as _;
+
+    // \\?\
+    const VERBATIM_PREFIX: &[u16] = &[SEP, SEP, QUERY, SEP];
+    // \??\
+    const NT_PREFIX: &[u16] = &[SEP, QUERY, QUERY, SEP];
+    // \\?\UNC\
+    const UNC_PREFIX: &[u16] = &[SEP, SEP, QUERY, SEP, U, N, C, SEP];
+
+    let mut path = to_u16s(path)?;
+    if path.starts_with(VERBATIM_PREFIX) || path.starts_with(NT_PREFIX) {
+        // Early return for paths that are already verbatim.
+        return Ok(path);
+    } else if path.len() < LEGACY_MAX_PATH {
+        // Early return if an absolute path is less < 260 UTF-16 code units.
+        // This is an optimization to avoid calling `GetFullPathNameW` unnecessarily.
+        match path.as_slice() {
+            // Starts with `D:`, `D:\`, `D:/`, etc.
+            // Does not match if the path starts with a `\` or `/`.
+            [drive, COLON, 0] | [drive, COLON, SEP | ALT_SEP, ..]
+                if *drive != SEP && *drive != ALT_SEP =>
+            {
+                return Ok(path);
+            }
+            // Starts with `\\`, `//`, etc
+            [SEP | ALT_SEP, SEP | ALT_SEP, ..] => return Ok(path),
+            _ => {}
+        }
+    }
+
+    // Firstly, get the absolute path using `GetFullPathNameW`.
+    // https://docs.microsoft.com/en-us/windows/win32/api/fileapi/nf-fileapi-getfullpathnamew
+    let lpfilename = path.as_ptr();
+    fill_utf16_buf(
+        // SAFETY: `fill_utf16_buf` ensures the `buffer` and `size` are valid.
+        // `lpfilename` is a pointer to a null terminated string that is not
+        // invalidated until after `GetFullPathNameW` returns successfully.
+        |buffer, size| unsafe {
+            // While the docs for `GetFullPathNameW` have the standard note
+            // about needing a `\\?\` path for a long lpfilename, this does not
+            // appear to be true in practice.
+            // See:
+            // https://stackoverflow.com/questions/38036943/getfullpathnamew-and-long-windows-file-paths
+            // https://googleprojectzero.blogspot.com/2016/02/the-definitive-guide-on-win32-to-nt.html
+            c::GetFullPathNameW(lpfilename, size, buffer, ptr::null_mut())
+        },
+        |mut absolute| {
+            path.clear();
+
+            // Secondly, add the verbatim prefix. This is easier here because we know the
+            // path is now absolute and fully normalized (e.g. `/` has been changed to `\`).
+            let prefix = match absolute {
+                // C:\ => \\?\C:\
+                [_, COLON, SEP, ..] => VERBATIM_PREFIX,
+                // \\.\ => \\?\
+                [SEP, SEP, DOT, SEP, ..] => {
+                    absolute = &absolute[4..];
+                    VERBATIM_PREFIX
+                }
+                // Leave \\?\ and \??\ as-is.
+                [SEP, SEP, QUERY, SEP, ..] | [SEP, QUERY, QUERY, SEP, ..] => &[],
+                // \\ => \\?\UNC\
+                [SEP, SEP, ..] => {
+                    absolute = &absolute[2..];
+                    UNC_PREFIX
+                }
+                // Anything else we leave alone.
+                _ => &[],
+            };
+
+            path.reserve_exact(prefix.len() + absolute.len() + 1);
+            path.extend_from_slice(prefix);
+            path.extend_from_slice(absolute);
+            path.push(0);
+        },
+    )?;
+    Ok(path)
+}
diff --git a/library/std/src/sys/windows/path/tests.rs b/library/std/src/sys/windows/path/tests.rs
@@ -42,3 +42,49 @@ fn test_parse_next_component() {
         (OsStr::new(r"server"), OsStr::new(r"\\\\\\\\\\\\\share"))
     );
 }
+
+#[test]
+fn verbatim() {
+    use crate::path::Path;
+    fn check(path: &str, expected: &str) {
+        let verbatim = maybe_verbatim(Path::new(path)).unwrap();
+        let verbatim = String::from_utf16_lossy(verbatim.strip_suffix(&[0]).unwrap());
+        assert_eq!(&verbatim, expected, "{}", path);
+    }
+
+    // Ensure long paths are correctly prefixed.
+    check(
+        r"C:\Program Files\Rust\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+        r"\\?\C:\Program Files\Rust\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+    );
+    check(
+        r"\\server\share\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+        r"\\?\UNC\server\share\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+    );
+    check(
+        r"\\.\PIPE\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+        r"\\?\PIPE\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+    );
+    // `\\?\` prefixed paths are left unchanged...
+    check(
+        r"\\?\verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+        r"\\?\verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+    );
+    // But `//?/` is not a verbatim prefix so it will be normalized.
+    check(
+        r"//?/E:/verbatim.\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+        r"\\?\E:\verbatim\aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa\bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb\foo.txt",
+    );
+
+    // For performance, short absolute paths are left unchanged.
+    check(r"C:\Program Files\Rust", r"C:\Program Files\Rust");
+    check(r"\\server\share", r"\\server\share");
+    check(r"\\.\COM1", r"\\.\COM1");
+
+    // Make sure opening a drive will work.
+    check("Z:", "Z:");
+
+    // An empty path or a path that contains null are not valid paths.
+    assert!(maybe_verbatim(Path::new("")).is_err());
+    assert!(maybe_verbatim(Path::new("\0")).is_err());
+}
