Add IsTerminal trait to determine if a descriptor or handle is a terminal

The UNIX and WASI implementations use `isatty`. The Windows
implementation uses the same logic the `atty` crate uses, including the
hack needed to detect msys terminals.

Implement this trait for `File` and for `Stdin`/`Stdout`/`Stderr` and
their locked counterparts on all platforms. On UNIX and WASI, implement
it for `BorrowedFd`/`OwnedFd`. On Windows, implement it for
`BorrowedHandle`/`OwnedHandle`.

Based on rust-lang/rust#91121

Co-authored-by: Matt Wilkinson <mattwilki17@gmail.com>

Author: joshtriplett

std/src/io/mod.rs

@@ -266,6 +266,8 @@ pub(crate) use self::stdio::attempt_print_to_stderr;
 #[unstable(feature = "internal_output_capture", issue = "none")]
 #[doc(no_inline, hidden)]
 pub use self::stdio::set_output_capture;
+#[unstable(feature = "is_terminal", issue = "98070")]
+pub use self::stdio::IsTerminal;
 #[unstable(feature = "print_internals", issue = "none")]
 pub use self::stdio::{_eprint, _print};
 #[stable(feature = "rust1", since = "1.0.0")]

std/src/io/stdio.rs

@@ -7,6 +7,7 @@ use crate::io::prelude::*;
 
 use crate::cell::{Cell, RefCell};
 use crate::fmt;
+use crate::fs::File;
 use crate::io::{self, BufReader, IoSlice, IoSliceMut, LineWriter, Lines};
 use crate::sync::atomic::{AtomicBool, Ordering};
 use crate::sync::{Arc, Mutex, MutexGuard, OnceLock};
@@ -1035,6 +1036,34 @@ pub(crate) fn attempt_print_to_stderr(args: fmt::Arguments<'_>) {
     let _ = stderr().write_fmt(args);
 }
 
+/// Trait to determine if a descriptor/handle refers to a terminal/tty.
+#[unstable(feature = "is_terminal", issue = "98070")]
+pub trait IsTerminal: crate::sealed::Sealed {
+    /// Returns `true` if the descriptor/handle refers to a terminal/tty.
+    ///
+    /// On platforms where Rust does not know how to detect a terminal yet, this will return
+    /// `false`. This will also return `false` if an unexpected error occurred, such as from
+    /// passing an invalid file descriptor.
+    fn is_terminal(&self) -> bool;
+}
+
+macro_rules! impl_is_terminal {
+    ($($t:ty),*$(,)?) => {$(
+        #[unstable(feature = "sealed", issue = "none")]
+        impl crate::sealed::Sealed for $t {}
+
+        #[unstable(feature = "is_terminal", issue = "98070")]
+        impl IsTerminal for $t {
+            #[inline]
+            fn is_terminal(&self) -> bool {
+                crate::sys::io::is_terminal(self)
+            }
+        }
+    )*}
+}
+
+impl_is_terminal!(File, Stdin, StdinLock<'_>, Stdout, StdoutLock<'_>, Stderr, StderrLock<'_>);
+
 #[unstable(
     feature = "print_internals",
     reason = "implementation detail which may disappear or be replaced at any time",

std/src/lib.rs

@@ -253,6 +253,7 @@
 #![feature(exhaustive_patterns)]
 #![feature(if_let_guard)]
 #![feature(intra_doc_pointers)]
+#![feature(is_terminal)]
 #![feature(lang_items)]
 #![feature(let_chains)]
 #![feature(linkage)]

std/src/os/fd/owned.rs

@@ -193,6 +193,23 @@ impl fmt::Debug for OwnedFd {
     }
 }
 
+macro_rules! impl_is_terminal {
+    ($($t:ty),*$(,)?) => {$(
+        #[unstable(feature = "sealed", issue = "none")]
+        impl crate::sealed::Sealed for $t {}
+
+        #[unstable(feature = "is_terminal", issue = "98070")]
+        impl crate::io::IsTerminal for $t {
+            #[inline]
+            fn is_terminal(&self) -> bool {
+                crate::sys::io::is_terminal(self)
+            }
+        }
+    )*}
+}
+
+impl_is_terminal!(BorrowedFd<'_>, OwnedFd);
+
 /// A trait to borrow the file descriptor from an underlying object.
 ///
 /// This is only available on unix platforms and must be imported in order to

std/src/os/windows/io/handle.rs

@@ -384,6 +384,23 @@ impl fmt::Debug for OwnedHandle {
     }
 }
 
+macro_rules! impl_is_terminal {
+    ($($t:ty),*$(,)?) => {$(
+        #[unstable(feature = "sealed", issue = "none")]
+        impl crate::sealed::Sealed for $t {}
+
+        #[unstable(feature = "is_terminal", issue = "98070")]
+        impl crate::io::IsTerminal for $t {
+            #[inline]
+            fn is_terminal(&self) -> bool {
+                crate::sys::io::is_terminal(self)
+            }
+        }
+    )*}
+}
+
+impl_is_terminal!(BorrowedHandle<'_>, OwnedHandle);
+
 /// A trait to borrow the handle from an underlying object.
 #[stable(feature = "io_safety", since = "1.63.0")]
 pub trait AsHandle {

std/src/sys/unix/io.rs

@@ -1,4 +1,5 @@
 use crate::marker::PhantomData;
+use crate::os::fd::{AsFd, AsRawFd};
 use crate::slice;
 
 use libc::{c_void, iovec};
@@ -74,3 +75,8 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.iov_base as *mut u8, self.vec.iov_len) }
     }
 }
+
+pub fn is_terminal(fd: &impl AsFd) -> bool {
+    let fd = fd.as_fd();
+    unsafe { libc::isatty(fd.as_raw_fd()) != 0 }
+}

std/src/sys/unsupported/io.rs

@@ -45,3 +45,7 @@ impl<'a> IoSliceMut<'a> {
         self.0
     }
 }
+
+pub fn is_terminal<T>(_: &T) -> bool {
+    false
+}

std/src/sys/wasi/io.rs

@@ -1,6 +1,7 @@
 #![deny(unsafe_op_in_unsafe_fn)]
 
 use crate::marker::PhantomData;
+use crate::os::fd::{AsFd, AsRawFd};
 use crate::slice;
 
 #[derive(Copy, Clone)]
@@ -71,3 +72,8 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.buf_len) }
     }
 }
+
+pub fn is_terminal(fd: &impl AsFd) -> bool {
+    let fd = fd.as_fd();
+    unsafe { libc::isatty(fd.as_raw_fd()) != 0 }
+}

std/src/sys/windows/c.rs

@@ -127,6 +127,8 @@ pub const SECURITY_SQOS_PRESENT: DWORD = 0x00100000;
 
 pub const FIONBIO: c_ulong = 0x8004667e;
 
+pub const MAX_PATH: usize = 260;
+
 #[repr(C)]
 #[derive(Copy)]
 pub struct WIN32_FIND_DATAW {
@@ -538,6 +540,12 @@ pub struct SYMBOLIC_LINK_REPARSE_BUFFER {
 
 /// NB: Use carefully! In general using this as a reference is likely to get the
 /// provenance wrong for the `PathBuffer` field!
+#[repr(C)]
+pub struct FILE_NAME_INFO {
+    pub FileNameLength: DWORD,
+    pub FileName: [WCHAR; 1],
+}
+
 #[repr(C)]
 pub struct MOUNT_POINT_REPARSE_BUFFER {
     pub SubstituteNameOffset: c_ushort,

std/src/sys/windows/io.rs

@@ -1,6 +1,10 @@
 use crate::marker::PhantomData;
+use crate::mem::size_of;
+use crate::os::windows::io::{AsHandle, AsRawHandle, BorrowedHandle};
 use crate::slice;
 use crate::sys::c;
+use core;
+use libc;
 
 #[derive(Copy, Clone)]
 #[repr(transparent)]
@@ -78,3 +82,58 @@ impl<'a> IoSliceMut<'a> {
         unsafe { slice::from_raw_parts_mut(self.vec.buf as *mut u8, self.vec.len as usize) }
     }
 }
+
+pub fn is_terminal(h: &impl AsHandle) -> bool {
+    unsafe { handle_is_console(h.as_handle()) }
+}
+
+unsafe fn handle_is_console(handle: BorrowedHandle<'_>) -> bool {
+    let handle = handle.as_raw_handle();
+
+    let mut out = 0;
+    if c::GetConsoleMode(handle, &mut out) != 0 {
+        // False positives aren't possible. If we got a console then we definitely have a console.
+        return true;
+    }
+
+    // At this point, we *could* have a false negative. We can determine that this is a true
+    // negative if we can detect the presence of a console on any of the standard I/O streams. If
+    // another stream has a console, then we know we're in a Windows console and can therefore
+    // trust the negative.
+    for std_handle in [c::STD_INPUT_HANDLE, c::STD_OUTPUT_HANDLE, c::STD_ERROR_HANDLE] {
+        let std_handle = c::GetStdHandle(std_handle);
+        if std_handle != handle && c::GetConsoleMode(std_handle, &mut out) != 0 {
+            return false;
+        }
+    }
+
+    // Otherwise, we fall back to an msys hack to see if we can detect the presence of a pty.
+    msys_tty_on(handle)
+}
+
+unsafe fn msys_tty_on(handle: c::HANDLE) -> bool {
+    let size = size_of::<c::FILE_NAME_INFO>() + c::MAX_PATH * size_of::<c::WCHAR>();
+    let mut name_info_bytes = vec![0u8; size];
+    let res = c::GetFileInformationByHandleEx(
+        handle,
+        c::FileNameInfo,
+        name_info_bytes.as_mut_ptr() as *mut libc::c_void,
+        size as u32,
+    );
+    if res == 0 {
+        return false;
+    }
+    let name_info: &c::FILE_NAME_INFO = &*(name_info_bytes.as_ptr() as *const c::FILE_NAME_INFO);
+    let s = core::slice::from_raw_parts(
+        name_info.FileName.as_ptr(),
+        name_info.FileNameLength as usize / 2,
+    );
+    let name = String::from_utf16_lossy(s);
+    // This checks whether 'pty' exists in the file name, which indicates that
+    // a pseudo-terminal is attached. To mitigate against false positives
+    // (e.g., an actual file name that contains 'pty'), we also require that
+    // either the strings 'msys-' or 'cygwin-' are in the file name as well.)
+    let is_msys = name.contains("msys-") || name.contains("cygwin-");
+    let is_pty = name.contains("-pty");
+    is_msys && is_pty
+}

test/src/cli.rs

@@ -3,9 +3,9 @@
 use std::env;
 use std::path::PathBuf;
 
-use super::helpers::isatty;
 use super::options::{ColorConfig, Options, OutputFormat, RunIgnored};
 use super::time::TestTimeOptions;
+use std::io::{self, IsTerminal};
 
 #[derive(Debug)]
 pub struct TestOpts {
@@ -32,7 +32,7 @@ pub struct TestOpts {
 impl TestOpts {
     pub fn use_color(&self) -> bool {
         match self.color {
-            ColorConfig::AutoColor => !self.nocapture && isatty::stdout_isatty(),
+            ColorConfig::AutoColor => !self.nocapture && io::stdout().is_terminal(),
             ColorConfig::AlwaysColor => true,
             ColorConfig::NeverColor => false,
         }

test/src/helpers/isatty.rs

@@ -3,6 +3,5 @@
 
 pub mod concurrency;
 pub mod exit_code;
-pub mod isatty;
 pub mod metrics;
 pub mod shuffle;

test/src/helpers/mod.rs

@@ -17,6 +17,7 @@
 #![unstable(feature = "test", issue = "50297")]
 #![doc(test(attr(deny(warnings))))]
 #![feature(internal_output_capture)]
+#![feature(is_terminal)]
 #![feature(staged_api)]
 #![feature(process_exitcode_internals)]
 #![feature(test)]