Auto merge of #122246 - RalfJung:miri, r=RalfJung

Miri subtree update

r? `@ghost`

`@WaffleLapkin` when this lands, setting `MIRI_TEMP` should not be needed any more on the dev desktops.

Cargo.lock

@@ -2485,6 +2485,7 @@ dependencies = [
  "regex",
  "rustc_version",
  "smallvec",
+ "tempfile",
  "ui_test 0.21.2",
 ]
 

src/tools/miri/.github/workflows/ci.yml

@@ -187,7 +187,7 @@ jobs:
         run: RUSTFLAGS="--cap-lints warn" cargo +stable install josh-proxy --git https://github.com/josh-project/josh --tag r22.12.06
       - name: setup bot git name and email
         run: |
-          git config --global user.name 'The Miri Conjob Bot'
+          git config --global user.name 'The Miri Cronjob Bot'
           git config --global user.email 'miri@cron.bot'
       - name: get changes from rustc
         run: ./miri rustc-pull

src/tools/miri/Cargo.lock

@@ -497,6 +497,7 @@ dependencies = [
  "regex",
  "rustc_version",
  "smallvec",
+ "tempfile",
  "ui_test",
 ]
 

src/tools/miri/Cargo.toml

@@ -45,14 +45,15 @@ ui_test = "0.21.1"
 rustc_version = "0.4"
 regex = "1.5.5"
 lazy_static = "1.4.0"
+tempfile = "3"
 
 [package.metadata.rust-analyzer]
 # This crate uses #[feature(rustc_private)].
 # See https://github.com/rust-analyzer/rust-analyzer/pull/7891
 rustc_private = true
 
 [[test]]
-name = "compiletest"
+name = "ui"
 harness = false
 
 [features]

src/tools/miri/README.md

@@ -318,8 +318,8 @@ environment variable. We first document the most relevant and most commonly used
   and `warn-nobacktrace` are the supported actions. The default is to `abort`,
   which halts the machine. Some (but not all) operations also support continuing
   execution with a "permission denied" error being returned to the program.
-  `warn` prints a full backtrace when that happens; `warn-nobacktrace` is less
-  verbose. `hide` hides the warning entirely.
+  `warn` prints a full backtrace each time that happens; `warn-nobacktrace` is less
+  verbose and shown at most once per operation. `hide` hides the warning entirely.
 * `-Zmiri-num-cpus` states the number of available CPUs to be reported by miri. By default, the
   number of available CPUs is `1`. Note that this flag does not affect how miri handles threads in
   any way.
@@ -359,8 +359,6 @@ The remaining flags are for advanced use only, and more likely to change or be r
 Some of these are **unsound**, which means they can lead
 to Miri failing to detect cases of undefined behavior in a program.
 
-* `-Zmiri-disable-abi-check` disables checking [function ABI]. Using this flag
-  is **unsound**. This flag is **deprecated**.
 * `-Zmiri-disable-alignment-check` disables checking pointer alignment, so you
   can focus on other failures, but it means Miri can miss bugs in your program.
   Using this flag is **unsound**.
@@ -465,11 +463,7 @@ Moreover, Miri recognizes some environment variables:
 * `MIRI_LIB_SRC` defines the directory where Miri expects the sources of the
   standard library that it will build and use for interpretation. This directory
   must point to the `library` subdirectory of a `rust-lang/rust` repository
-  checkout. Note that changing files in that directory does not automatically
-  trigger a re-build of the standard library; you have to clear the Miri build
-  cache with `cargo miri clean` or deleting it manually (on Linux, `rm -rf ~/.cache/miri`;
-  on Windows, `rmdir /S "%LOCALAPPDATA%\rust-lang\miri\cache"`;
-  and on macOS, `rm -rf ~/Library/Caches/org.rust-lang.miri`).
+  checkout.
 * `MIRI_SYSROOT` (recognized by `cargo miri` and the Miri driver) indicates the sysroot to use. When
   using `cargo miri`, this skips the automatic setup -- only set this if you do not want to use the
   automatically created sysroot. For directly invoking the Miri driver, this variable (or a

src/tools/miri/ci/build-all-targets.sh

@@ -12,7 +12,7 @@ PLATFORM_SUPPORT_FILE=$(rustc +miri --print sysroot)/share/doc/rust/html/rustc/p
 
 for target in $(python3 ci/scrape-targets.py $PLATFORM_SUPPORT_FILE); do
     # Wipe the cache before every build to minimize disk usage
-    rm -rf ~/.cache/miri
+    cargo +miri miri clean
     if cargo +miri miri setup --target $target 2>&1 | tee failures/$target; then
         # If the build succeeds, delete its output. If we have output, a build failed.
         rm $FAILS_DIR/$target

src/tools/miri/miri-script/src/commands.rs

@@ -510,11 +510,11 @@ impl Command {
         let miri_flags = flagsplit(&miri_flags);
         let toolchain = &e.toolchain;
         let extra_flags = &e.cargo_extra_flags;
-        let edition_flags = (!have_edition).then_some("--edition=2021"); // keep in sync with `compiletest.rs`.`
+        let edition_flags = (!have_edition).then_some("--edition=2021"); // keep in sync with `tests/ui.rs`.`
         if dep {
             cmd!(
                 e.sh,
-                "cargo +{toolchain} --quiet test --test compiletest {extra_flags...} --manifest-path {miri_manifest} -- --miri-run-dep-mode {miri_flags...} {edition_flags...} {flags...}"
+                "cargo +{toolchain} --quiet test {extra_flags...} --manifest-path {miri_manifest} --test ui -- --miri-run-dep-mode {miri_flags...} {edition_flags...} {flags...}"
             ).quiet().run()?;
         } else {
             cmd!(
@@ -526,37 +526,27 @@ impl Command {
     }
 
     fn fmt(flags: Vec<OsString>) -> Result<()> {
+        use itertools::Itertools;
+
         let e = MiriEnv::new()?;
-        let toolchain = &e.toolchain;
         let config_path = path!(e.miri_dir / "rustfmt.toml");
 
-        let mut cmd = cmd!(
-            e.sh,
-            "rustfmt +{toolchain} --edition=2021 --config-path {config_path} --unstable-features --skip-children {flags...}"
-        );
-        eprintln!("$ {cmd} ...");
-
-        // Add all the filenames to the command.
-        // FIXME: `rustfmt` will follow the `mod` statements in these files, so we get a bunch of
-        // duplicate diffs.
-        for item in WalkDir::new(&e.miri_dir).into_iter().filter_entry(|entry| {
-            let name = entry.file_name().to_string_lossy();
-            let ty = entry.file_type();
-            if ty.is_file() {
-                name.ends_with(".rs")
-            } else {
-                // dir or symlink. skip `target` and `.git`.
-                &name != "target" && &name != ".git"
-            }
-        }) {
-            let item = item?;
-            if item.file_type().is_file() {
-                cmd = cmd.arg(item.into_path());
-            }
-        }
+        // Collect each rust file in the miri repo.
+        let files = WalkDir::new(&e.miri_dir)
+            .into_iter()
+            .filter_entry(|entry| {
+                let name = entry.file_name().to_string_lossy();
+                let ty = entry.file_type();
+                if ty.is_file() {
+                    name.ends_with(".rs")
+                } else {
+                    // dir or symlink. skip `target` and `.git`.
+                    &name != "target" && &name != ".git"
+                }
+            })
+            .filter_ok(|item| item.file_type().is_file())
+            .map_ok(|item| item.into_path());
 
-        // We want our own error message, repeating the command is too much.
-        cmd.quiet().run().map_err(|_| anyhow!("`rustfmt` failed"))?;
-        Ok(())
+        e.format_files(files, &e.toolchain[..], &config_path, &flags[..])
     }
 }

src/tools/miri/miri-script/src/util.rs

@@ -1,7 +1,7 @@
 use std::ffi::{OsStr, OsString};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 
-use anyhow::{Context, Result};
+use anyhow::{anyhow, Context, Result};
 use dunce::canonicalize;
 use path_macro::path;
 use xshell::{cmd, Shell};
@@ -145,4 +145,48 @@ impl MiriEnv {
         .run()?;
         Ok(())
     }
+
+    /// Receives an iterator of files.
+    /// Will format each file with the miri rustfmt config.
+    /// Does not recursively format modules.
+    pub fn format_files(
+        &self,
+        files: impl Iterator<Item = Result<PathBuf, walkdir::Error>>,
+        toolchain: &str,
+        config_path: &Path,
+        flags: &[OsString],
+    ) -> anyhow::Result<()> {
+        use itertools::Itertools;
+
+        let mut first = true;
+
+        // Format in batches as not all our files fit into Windows' command argument limit.
+        for batch in &files.chunks(256) {
+            // Build base command.
+            let mut cmd = cmd!(
+                self.sh,
+                "rustfmt +{toolchain} --edition=2021 --config-path {config_path} --unstable-features --skip-children {flags...}"
+            );
+            if first {
+                // Log an abbreviating command, and only once.
+                eprintln!("$ {cmd} ...");
+                first = false;
+            }
+            // Add files.
+            for file in batch {
+                // Make it a relative path so that on platforms with extremely tight argument
+                // limits (like Windows), we become immune to someone cloning the repo
+                // 50 directories deep.
+                let file = file?;
+                let file = file.strip_prefix(&self.miri_dir)?;
+                cmd = cmd.arg(file);
+            }
+
+            // Run rustfmt.
+            // We want our own error message, repeating the command is too much.
+            cmd.quiet().run().map_err(|_| anyhow!("`rustfmt` failed"))?;
+        }
+
+        Ok(())
+    }
 }

src/tools/miri/rust-version

@@ -1 +1 @@
-1a1876c9790f168fb51afa335a7ba3e6fc267d75
+4d4bb491b65c300835442f6cb4f34fc9a5685c26

src/tools/miri/src/bin/miri.rs

@@ -8,6 +8,11 @@
     rustc::untranslatable_diagnostic
 )]
 
+// Some "regular" crates we want to share with rustc
+#[macro_use]
+extern crate tracing;
+
+// The rustc crates we need
 extern crate rustc_data_structures;
 extern crate rustc_driver;
 extern crate rustc_hir;
@@ -16,8 +21,6 @@ extern crate rustc_log;
 extern crate rustc_metadata;
 extern crate rustc_middle;
 extern crate rustc_session;
-#[macro_use]
-extern crate tracing;
 
 use std::env::{self, VarError};
 use std::num::NonZero;
@@ -202,16 +205,12 @@ fn rustc_logger_config() -> rustc_log::LoggerConfig {
             // rustc traced, but you can also do `MIRI_LOG=miri=trace,rustc_const_eval::interpret=debug`.
             if tracing::Level::from_str(&var).is_ok() {
                 cfg.filter = Ok(format!(
-                    "rustc_middle::mir::interpret={var},rustc_const_eval::interpret={var}"
+                    "rustc_middle::mir::interpret={var},rustc_const_eval::interpret={var},miri={var}"
                 ));
             } else {
                 cfg.filter = Ok(var);
             }
         }
-        // Enable verbose entry/exit logging by default if MIRI_LOG is set.
-        if matches!(cfg.verbose_entry_exit, Err(VarError::NotPresent)) {
-            cfg.verbose_entry_exit = Ok(format!("1"));
-        }
     }
 
     cfg
@@ -342,7 +341,8 @@ fn main() {
     // (`install_ice_hook` might change `RUST_BACKTRACE`.)
     let env_snapshot = env::vars_os().collect::<Vec<_>>();
 
-    let args = rustc_driver::args::raw_args(&early_dcx).unwrap_or_else(|_| std::process::exit(rustc_driver::EXIT_FAILURE));
+    let args = rustc_driver::args::raw_args(&early_dcx)
+        .unwrap_or_else(|_| std::process::exit(rustc_driver::EXIT_FAILURE));
 
     // If the environment asks us to actually be rustc, then do that.
     if let Some(crate_kind) = env::var_os("MIRI_BE_RUSTC") {
@@ -408,17 +408,11 @@ fn main() {
             miri_config.check_alignment = miri::AlignmentCheck::None;
         } else if arg == "-Zmiri-symbolic-alignment-check" {
             miri_config.check_alignment = miri::AlignmentCheck::Symbolic;
-        } else if arg == "-Zmiri-check-number-validity" {
-            eprintln!(
-                "WARNING: the flag `-Zmiri-check-number-validity` no longer has any effect \
-                        since it is now enabled by default"
-            );
         } else if arg == "-Zmiri-disable-abi-check" {
             eprintln!(
-                "WARNING: the flag `-Zmiri-disable-abi-check` is deprecated and planned to be removed.\n\
-                If you have a use-case for it, please file an issue."
+                "WARNING: the flag `-Zmiri-disable-abi-check` no longer has any effect; \
+                    ABI checks cannot be disabled any more"
             );
-            miri_config.check_abi = false;
         } else if arg == "-Zmiri-disable-isolation" {
             if matches!(isolation_enabled, Some(true)) {
                 show_error!(
@@ -459,8 +453,6 @@ fn main() {
             miri_config.collect_leak_backtraces = false;
         } else if arg == "-Zmiri-panic-on-unsupported" {
             miri_config.panic_on_unsupported = true;
-        } else if arg == "-Zmiri-tag-raw-pointers" {
-            eprintln!("WARNING: `-Zmiri-tag-raw-pointers` has no effect; it is enabled by default");
         } else if arg == "-Zmiri-strict-provenance" {
             miri_config.provenance_mode = ProvenanceMode::Strict;
         } else if arg == "-Zmiri-permissive-provenance" {
@@ -476,10 +468,6 @@ fn main() {
                 "scalar" => RetagFields::OnlyScalar,
                 _ => show_error!("`-Zmiri-retag-fields` can only be `all`, `none`, or `scalar`"),
             };
-        } else if arg == "-Zmiri-track-raw-pointers" {
-            eprintln!(
-                "WARNING: `-Zmiri-track-raw-pointers` has no effect; it is enabled by default"
-            );
         } else if let Some(param) = arg.strip_prefix("-Zmiri-seed=") {
             if miri_config.seed.is_some() {
                 show_error!("Cannot specify -Zmiri-seed multiple times!");

src/tools/miri/src/concurrency/thread.rs

@@ -445,10 +445,13 @@ impl<'mir, 'tcx: 'mir> ThreadManager<'mir, 'tcx> {
 
     /// Set an active thread and return the id of the thread that was active before.
     fn set_active_thread_id(&mut self, id: ThreadId) -> ThreadId {
-        let active_thread_id = self.active_thread;
-        self.active_thread = id;
-        assert!(self.active_thread.index() < self.threads.len());
-        active_thread_id
+        assert!(id.index() < self.threads.len());
+        info!(
+            "---------- Now executing on thread `{}` (previous: `{}`) ----------------------------------------",
+            self.get_thread_display_name(id),
+            self.get_thread_display_name(self.active_thread)
+        );
+        std::mem::replace(&mut self.active_thread, id)
     }
 
     /// Get the id of the currently active thread.
@@ -735,6 +738,11 @@ impl<'mir, 'tcx: 'mir> ThreadManager<'mir, 'tcx> {
         for (id, thread) in threads {
             debug_assert_ne!(self.active_thread, id);
             if thread.state == ThreadState::Enabled {
+                info!(
+                    "---------- Now executing on thread `{}` (previous: `{}`) ----------------------------------------",
+                    self.get_thread_display_name(id),
+                    self.get_thread_display_name(self.active_thread)
+                );
                 self.active_thread = id;
                 break;
             }
@@ -882,7 +890,7 @@ pub trait EvalContextExt<'mir, 'tcx: 'mir>: crate::MiriInterpCxExt<'mir, 'tcx> {
             instance,
             start_abi,
             &[*func_arg],
-            Some(&ret_place.into()),
+            Some(&ret_place),
             StackPopCleanup::Root { cleanup: true },
         )?;
 

src/tools/miri/src/eval.rs

@@ -94,8 +94,6 @@ pub struct MiriConfig {
     pub unique_is_unique: bool,
     /// Controls alignment checking.
     pub check_alignment: AlignmentCheck,
-    /// Controls function [ABI](Abi) checking.
-    pub check_abi: bool,
     /// Action for an op requiring communication with the host.
     pub isolated_op: IsolatedOp,
     /// Determines if memory leaks should be ignored.
@@ -162,7 +160,6 @@ impl Default for MiriConfig {
             borrow_tracker: Some(BorrowTrackerMethod::StackedBorrows),
             unique_is_unique: false,
             check_alignment: AlignmentCheck::Int,
-            check_abi: true,
             isolated_op: IsolatedOp::Reject(RejectOpWith::Abort),
             ignore_leaks: false,
             forwarded_env_vars: vec![],
@@ -394,7 +391,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
                     argv,
                     Scalar::from_u8(sigpipe).into(),
                 ],
-                Some(&ret_place.into()),
+                Some(&ret_place),
                 StackPopCleanup::Root { cleanup: true },
             )?;
         }
@@ -403,7 +400,7 @@ pub fn create_ecx<'mir, 'tcx: 'mir>(
                 entry_instance,
                 Abi::Rust,
                 &[argc.into(), argv],
-                Some(&ret_place.into()),
+                Some(&ret_place),
                 StackPopCleanup::Root { cleanup: true },
             )?;
         }