Rollup merge of rust-lang#69624 - ehuss:toolstate-beta-regress, r=Mark-Simulacrum

Toolstate: Don't block beta week on already broken tools.

This changes it so that tools are allowed to be broken entering the beta week if they are already broken.  This restores the original behavior before the changes in rust-lang#69332.

Closes rust-lang#68458

Author: Dylan-DPC

Diff for: src/bootstrap/toolstate.rs

@@ -6,7 +6,7 @@ use std::env;
 use std::fmt;
 use std::fs;
 use std::io::{Seek, SeekFrom};
-use std::path::PathBuf;
+use std::path::{Path, PathBuf};
 use std::process::Command;
 use std::time;
 
@@ -24,7 +24,7 @@ const OS: Option<&str> = None;
 
 type ToolstateData = HashMap<Box<str>, ToolState>;
 
-#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq)]
+#[derive(Copy, Clone, Debug, Deserialize, Serialize, PartialEq, Eq, PartialOrd)]
 #[serde(rename_all = "kebab-case")]
 /// Whether a tool can be compiled, tested or neither
 pub enum ToolState {
@@ -143,10 +143,31 @@ pub struct ToolStateCheck;
 impl Step for ToolStateCheck {
     type Output = ();
 
-    /// Runs the `linkchecker` tool as compiled in `stage` by the `host` compiler.
+    /// Checks tool state status.
     ///
-    /// This tool in `src/tools` will verify the validity of all our links in the
-    /// documentation to ensure we don't have a bunch of dead ones.
+    /// This is intended to be used in the `checktools.sh` script. To use
+    /// this, set `save-toolstates` in `config.toml` so that tool status will
+    /// be saved to a JSON file. Then, run `x.py test --no-fail-fast` for all
+    /// of the tools to populate the JSON file. After that is done, this
+    /// command can be run to check for any status failures, and exits with an
+    /// error if there are any.
+    ///
+    /// This also handles publishing the results to the `history` directory of
+    /// the toolstate repo https://github.com/rust-lang-nursery/rust-toolstate
+    /// if the env var `TOOLSTATE_PUBLISH` is set. Note that there is a
+    /// *separate* step of updating the `latest.json` file and creating GitHub
+    /// issues and comments in `src/ci/publish_toolstate.sh`, which is only
+    /// performed on master. (The shell/python code is intended to be migrated
+    /// here eventually.)
+    ///
+    /// The rules for failure are:
+    /// * If the PR modifies a tool, the status must be test-pass.
+    ///   NOTE: There is intent to change this, see
+    ///   https://github.com/rust-lang/rust/issues/65000.
+    /// * All "stable" tools must be test-pass on the stable or beta branches.
+    /// * During beta promotion week, a PR is not allowed to "regress" a
+    ///   stable tool. That is, the status is not allowed to get worse
+    ///   (test-pass to test-fail or build-fail).
     fn run(self, builder: &Builder<'_>) {
         if builder.config.dry_run {
             return;
@@ -171,6 +192,8 @@ impl Step for ToolStateCheck {
         }
 
         check_changed_files(&toolstates);
+        checkout_toolstate_repo();
+        let old_toolstate = read_old_toolstate();
 
         for (tool, _) in STABLE_TOOLS.iter() {
             let state = toolstates[*tool];
@@ -180,11 +203,24 @@ impl Step for ToolStateCheck {
                     did_error = true;
                     eprintln!("error: Tool `{}` should be test-pass but is {}", tool, state);
                 } else if in_beta_week {
-                    did_error = true;
-                    eprintln!(
-                        "error: Tool `{}` should be test-pass but is {} during beta week.",
-                        tool, state
-                    );
+                    let old_state = old_toolstate
+                        .iter()
+                        .find(|ts| ts.tool == *tool)
+                        .expect("latest.json missing tool")
+                        .state();
+                    if state < old_state {
+                        did_error = true;
+                        eprintln!(
+                            "error: Tool `{}` has regressed from {} to {} during beta week.",
+                            tool, old_state, state
+                        );
+                    } else {
+                        eprintln!(
+                            "warning: Tool `{}` is not test-pass (is `{}`), \
+                            this should be fixed before beta is branched.",
+                            tool, state
+                        );
+                    }
                 }
             }
         }
@@ -247,6 +283,70 @@ impl Builder<'_> {
     }
 }
 
+fn toolstate_repo() -> String {
+    env::var("TOOLSTATE_REPO")
+        .unwrap_or_else(|_| "https://github.com/rust-lang-nursery/rust-toolstate.git".to_string())
+}
+
+/// Directory where the toolstate repo is checked out.
+const TOOLSTATE_DIR: &str = "rust-toolstate";
+
+/// Checks out the toolstate repo into `TOOLSTATE_DIR`.
+fn checkout_toolstate_repo() {
+    if let Ok(token) = env::var("TOOLSTATE_REPO_ACCESS_TOKEN") {
+        prepare_toolstate_config(&token);
+    }
+    if Path::new(TOOLSTATE_DIR).exists() {
+        eprintln!("Cleaning old toolstate directory...");
+        t!(fs::remove_dir_all(TOOLSTATE_DIR));
+    }
+
+    let status = Command::new("git")
+        .arg("clone")
+        .arg("--depth=1")
+        .arg(toolstate_repo())
+        .arg(TOOLSTATE_DIR)
+        .status();
+    let success = match status {
+        Ok(s) => s.success(),
+        Err(_) => false,
+    };
+    if !success {
+        panic!("git clone unsuccessful (status: {:?})", status);
+    }
+}
+
+/// Sets up config and authentication for modifying the toolstate repo.
+fn prepare_toolstate_config(token: &str) {
+    fn git_config(key: &str, value: &str) {
+        let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status();
+        let success = match status {
+            Ok(s) => s.success(),
+            Err(_) => false,
+        };
+        if !success {
+            panic!("git config key={} value={} successful (status: {:?})", key, value, status);
+        }
+    }
+
+    // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date
+    // as well.
+    git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com");
+    git_config("user.name", "Rust Toolstate Update");
+    git_config("credential.helper", "store");
+
+    let credential = format!("https://{}:x-oauth-basic@github.com\n", token,);
+    let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials");
+    t!(fs::write(&git_credential_path, credential));
+}
+
+/// Reads the latest toolstate from the toolstate repo.
+fn read_old_toolstate() -> Vec<RepoState> {
+    let latest_path = Path::new(TOOLSTATE_DIR).join("_data").join("latest.json");
+    let old_toolstate = t!(fs::read(latest_path));
+    t!(serde_json::from_slice(&old_toolstate))
+}
+
 /// This function `commit_toolstate_change` provides functionality for pushing a change
 /// to the `rust-toolstate` repository.
 ///
@@ -274,45 +374,7 @@ impl Builder<'_> {
 ///       * See <https://help.github.com/articles/about-commit-email-addresses/>
 ///           if a private email by GitHub is wanted.
 fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool) {
-    fn git_config(key: &str, value: &str) {
-        let status = Command::new("git").arg("config").arg("--global").arg(key).arg(value).status();
-        let success = match status {
-            Ok(s) => s.success(),
-            Err(_) => false,
-        };
-        if !success {
-            panic!("git config key={} value={} successful (status: {:?})", key, value, status);
-        }
-    }
-
-    // If changing anything here, then please check that src/ci/publish_toolstate.sh is up to date
-    // as well.
-    git_config("user.email", "7378925+rust-toolstate-update@users.noreply.github.com");
-    git_config("user.name", "Rust Toolstate Update");
-    git_config("credential.helper", "store");
-
-    let credential = format!(
-        "https://{}:x-oauth-basic@github.com\n",
-        t!(env::var("TOOLSTATE_REPO_ACCESS_TOKEN")),
-    );
-    let git_credential_path = PathBuf::from(t!(env::var("HOME"))).join(".git-credentials");
-    t!(fs::write(&git_credential_path, credential));
-
-    let status = Command::new("git")
-        .arg("clone")
-        .arg("--depth=1")
-        .arg(t!(env::var("TOOLSTATE_REPO")))
-        .status();
-    let success = match status {
-        Ok(s) => s.success(),
-        Err(_) => false,
-    };
-    if !success {
-        panic!("git clone successful (status: {:?})", status);
-    }
-
-    let old_toolstate = t!(fs::read("rust-toolstate/_data/latest.json"));
-    let old_toolstate: Vec<RepoState> = t!(serde_json::from_slice(&old_toolstate));
+    let old_toolstate = read_old_toolstate();
 
     let message = format!("({} CI update)", OS.expect("linux/windows only"));
     let mut success = false;
@@ -322,7 +384,7 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool
 
         // `git commit` failing means nothing to commit.
         let status = t!(Command::new("git")
-            .current_dir("rust-toolstate")
+            .current_dir(TOOLSTATE_DIR)
             .arg("commit")
             .arg("-a")
             .arg("-m")
@@ -334,7 +396,7 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool
         }
 
         let status = t!(Command::new("git")
-            .current_dir("rust-toolstate")
+            .current_dir(TOOLSTATE_DIR)
             .arg("push")
             .arg("origin")
             .arg("master")
@@ -347,14 +409,14 @@ fn commit_toolstate_change(current_toolstate: &ToolstateData, in_beta_week: bool
         eprintln!("Sleeping for 3 seconds before retrying push");
         std::thread::sleep(std::time::Duration::from_secs(3));
         let status = t!(Command::new("git")
-            .current_dir("rust-toolstate")
+            .current_dir(TOOLSTATE_DIR)
             .arg("fetch")
             .arg("origin")
             .arg("master")
             .status());
         assert!(status.success());
         let status = t!(Command::new("git")
-            .current_dir("rust-toolstate")
+            .current_dir(TOOLSTATE_DIR)
             .arg("reset")
             .arg("--hard")
             .arg("origin/master")
@@ -375,18 +437,12 @@ fn change_toolstate(
     let mut regressed = false;
     for repo_state in old_toolstate {
         let tool = &repo_state.tool;
-        let state = if cfg!(target_os = "linux") {
-            &repo_state.linux
-        } else if cfg!(windows) {
-            &repo_state.windows
-        } else {
-            unimplemented!()
-        };
+        let state = repo_state.state();
         let new_state = current_toolstate[tool.as_str()];
 
-        if new_state != *state {
+        if new_state != state {
             eprintln!("The state of `{}` has changed from `{}` to `{}`", tool, state, new_state);
-            if (new_state as u8) < (*state as u8) {
+            if new_state < state {
                 if !["rustc-guide", "miri", "embedded-book"].contains(&tool.as_str()) {
                     regressed = true;
                 }
@@ -403,7 +459,9 @@ fn change_toolstate(
 
     let toolstate_serialized = t!(serde_json::to_string(&current_toolstate));
 
-    let history_path = format!("rust-toolstate/history/{}.tsv", OS.expect("linux/windows only"));
+    let history_path = Path::new(TOOLSTATE_DIR)
+        .join("history")
+        .join(format!("{}.tsv", OS.expect("linux/windows only")));
     let mut file = t!(fs::read_to_string(&history_path));
     let end_of_first_line = file.find('\n').unwrap();
     file.insert_str(end_of_first_line, &format!("\n{}\t{}", commit.trim(), toolstate_serialized));
@@ -418,3 +476,15 @@ struct RepoState {
     commit: String,
     datetime: String,
 }
+
+impl RepoState {
+    fn state(&self) -> ToolState {
+        if cfg!(target_os = "linux") {
+            self.linux
+        } else if cfg!(windows) {
+            self.windows
+        } else {
+            unimplemented!()
+        }
+    }
+}