Josh preparations

clippy_dev/Cargo.toml

@@ -7,10 +7,12 @@ edition = "2024"
 [dependencies]
 chrono = { version = "0.4.38", default-features = false, features = ["clock"] }
 clap = { version = "4.4", features = ["derive"] }
+directories = "5"
 indoc = "1.0"
 itertools = "0.12"
 opener = "0.7"
 walkdir = "2.3"
+xshell = "0.2"
 
 [package.metadata.rust-analyzer]
 # This package uses #[feature(rustc_private)]

clippy_dev/src/main.rs

@@ -86,10 +86,17 @@ fn main() {
         ),
         DevCommand::Deprecate { name, reason } => deprecate_lint::deprecate(clippy.version, &name, &reason),
         DevCommand::Sync(SyncCommand { subcommand }) => match subcommand {
-            SyncSubcommand::UpdateNightly => sync::update_nightly(),
+            SyncSubcommand::Pull => sync::rustc_pull(),
+            SyncSubcommand::Push {
+                repo_path,
+                user,
+                branch,
+                force,
+            } => sync::rustc_push(repo_path, &user, &branch, force),
         },
         DevCommand::Release(ReleaseCommand { subcommand }) => match subcommand {
             ReleaseSubcommand::BumpVersion => release::bump_version(clippy.version),
+            ReleaseSubcommand::Commit { repo_path, branch } => release::rustc_clippy_commit(repo_path, branch),
         },
     }
 }
@@ -328,9 +335,24 @@ struct SyncCommand {
 
 #[derive(Subcommand)]
 enum SyncSubcommand {
-    #[command(name = "update_nightly")]
-    /// Update nightly version in `rust-toolchain.toml` and `clippy_utils`
-    UpdateNightly,
+    /// Pull changes from rustc and update the toolchain
+    Pull,
+    /// Push changes to rustc
+    Push {
+        /// The path to a rustc repo that will be used for pushing changes
+        repo_path: String,
+        #[arg(long)]
+        /// The GitHub username to use for pushing changes
+        user: String,
+        #[arg(long, short, default_value = "clippy-subtree-update")]
+        /// The branch to push to
+        ///
+        /// This is mostly for experimentation and usually the default should be used.
+        branch: String,
+        #[arg(long, short)]
+        /// Force push changes
+        force: bool,
+    },
 }
 
 #[derive(Args)]
@@ -344,4 +366,11 @@ enum ReleaseSubcommand {
     #[command(name = "bump_version")]
     /// Bump the version in the Cargo.toml files
     BumpVersion,
+    /// Print the Clippy commit in the rustc repo for the specified branch
+    Commit {
+        /// The path to a rustc repo to look for the commit
+        repo_path: String,
+        /// For which branch to print the commit
+        branch: release::Branch,
+    },
 }

clippy_dev/src/release.rs

@@ -1,5 +1,10 @@
+use crate::sync::PUSH_PR_DESCRIPTION;
 use crate::utils::{FileUpdater, UpdateStatus, Version, parse_cargo_package};
-use std::fmt::Write;
+
+use std::fmt::{Display, Write};
+
+use clap::ValueEnum;
+use xshell::{Shell, cmd};
 
 static CARGO_TOML_FILES: &[&str] = &[
     "clippy_config/Cargo.toml",
@@ -27,3 +32,47 @@ pub fn bump_version(mut version: Version) {
         });
     }
 }
+
+#[derive(ValueEnum, Copy, Clone)]
+pub enum Branch {
+    Stable,
+    Beta,
+    Master,
+}
+
+impl Display for Branch {
+    fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+        match self {
+            Branch::Stable => write!(f, "stable"),
+            Branch::Beta => write!(f, "beta"),
+            Branch::Master => write!(f, "master"),
+        }
+    }
+}
+
+pub fn rustc_clippy_commit(rustc_path: String, branch: Branch) {
+    let sh = Shell::new().expect("failed to create shell");
+    sh.change_dir(rustc_path);
+
+    let base = branch.to_string();
+    cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}")
+        .run()
+        .expect("failed to fetch base commit");
+    let last_rustup_commit = cmd!(
+        sh,
+        "git log -1 --merges --grep=\"{PUSH_PR_DESCRIPTION}\" FETCH_HEAD -- src/tools/clippy"
+    )
+    .read()
+    .expect("failed to run git log");
+
+    let commit = last_rustup_commit
+        .lines()
+        .find(|c| c.contains("Sync from Clippy commit:"))
+        .expect("no commit found")
+        .trim()
+        .rsplit_once('@')
+        .expect("no commit hash found")
+        .1;
+
+    println!("{commit}");
+}

clippy_dev/src/sync.rs

@@ -1,18 +1,248 @@
 use crate::utils::{FileUpdater, update_text_region_fn};
 use chrono::offset::Utc;
 use std::fmt::Write;
+use std::process;
+use std::process::exit;
 
-pub fn update_nightly() {
+use xshell::{Shell, cmd};
+
+const JOSH_FILTER: &str = ":rev(0450db33a5d8587f7c1d4b6d233dac963605766b:prefix=src/tools/clippy):/src/tools/clippy";
+const JOSH_PORT: &str = "42042";
+const TOOLCHAIN_TOML: &str = "rust-toolchain.toml";
+const UTILS_README: &str = "clippy_utils/README.md";
+
+fn start_josh() -> impl Drop {
+    // Create a wrapper that stops it on drop.
+    struct Josh(process::Child);
+    impl Drop for Josh {
+        fn drop(&mut self) {
+            #[cfg(unix)]
+            {
+                // Try to gracefully shut it down.
+                process::Command::new("kill")
+                    .args(["-s", "INT", &self.0.id().to_string()])
+                    .output()
+                    .expect("failed to SIGINT josh-proxy");
+                // Sadly there is no "wait with timeout"... so we just give it some time to finish.
+                std::thread::sleep(std::time::Duration::from_secs(1));
+                // Now hopefully it is gone.
+                if self.0.try_wait().expect("failed to wait for josh-proxy").is_some() {
+                    return;
+                }
+            }
+            // If that didn't work (or we're not on Unix), kill it hard.
+            eprintln!("I have to kill josh-proxy the hard way, let's hope this does not break anything.");
+            self.0.kill().expect("failed to SIGKILL josh-proxy");
+        }
+    }
+
+    // Determine cache directory.
+    let local_dir = {
+        let user_dirs = directories::ProjectDirs::from("org", "rust-lang", "clippy-josh").unwrap();
+        user_dirs.cache_dir().to_owned()
+    };
+    println!("Using local cache directory: {}", local_dir.display());
+
+    // Start josh, silencing its output.
+    let mut cmd = process::Command::new("josh-proxy");
+    cmd.arg("--local").arg(local_dir);
+    cmd.arg("--remote").arg("https://github.com");
+    cmd.arg("--port").arg(JOSH_PORT);
+    cmd.arg("--no-background");
+    cmd.stdout(process::Stdio::null());
+    cmd.stderr(process::Stdio::null());
+    let josh = cmd
+        .spawn()
+        .expect("failed to start josh-proxy, make sure it is installed");
+    // Give it some time so hopefully the port is open.
+    std::thread::sleep(std::time::Duration::from_secs(1));
+
+    Josh(josh)
+}
+
+fn rustc_hash() -> String {
+    let sh = Shell::new().expect("failed to create shell");
+    // Make sure we pick up the updated toolchain (usually rustup pins the toolchain
+    // inside a single cargo/rustc invocation via this env var).
+    sh.set_var("RUSTUP_TOOLCHAIN", "");
+    cmd!(sh, "rustc --version --verbose")
+        .read()
+        .expect("failed to run `rustc -vV`")
+        .lines()
+        .find(|line| line.starts_with("commit-hash:"))
+        .expect("failed to parse `rustc -vV`")
+        .split_whitespace()
+        .last()
+        .expect("failed to get commit from `rustc -vV`")
+        .to_string()
+}
+
+fn assert_clean_repo(sh: &Shell) {
+    if !cmd!(sh, "git status --untracked-files=no --porcelain")
+        .read()
+        .expect("failed to run git status")
+        .is_empty()
+    {
+        eprintln!("working directory must be clean before running `cargo dev sync pull`");
+        exit(1);
+    }
+}
+
+pub fn rustc_pull() {
+    const MERGE_COMMIT_MESSAGE: &str = "Merge from rustc";
+
+    let sh = Shell::new().expect("failed to create shell");
+
+    assert_clean_repo(&sh);
+
+    // Update rust-toolchain file
     let date = Utc::now().format("%Y-%m-%d").to_string();
-    let update = &mut update_text_region_fn(
+    let toolchain_update = &mut update_text_region_fn(
         "# begin autogenerated nightly\n",
         "# end autogenerated nightly",
         |dst| {
             writeln!(dst, "channel = \"nightly-{date}\"").unwrap();
         },
     );
+    let readme_update = &mut update_text_region_fn(
+        "<!-- begin autogenerated nightly -->\n",
+        "<!-- end autogenerated nightly -->",
+        |dst| {
+            writeln!(dst, "```\nnightly-{date}\n```").unwrap();
+        },
+    );
 
     let mut updater = FileUpdater::default();
-    updater.update_file("rust-toolchain.toml", update);
-    updater.update_file("clippy_utils/README.md", update);
+    updater.update_file(TOOLCHAIN_TOML, toolchain_update);
+    updater.update_file(UTILS_README, readme_update);
+
+    let message = format!("Bump nightly version -> {date}");
+    cmd!(
+        sh,
+        "git commit --no-verify -m {message} -- {TOOLCHAIN_TOML} {UTILS_README}"
+    )
+    .run()
+    .expect("FAILED to commit rust-toolchain.toml file, something went wrong");
+
+    let commit = rustc_hash();
+
+    // Make sure josh is running in this scope
+    {
+        let _josh = start_josh();
+
+        // Fetch given rustc commit.
+        cmd!(
+            sh,
+            "git fetch http://localhost:{JOSH_PORT}/rust-lang/rust.git@{commit}{JOSH_FILTER}.git"
+        )
+        .run()
+        .inspect_err(|_| {
+            // Try to un-do the previous `git commit`, to leave the repo in the state we found it.
+            cmd!(sh, "git reset --hard HEAD^")
+                .run()
+                .expect("FAILED to clean up again after failed `git fetch`, sorry for that");
+        })
+        .expect("FAILED to fetch new commits, something went wrong");
+    }
+
+    // This should not add any new root commits. So count those before and after merging.
+    let num_roots = || -> u32 {
+        cmd!(sh, "git rev-list HEAD --max-parents=0 --count")
+            .read()
+            .expect("failed to determine the number of root commits")
+            .parse::<u32>()
+            .unwrap()
+    };
+    let num_roots_before = num_roots();
+
+    // Merge the fetched commit.
+    cmd!(sh, "git merge FETCH_HEAD --no-verify --no-ff -m {MERGE_COMMIT_MESSAGE}")
+        .run()
+        .expect("FAILED to merge new commits, something went wrong");
+
+    // Check that the number of roots did not increase.
+    if num_roots() != num_roots_before {
+        eprintln!("Josh created a new root commit. This is probably not the history you want.");
+        exit(1);
+    }
+}
+
+pub(crate) const PUSH_PR_DESCRIPTION: &str = "Sync from Clippy commit:";
+
+pub fn rustc_push(rustc_path: String, github_user: &str, branch: &str, force: bool) {
+    let sh = Shell::new().expect("failed to create shell");
+
+    assert_clean_repo(&sh);
+
+    // Prepare the branch. Pushing works much better if we use as base exactly
+    // the commit that we pulled from last time, so we use the `rustc --version`
+    // to find out which commit that would be.
+    let base = rustc_hash();
+
+    println!("Preparing {github_user}/rust (base: {base})...");
+    sh.change_dir(rustc_path);
+    if !force
+        && cmd!(sh, "git fetch https://github.com/{github_user}/rust {branch}")
+            .ignore_stderr()
+            .read()
+            .is_ok()
+    {
+        eprintln!(
+            "The branch '{branch}' seems to already exist in 'https://github.com/{github_user}/rust'. Please delete it and try again."
+        );
+        exit(1);
+    }
+    cmd!(sh, "git fetch https://github.com/rust-lang/rust {base}")
+        .run()
+        .expect("failed to fetch base commit");
+    let force_flag = if force { "--force" } else { "" };
+    cmd!(
+        sh,
+        "git push https://github.com/{github_user}/rust {base}:refs/heads/{branch} {force_flag}"
+    )
+    .ignore_stdout()
+    .ignore_stderr() // silence the "create GitHub PR" message
+    .run()
+    .expect("failed to push base commit to the new branch");
+
+    // Make sure josh is running in this scope
+    {
+        let _josh = start_josh();
+
+        // Do the actual push.
+        println!("Pushing Clippy changes...");
+        cmd!(
+            sh,
+            "git push http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git HEAD:{branch}"
+        )
+        .run()
+        .expect("failed to push changes to Josh");
+
+        // Do a round-trip check to make sure the push worked as expected.
+        cmd!(
+            sh,
+            "git fetch http://localhost:{JOSH_PORT}/{github_user}/rust.git{JOSH_FILTER}.git {branch}"
+        )
+        .ignore_stderr()
+        .read()
+        .expect("failed to fetch the branch from Josh");
+    }
+
+    let head = cmd!(sh, "git rev-parse HEAD")
+        .read()
+        .expect("failed to get HEAD commit");
+    let fetch_head = cmd!(sh, "git rev-parse FETCH_HEAD")
+        .read()
+        .expect("failed to get FETCH_HEAD");
+    if head != fetch_head {
+        eprintln!("Josh created a non-roundtrip push! Do NOT merge this into rustc!");
+        exit(1);
+    }
+    println!("Confirmed that the push round-trips back to Clippy properly. Please create a rustc PR:");
+    let description = format!("{}+rust-lang/rust-clippy@{head}", PUSH_PR_DESCRIPTION.replace(' ', "+"));
+    println!(
+        // Open PR with `subtree update` title to silence the `no-merges` triagebot check
+        // See https://github.com/rust-lang/rust/pull/114157
+        "    https://github.com/rust-lang/rust/compare/{github_user}:{branch}?quick_pull=1&title=Clippy+subtree+update&body=r?+@ghost%0A%0A{description}"
+    );
 }