Auto merge of #99603 - fasterthanlime:ra-subtree-2, r=jyn514

Convert rust-analyzer to an in-tree tool

This re-adds `rust-lang/rust-analyzer` as a git subtree rather than a submodule.

Closes rust-lang/rust-analyzer#12815.

Prior attempt (research PR): #99465

  * [x] Remove submodule: `git rm -f src/tools/rust-analyzer`
  * [x] Add subtree: `git subtree add -P src/tools/rust-analyzer https://github.com/rust-lang/rust-analyzer.git master`
  * [x] Move to `SourceType::InTree`,
  * [x] Enable `rust-analyzer/in-rust-tree` feature when built through `x.py`
  * [x] Add 'check' step
  * [x] Add 'test' step

With this PR, rust-analyzer becomes an "in-tree" tool. Syncs can happen in both directions, see [clippy's relevant book section](https://doc.rust-lang.org/nightly/clippy/development/infrastructure/sync.html).

Making sure `proc-macro-srv` doesn't break when the proc_macro bridge changes effectively becomes the responsibility of `rust-lang/rust` contributors. These days, that's mostly `@mystor,` who has been consulted throughout the process. I'm also making myself available in case there's questions / work needed that nobody else signed up for.

This doesn't change rust-analyzer's release cycle. After this PR is merged and the next nightly goes out, one can point `rust-analyzer.procMacro.server` to the rustup-provided `rust-analyzer` binary. Changes to improve the situation further (auto-discovery/install of the rust-analyzer component) will happen in `rust-lang/rust-analyzer` and be synced here eventually.

.gitmodules

@@ -38,9 +38,6 @@
 [submodule "src/doc/embedded-book"]
 	path = src/doc/embedded-book
 	url = https://github.com/rust-embedded/book.git
-[submodule "src/tools/rust-analyzer"]
-	path = src/tools/rust-analyzer
-	url = https://github.com/rust-analyzer/rust-analyzer.git
 [submodule "library/backtrace"]
 	path = library/backtrace
 	url = https://github.com/rust-lang/backtrace-rs.git

src/bootstrap/builder.rs

@@ -621,6 +621,7 @@ impl<'a> Builder<'a> {
                 check::Clippy,
                 check::Miri,
                 check::Rls,
+                check::RustAnalyzer,
                 check::Rustfmt,
                 check::Bootstrap
             ),
@@ -648,6 +649,7 @@ impl<'a> Builder<'a> {
                 test::Cargotest,
                 test::Cargo,
                 test::Rls,
+                test::RustAnalyzer,
                 test::ErrorIndex,
                 test::Distcheck,
                 test::RunMakeFullDeps,
@@ -1551,7 +1553,7 @@ impl<'a> Builder<'a> {
             Mode::ToolStd => {
                 // Right now this is just compiletest and a few other tools that build on stable.
                 // Allow them to use `feature(test)`, but nothing else.
-                rustflags.arg("-Zallow-features=binary-dep-depinfo,test,backtrace");
+                rustflags.arg("-Zallow-features=binary-dep-depinfo,test,backtrace,proc_macro_internals,proc_macro_diagnostic,proc_macro_span");
             }
             Mode::Std | Mode::Rustc | Mode::Codegen | Mode::ToolRustc => {}
         }

src/bootstrap/check.rs

@@ -301,6 +301,68 @@ impl Step for CodegenBackend {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzer {
+    pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-analyzer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustAnalyzer { target: run.target });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        let compiler = builder.compiler(builder.top_stage, builder.config.build);
+        let target = self.target;
+
+        builder.ensure(Std { target });
+
+        let mut cargo = prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolStd,
+            target,
+            cargo_subcommand(builder.kind),
+            "src/tools/rust-analyzer",
+            SourceType::InTree,
+            &["rust-analyzer/in-rust-tree".to_owned()],
+        );
+
+        cargo.rustflag(
+            "-Zallow-features=proc_macro_internals,proc_macro_diagnostic,proc_macro_span",
+        );
+
+        // For ./x.py clippy, don't check those targets because
+        // linting tests and benchmarks can produce very noisy results
+        if builder.kind != Kind::Clippy {
+            // can't use `--all-targets` because `--examples` doesn't work well
+            cargo.arg("--bins");
+            cargo.arg("--tests");
+            cargo.arg("--benches");
+        }
+
+        builder.info(&format!(
+            "Checking stage{} {} artifacts ({} -> {})",
+            compiler.stage, "rust-analyzer", &compiler.host.triple, target.triple
+        ));
+        run_cargo(builder, cargo, args(builder), &stamp(builder, compiler, target), vec![], true);
+
+        /// Cargo's output path in a given stage, compiled by a particular
+        /// compiler for the specified target.
+        fn stamp(builder: &Builder<'_>, compiler: Compiler, target: TargetSelection) -> PathBuf {
+            builder.cargo_out(compiler, Mode::ToolStd, target).join(".rust-analyzer-check.stamp")
+        }
+    }
+}
+
 macro_rules! tool_check_step {
     ($name:ident, $path:literal, $($alias:literal, )* $source_type:path $(, $default:literal )?) => {
         #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]

src/bootstrap/dist.rs

@@ -1058,7 +1058,7 @@ impl Step for RustAnalyzer {
         }
 
         let rust_analyzer = builder
-            .ensure(tool::RustAnalyzer { compiler, target, extra_features: Vec::new() })
+            .ensure(tool::RustAnalyzer { compiler, target })
             .expect("rust-analyzer always builds");
 
         let mut tarball = Tarball::new(builder, "rust-analyzer", &target.triple);

src/bootstrap/test.rs

@@ -352,6 +352,64 @@ impl Step for Rls {
     }
 }
 
+#[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
+pub struct RustAnalyzer {
+    stage: u32,
+    host: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+    const DEFAULT: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/tools/rust-analyzer")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(Self { stage: run.builder.top_stage, host: run.target });
+    }
+
+    /// Runs `cargo test` for rust-analyzer
+    fn run(self, builder: &Builder<'_>) {
+        let stage = self.stage;
+        let host = self.host;
+        let compiler = builder.compiler(stage, host);
+
+        builder.ensure(tool::RustAnalyzer { compiler, target: self.host }).expect("in-tree tool");
+
+        let workspace_path = "src/tools/rust-analyzer";
+        // until the whole RA test suite runs on `i686`, we only run
+        // `proc-macro-srv` tests
+        let crate_path = "src/tools/rust-analyzer/crates/proc-macro-srv";
+        let mut cargo = tool::prepare_tool_cargo(
+            builder,
+            compiler,
+            Mode::ToolStd,
+            host,
+            "test",
+            crate_path,
+            SourceType::InTree,
+            &["sysroot-abi".to_owned()],
+        );
+
+        let dir = builder.src.join(workspace_path);
+        // needed by rust-analyzer to find its own text fixtures, cf.
+        // https://github.com/rust-analyzer/expect-test/issues/33
+        cargo.env("CARGO_WORKSPACE_DIR", &dir);
+
+        // RA's test suite tries to write to the source directory, that can't
+        // work in Rust CI
+        cargo.env("SKIP_SLOW_TESTS", "1");
+
+        cargo.add_rustc_lib_path(builder, compiler);
+        cargo.arg("--").args(builder.config.cmd.test_args());
+
+        builder.run(&mut cargo.into());
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Rustfmt {
     stage: u32,

src/bootstrap/tool.rs

@@ -1,7 +1,7 @@
 use std::collections::HashSet;
 use std::env;
 use std::fs;
-use std::path::{Path, PathBuf};
+use std::path::PathBuf;
 use std::process::Command;
 
 use crate::builder::{Builder, Cargo as CargoCommand, RunConfig, ShouldRun, Step};
@@ -683,6 +683,50 @@ impl Step for LldWrapper {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustAnalyzer {
+    pub compiler: Compiler,
+    pub target: TargetSelection,
+}
+
+impl Step for RustAnalyzer {
+    type Output = Option<PathBuf>;
+    const DEFAULT: bool = true;
+    const ONLY_HOSTS: bool = false;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        let builder = run.builder;
+        run.path("src/tools/rust-analyzer").default_condition(
+            builder.config.extended
+                && builder
+                    .config
+                    .tools
+                    .as_ref()
+                    .map_or(true, |tools| tools.iter().any(|tool| tool == "rust-analyzer")),
+        )
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        run.builder.ensure(RustAnalyzer {
+            compiler: run.builder.compiler(run.builder.top_stage, run.builder.config.build),
+            target: run.target,
+        });
+    }
+
+    fn run(self, builder: &Builder<'_>) -> Option<PathBuf> {
+        builder.ensure(ToolBuild {
+            compiler: self.compiler,
+            target: self.target,
+            tool: "rust-analyzer",
+            mode: Mode::ToolStd,
+            path: "src/tools/rust-analyzer",
+            extra_features: vec!["rust-analyzer/in-rust-tree".to_owned()],
+            is_optional_tool: false,
+            source_type: SourceType::InTree,
+        })
+    }
+}
+
 macro_rules! tool_extended {
     (($sel:ident, $builder:ident),
        $($name:ident,
@@ -780,7 +824,6 @@ tool_extended!((self, builder),
     // and this is close enough for now.
     RustDemangler, rust_demangler, "src/tools/rust-demangler", "rust-demangler", stable=false, in_tree=true, tool_std=true, {};
     Rustfmt, rustfmt, "src/tools/rustfmt", "rustfmt", stable=true, in_tree=true, {};
-    RustAnalyzer, rust_analyzer, "src/tools/rust-analyzer/crates/rust-analyzer", "rust-analyzer", stable=true, submodule="rust-analyzer", {};
 );
 
 impl<'a> Builder<'a> {

src/tools/rust-analyzer/.cargo/config.toml

@@ -0,0 +1,11 @@
+[alias]
+xtask = "run --package xtask --bin xtask --"
+tq = "test -- -q"
+qt = "tq"
+lint = "clippy --all-targets -- -Aclippy::collapsible_if -Aclippy::needless_pass_by_value -Aclippy::nonminimal_bool -Aclippy::redundant_pattern_matching --cap-lints warn"
+
+[target.x86_64-pc-windows-msvc]
+linker = "rust-lld"
+
+[env]
+CARGO_WORKSPACE_DIR = { value = "", relative = true }

src/tools/rust-analyzer/.editorconfig

@@ -0,0 +1,19 @@
+# https://EditorConfig.org
+root = true
+
+[*]
+charset = utf-8
+trim_trailing_whitespace = true
+end_of_line = lf
+insert_final_newline = true
+indent_style = space
+
+[*.{rs,toml}]
+indent_size = 4
+
+[*.ts]
+indent_size = 4
+[*.js]
+indent_size = 4
+[*.json]
+indent_size = 4

src/tools/rust-analyzer/.git-blame-ignore-revs

@@ -0,0 +1,8 @@
+# for this file to take effect make sure you use git ^2.23 and
+# add ignoreFile to your git configuration:
+# ```
+# git config --global blame.ignoreRevsFile .git-blame-ignore-revs
+# ```
+
+# prettier format
+f247090558c9ba3c551566eae5882b7ca865225f

src/tools/rust-analyzer/.gitattributes

@@ -0,0 +1,9 @@
+* text=auto eol=lf
+
+# git grep shouldn't match entries in this benchmark data
+bench_data/** binary
+
+# Older git versions try to fix line endings on images, this prevents it.
+*.png binary
+*.jpg binary
+*.ico binary

src/tools/rust-analyzer/.github/ISSUE_TEMPLATE/blank_issue.md

@@ -0,0 +1,10 @@
+---
+name: Blank Issue
+about: Create a blank issue.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+

src/tools/rust-analyzer/.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,26 @@
+---
+name: Bug report
+about: Create a bug report for rust-analyzer.
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+<!--
+Troubleshooting guide: https://rust-analyzer.github.io/manual.html#troubleshooting
+Forum for questions: https://users.rust-lang.org/c/ide/14
+
+Before submitting, please make sure that you're not running into one of these known issues:
+
+ 1. extension doesn't load in VSCodium: #11080
+ 2. on-the-fly diagnostics are mostly unimplemented (`cargo check` diagnostics will be shown when saving a file): #3107
+
+Otherwise please try to provide information which will help us to fix the issue faster. Minimal reproducible examples with few dependencies are especially lovely <3.
+-->
+
+**rust-analyzer version**: (eg. output of "Rust Analyzer: Show RA Version" command)
+
+**rustc version**: (eg. output of `rustc -V`)
+
+**relevant settings**: (eg. client settings, or environment variables like `CARGO`, `RUSTUP_HOME` or `CARGO_HOME`)

src/tools/rust-analyzer/.github/ISSUE_TEMPLATE/critical_nightly_regression.md

@@ -0,0 +1,17 @@
+---
+name: Critical Nightly Regression
+about: You are using nightly rust-analyzer and the latest version is unusable.
+title: ''
+labels: ''
+assignees: 'matklad'
+
+---
+
+<!--
+Troubleshooting guide: https://rust-analyzer.github.io/manual.html#troubleshooting
+
+Please try to provide information which will help us to fix the issue faster. Minimal reproducible examples with few dependencies are especially lovely <3.
+-->
+
+This is a serious regression in nightly and it's important to fix it before the next release.
+@matklad, please take a look.

src/tools/rust-analyzer/.github/actions/github-release/Dockerfile

@@ -0,0 +1,8 @@
+FROM node:slim
+
+COPY . /action
+WORKDIR /action
+
+RUN npm install --production
+
+ENTRYPOINT ["node", "/action/main.js"]

src/tools/rust-analyzer/.github/actions/github-release/README.md

@@ -0,0 +1,21 @@
+# github-release
+
+Copy-pasted from
+https://github.com/bytecodealliance/wasmtime/tree/8acfdbdd8aa550d1b84e0ce1e6222a6605d14e38/.github/actions/github-release
+
+An action used to publish GitHub releases for `wasmtime`.
+
+As of the time of this writing there's a few actions floating around which
+perform github releases but they all tend to have their set of drawbacks.
+Additionally nothing handles deleting releases which we need for our rolling
+`dev` release.
+
+To handle all this this action rolls-its-own implementation using the
+actions/toolkit repository and packages published there. These run in a Docker
+container and take various inputs to orchestrate the release from the build.
+
+More comments can be found in `main.js`.
+
+Testing this is really hard. If you want to try though run `npm install` and
+then `node main.js`. You'll have to configure a bunch of env vars though to get
+anything reasonably working.