Auto merge of #42612 - est31:master, r=nagisa

Autogenerate stubs and SUMMARY.md in the unstable book

Removes a speed bump in compiler development by autogenerating stubs for features in the unstable book. See #42454 for discussion.

The PR contains three commits, separated in order to make review easy:

* The first commit converts the tidy tool from a binary crate to a crate that contains both a library and a binary. In the second commit, we'll use the tidy library
* The second and main commit introduces autogeneration of SUMMARY.md and feature stub files
* The third commit turns off the tidy lint that checks for features without a stub, and removes the stub files. A separate commit due to the large number of files touched

Members of the doc team who wish to document some features can either do this (where `$rustsrc` is the root of the rust repo git checkout):

1. cd to `$rustsrc/src/tools/unstable-book-gen` and then do `cargo run $rustsrc/src $rustsrc/src/doc/unstable-book` to put the stubs into the unstable book
2. cd to `$rustsrc` and run `git ls-files --others --exclude-standard` to list the newly added stubs
3. choose a file to edit, then `git add` it and `git commit`
4. afterwards, remove all changes by the tool by doing `git --reset hard` and `git clean -f`

Or they can do this:

1. remove the comment marker in `src/tools/tidy/src/unstable_book.rs` line 122
2. run `./x.py test src/tools/tidy` to list the unstable features which only have stubs
3. revert the change in 1
3. document one of the chosen unstable features

The changes done by this PR also allow for further development:

* tidy obtains information about tracking issues. We can now forbid differing tracking issues between differing `#![unstable]` annotations. I haven't done this but plan to in a future PR
* we now have a general framework for generating stuff for the unstable book at build time. Further changes can autogenerate a list of the API a given library feature exposes.

The old way to simply click through the documentation after it has been uploaded to rust-lang.org works as well.

r? @nagisa

Fixes #42454

src/Cargo.toml

@@ -9,6 +9,7 @@ members = [
  "tools/error_index_generator",
  "tools/linkchecker",
  "tools/rustbook",
+ "tools/unstable-book-gen",
  "tools/tidy",
  "tools/build-manifest",
  "tools/remote-test-client",

src/bootstrap/doc.rs

@@ -27,18 +27,26 @@ use {Build, Compiler, Mode};
 use util::{cp_r, symlink_dir};
 use build_helper::up_to_date;
 
-/// Invoke `rustbook` as compiled in `stage` for `target` for the doc book
-/// `name` into the `out` path.
+/// Invoke `rustbook` for `target` for the doc book `name`.
 ///
 /// This will not actually generate any documentation if the documentation has
 /// already been generated.
 pub fn rustbook(build: &Build, target: &str, name: &str) {
+ let src = build.src.join("src/doc");
+ rustbook_src(build, target, name, &src);
+}
+
+/// Invoke `rustbook` for `target` for the doc book `name` from the `src` path.
+///
+/// This will not actually generate any documentation if the documentation has
+/// already been generated.
+pub fn rustbook_src(build: &Build, target: &str, name: &str, src: &Path) {
  let out = build.doc_out(target);
  t!(fs::create_dir_all(&out));
 
  let out = out.join(name);
  let compiler = Compiler::new(0, &build.config.build);
- let src = build.src.join("src/doc").join(name);
+ let src = src.join(name);
  let index = out.join("index.html");
  let rustbook = build.tool(&compiler, "rustbook");
  if up_to_date(&src, &index) && up_to_date(&rustbook, &index) {
@@ -354,6 +362,19 @@ pub fn error_index(build: &Build, target: &str) {
  build.run(&mut index);
 }
 
+pub fn unstable_book_gen(build: &Build, target: &str) {
+ println!("Generating unstable book md files ({})", target);
+ let out = build.md_doc_out(target).join("unstable-book");
+ t!(fs::create_dir_all(&out));
+ t!(fs::remove_dir_all(&out));
+ let compiler = Compiler::new(0, &build.config.build);
+ let mut cmd = build.tool_cmd(&compiler, "unstable-book-gen");
+ cmd.arg(build.src.join("src"));
+ cmd.arg(out);
+
+ build.run(&mut cmd);
+}
+
 fn symlink_dir_force(src: &Path, dst: &Path) -> io::Result<()> {
  if let Ok(m) = fs::symlink_metadata(dst) {
  if m.file_type().is_dir() {

src/bootstrap/lib.rs

@@ -680,6 +680,11 @@ impl Build {
  self.out.join(target).join("doc")
  }
 
+ /// Output directory for some generated md crate documentation for a target (temporary)
+ fn md_doc_out(&self, target: &str) -> PathBuf {
+ self.out.join(target).join("md-doc")
+ }
+
  /// Output directory for all crate documentation for a target (temporary)
  ///
  /// The artifacts here are then copied into `doc_out` above.

src/bootstrap/step.rs

@@ -548,6 +548,10 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
  .dep(|s| s.name("maybe-clean-tools"))
  .dep(|s| s.name("librustc-tool"))
  .run(move |s| compile::tool(build, s.stage, s.target, "error_index_generator"));
+ rules.build("tool-unstable-book-gen", "src/tools/unstable-book-gen")
+ .dep(|s| s.name("maybe-clean-tools"))
+ .dep(|s| s.name("libstd-tool"))
+ .run(move |s| compile::tool(build, s.stage, s.target, "unstable-book-gen"));
  rules.build("tool-tidy", "src/tools/tidy")
  .dep(|s| s.name("maybe-clean-tools"))
  .dep(|s| s.name("libstd-tool"))
@@ -662,8 +666,12 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
  .target(&build.config.build)
  .stage(0)
  })
+ .dep(move |s| s.name("doc-unstable-book-gen"))
  .default(build.config.docs)
- .run(move |s| doc::rustbook(build, s.target, "unstable-book"));
+ .run(move |s| doc::rustbook_src(build,
+ s.target,
+ "unstable-book",
+ &build.md_doc_out(s.target)));
  rules.doc("doc-standalone", "src/doc")
  .dep(move |s| {
  s.name("rustc")
@@ -679,6 +687,17 @@ pub fn build_rules<'a>(build: &'a Build) -> Rules {
  .default(build.config.docs)
  .host(true)
  .run(move |s| doc::error_index(build, s.target));
+ rules.doc("doc-unstable-book-gen", "src/tools/unstable-book-gen")
+ .dep(move |s| {
+ s.name("tool-unstable-book-gen")
+ .host(&build.config.build)
+ .target(&build.config.build)
+ .stage(0)
+ })
+ .dep(move |s| s.name("libstd-link"))
+ .default(build.config.docs)
+ .host(true)
+ .run(move |s| doc::unstable_book_gen(build, s.target));
  for (krate, path, default) in krates("std") {
  rules.doc(&krate.doc_step, path)
  .dep(|s| s.name("libstd-link"))

src/doc/unstable-book/src/language-features/asm.md

@@ -190,4 +190,4 @@ constraints, etc.
 [llvm-docs]: http://llvm.org/docs/LangRef.html#inline-assembler-expressions
 
 If you need more power and don't mind losing some of the niceties of
-`asm!`, check out [global_asm](language-features/global_asm.html).
+`asm!`, check out [global_asm](language-features/global-asm.html).