Auto merge of #32133 - alexcrichton:linkchecker, r=brson

Add a link validator to rustbuild

This commit was originally targeted at just adding a link checking script to the rustbuild system. This ended up snowballing a bit to extend rustbuild to be amenable to various tools we have as part of the build system in general.

There's a new `src/tools` directory which has a number of scripts/programs that are purely intended to be used as part of the build system and CI of this repository. This is currently inhabited by rustbook, the error index generator, and a new linkchecker script added as part of this PR. I suspect that more tools like compiletest, tidy scripts, snapshot scripts, etc will migrate their way into this directory over time.

The commit which adds the error index generator shows the steps necessary to add new tools to the build system, namely:

1. New steps are defined for building the tool and running the tool
2. The dependencies are configured
3. The steps are implemented

In terms of the link checker, these commits do a few things:

* A new `src/tools/linkchecker` script is added. This will read an entire documentation tree looking for broken relative links (HTTP links aren't followed yet).
* A large number of broken links throughout the documentation were fixed. Many of these were just broken when viewed from core as opposed to std, but were easily fixed.
* A few rustdoc bugs here and there were fixed

mk/crates.mk

@@ -126,8 +126,8 @@ TOOL_DEPS_error_index_generator := rustdoc syntax serialize
 TOOL_SOURCE_compiletest := $(S)src/compiletest/compiletest.rs
 TOOL_SOURCE_rustdoc := $(S)src/driver/driver.rs
 TOOL_SOURCE_rustc := $(S)src/driver/driver.rs
-TOOL_SOURCE_rustbook := $(S)src/rustbook/main.rs
-TOOL_SOURCE_error_index_generator := $(S)src/error_index_generator/main.rs
+TOOL_SOURCE_rustbook := $(S)src/tools/rustbook/main.rs
+TOOL_SOURCE_error_index_generator := $(S)src/tools/error_index_generator/main.rs
 
 ONLY_RLIB_core := 1
 ONLY_RLIB_libc := 1

src/bootstrap/Cargo.toml

@@ -15,6 +15,10 @@ path = "main.rs"
 name = "rustc"
 path = "rustc.rs"
 
+[[bin]]
+name = "rustdoc"
+path = "rustdoc.rs"
+
 [dependencies]
 build_helper = { path = "../build_helper" }
 cmake = "0.1.10"

src/bootstrap/build/check.rs

@@ -0,0 +1,21 @@
+// Copyright 2016 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+use std::process::Command;
+
+use build::{Build, Compiler};
+
+pub fn linkcheck(build: &Build, stage: u32, host: &str) {
+    println!("Linkcheck stage{} ({})", stage, host);
+    let compiler = Compiler::new(stage, host);
+    let linkchecker = build.tool(&compiler, "linkchecker");
+    build.run(Command::new(&linkchecker)
+                     .arg(build.out.join(host).join("doc")));
+}

src/bootstrap/build/compile.rs

@@ -16,7 +16,7 @@ use std::process::Command;
 use build_helper::output;
 
 use build::util::{exe, staticlib, libdir, mtime, is_dylib};
-use build::{Build, Compiler};
+use build::{Build, Compiler, Mode};
 
 /// Build the standard library.
 ///
@@ -39,9 +39,10 @@ pub fn std<'a>(build: &'a Build, stage: u32, target: &str,
 
     build_startup_objects(build, target, &libdir);
 
-    let out_dir = build.cargo_out(stage, &host, true, target);
+    let out_dir = build.cargo_out(stage, &host, Mode::Libstd, target);
     build.clear_if_dirty(&out_dir, &build.compiler_path(compiler));
-    let mut cargo = build.cargo(stage, compiler, true, target, "build");
+    let mut cargo = build.cargo(stage, compiler, Mode::Libstd, Some(target),
+                                "build");
     cargo.arg("--features").arg(build.std_features())
          .arg("--manifest-path")
          .arg(build.src.join("src/rustc/std_shim/Cargo.toml"));
@@ -71,7 +72,7 @@ pub fn std_link(build: &Build,
                 compiler: &Compiler,
                 host: &str) {
     let libdir = build.sysroot_libdir(stage, host, target);
-    let out_dir = build.cargo_out(stage, compiler.host, true, target);
+    let out_dir = build.cargo_out(stage, compiler.host, Mode::Libstd, target);
 
     // If we're linking one compiler host's output into another, then we weren't
     // called from the `std` method above. In that case we clean out what's
@@ -135,19 +136,15 @@ pub fn rustc<'a>(build: &'a Build, stage: u32, target: &str,
     println!("Building stage{} compiler artifacts ({} -> {})", stage,
              host, target);
 
-    let out_dir = build.cargo_out(stage, &host, false, target);
+    let out_dir = build.cargo_out(stage, &host, Mode::Librustc, target);
     build.clear_if_dirty(&out_dir, &libstd_shim(build, stage, &host, target));
 
-    let mut cargo = build.cargo(stage, compiler, false, target, "build");
-    cargo.arg("--features").arg(build.rustc_features(stage))
+    let mut cargo = build.cargo(stage, compiler, Mode::Librustc, Some(target),
+                                "build");
+    cargo.arg("--features").arg(build.rustc_features())
          .arg("--manifest-path")
          .arg(build.src.join("src/rustc/Cargo.toml"));
 
-    // In stage0 we may not need to build as many executables
-    if stage == 0 {
-        cargo.arg("--bin").arg("rustc");
-    }
-
     // Set some configuration variables picked up by build scripts and
     // the compiler alike
     cargo.env("CFG_RELEASE", &build.release)
@@ -200,14 +197,14 @@ pub fn rustc_link(build: &Build,
                   compiler: &Compiler,
                   host: &str) {
     let libdir = build.sysroot_libdir(stage, host, target);
-    let out_dir = build.cargo_out(stage, compiler.host, false, target);
+    let out_dir = build.cargo_out(stage, compiler.host, Mode::Librustc, target);
     add_to_sysroot(&out_dir, &libdir);
 }
 
 /// Cargo's output path for the standard library in a given stage, compiled
 /// by a particular compiler for the specified target.
 fn libstd_shim(build: &Build, stage: u32, host: &str, target: &str) -> PathBuf {
-    build.cargo_out(stage, host, true, target).join("libstd_shim.rlib")
+    build.cargo_out(stage, host, Mode::Libstd, target).join("libstd_shim.rlib")
 }
 
 fn compiler_file(compiler: &Path, file: &str) -> String {
@@ -239,7 +236,8 @@ pub fn assemble_rustc(build: &Build, stage: u32, host: &str) {
         }
     }
 
-    let out_dir = build.cargo_out(stage - 1, &build.config.build, false, host);
+    let out_dir = build.cargo_out(stage - 1, &build.config.build,
+                                  Mode::Librustc, host);
 
     // Link the compiler binary itself into place
     let rustc = out_dir.join(exe("rustc", host));
@@ -298,3 +296,27 @@ fn add_to_sysroot(out_dir: &Path, sysroot_dst: &Path) {
                          sysroot_dst.join(path.file_name().unwrap())));
     }
 }
+
+/// Build a tool in `src/tools`
+///
+/// This will build the specified tool with the specified `host` compiler in
+/// `stage` into the normal cargo output directory.
+pub fn tool(build: &Build, stage: u32, host: &str, tool: &str) {
+    println!("Building stage{} tool {} ({})", stage, tool, host);
+
+    let compiler = Compiler::new(stage, host);
+
+    // FIXME: need to clear out previous tool and ideally deps, may require
+    //        isolating output directories or require a pseudo shim step to
+    //        clear out all the info.
+    //
+    //        Maybe when libstd is compiled it should clear out the rustc of the
+    //        corresponding stage?
+    // let out_dir = build.cargo_out(stage, &host, Mode::Librustc, target);
+    // build.clear_if_dirty(&out_dir, &libstd_shim(build, stage, &host, target));
+
+    let mut cargo = build.cargo(stage, &compiler, Mode::Tool, None, "build");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join(format!("src/tools/{}/Cargo.toml", tool)));
+    build.run(&mut cargo);
+}

src/bootstrap/build/doc.rs

@@ -8,12 +8,13 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
-use std::path::Path;
 use std::fs::{self, File};
 use std::io::prelude::*;
+use std::path::Path;
+use std::process::Command;
 
-use build::{Build, Compiler};
-use build::util::up_to_date;
+use build::{Build, Compiler, Mode};
+use build::util::{up_to_date, cp_r};
 
 pub fn rustbook(build: &Build, stage: u32, host: &str, name: &str, out: &Path) {
     t!(fs::create_dir_all(out));
@@ -69,7 +70,7 @@ pub fn standalone(build: &Build, stage: u32, host: &str, out: &Path) {
         }
 
         let html = out.join(filename).with_extension("html");
-        let rustdoc = build.tool(&compiler, "rustdoc");
+        let rustdoc = build.rustdoc(&compiler);
         if up_to_date(&path, &html) &&
            up_to_date(&footer, &html) &&
            up_to_date(&favicon, &html) &&
@@ -79,7 +80,7 @@ pub fn standalone(build: &Build, stage: u32, host: &str, out: &Path) {
             continue
         }
 
-        let mut cmd = build.tool_cmd(&compiler, "rustdoc");
+        let mut cmd = Command::new(&rustdoc);
         cmd.arg("--html-after-content").arg(&footer)
            .arg("--html-before-content").arg(&version_info)
            .arg("--html-in-header").arg(&favicon)
@@ -102,3 +103,52 @@ pub fn standalone(build: &Build, stage: u32, host: &str, out: &Path) {
         build.run(&mut cmd);
     }
 }
+
+pub fn std(build: &Build, stage: u32, host: &str, out: &Path) {
+    println!("Documenting stage{} std ({})", stage, host);
+    let compiler = Compiler::new(stage, host);
+    let out_dir = build.stage_out(stage, host, Mode::Libstd)
+                       .join(host).join("doc");
+    let rustdoc = build.rustdoc(&compiler);
+
+    build.clear_if_dirty(&out_dir, &rustdoc);
+
+    let mut cargo = build.cargo(stage, &compiler, Mode::Libstd, Some(host),
+                                "doc");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join("src/rustc/std_shim/Cargo.toml"))
+         .arg("--features").arg(build.std_features());
+    build.run(&mut cargo);
+    cp_r(&out_dir, out)
+}
+
+pub fn rustc(build: &Build, stage: u32, host: &str, out: &Path) {
+    println!("Documenting stage{} compiler ({})", stage, host);
+    let compiler = Compiler::new(stage, host);
+    let out_dir = build.stage_out(stage, host, Mode::Librustc)
+                       .join(host).join("doc");
+    let rustdoc = build.rustdoc(&compiler);
+    if !up_to_date(&rustdoc, &out_dir.join("rustc/index.html")) {
+        t!(fs::remove_dir_all(&out_dir));
+    }
+    let mut cargo = build.cargo(stage, &compiler, Mode::Librustc, Some(host),
+                                "doc");
+    cargo.arg("--manifest-path")
+         .arg(build.src.join("src/rustc/Cargo.toml"))
+         .arg("--features").arg(build.rustc_features());
+    build.run(&mut cargo);
+    cp_r(&out_dir, out)
+}
+
+pub fn error_index(build: &Build, stage: u32, host: &str, out: &Path) {
+    println!("Documenting stage{} error index ({})", stage, host);
+    let compiler = Compiler::new(stage, host);
+    let mut index = Command::new(build.tool(&compiler, "error_index_generator"));
+    index.arg("html");
+    index.arg(out.join("error-index.html"));
+
+    // FIXME: shouldn't have to pass this env var
+    index.env("CFG_BUILD", &build.config.build);
+
+    build.run(&mut index);
+}