[WIP] Add rustdoc GUI tests

.github/workflows/ci.yml

@@ -151,7 +151,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: dist-x86_64-linux
+          - name: x86_64-gnu-aux
             os: ubuntu-latest-xl
             env: {}
     timeout-minutes: 600

src/bootstrap/builder.rs

@@ -429,6 +429,7 @@ impl<'a> Builder<'a> {
                 test::RustdocJSNotStd,
                 test::RustdocTheme,
                 test::RustdocUi,
+                test::RustdocGUI,
                 // Run bootstrap close to the end as it's unlikely to fail
                 test::Bootstrap,
                 // Run run-make last, since these won't pass without make on Windows

src/bootstrap/mk/Makefile.in

@@ -45,6 +45,10 @@ check-aux:
 		src/tools/cargo \
 		src/tools/cargotest \
 		$(BOOTSTRAP_ARGS)
+check-aux-and-gui: check-aux
+	$(Q)$(BOOTSTRAP) test --stage 2 \
+	src/test/rustdoc-gui \
+	$(BOOTSTRAP_ARGS)
 check-bootstrap:
 	$(Q)$(CFG_PYTHON) $(CFG_SRC_DIR)src/bootstrap/bootstrap_test.py
 dist:

src/bootstrap/test.rs

@@ -716,6 +716,83 @@ impl Step for RustdocUi {
     }
 }
 
+#[derive(Debug, Copy, Clone, Hash, PartialEq, Eq)]
+pub struct RustdocGUI {
+    pub host: TargetSelection,
+    pub target: TargetSelection,
+    pub compiler: Compiler,
+}
+
+impl Step for RustdocGUI {
+    type Output = ();
+    const ONLY_HOSTS: bool = true;
+
+    fn should_run(run: ShouldRun<'_>) -> ShouldRun<'_> {
+        run.path("src/test/rustdoc-gui")
+    }
+
+    fn make_run(run: RunConfig<'_>) {
+        let compiler = run.builder.compiler(run.builder.top_stage, run.host);
+        run.builder.ensure(RustdocGUI { host: run.host, target: run.target, compiler });
+    }
+
+    fn run(self, builder: &Builder<'_>) {
+        if let Some(ref nodejs) = builder.config.nodejs {
+            builder.ensure(compile::Std { target: self.target, compiler: self.compiler });
+
+            // First step: cloning repositories.
+            util::clone_repository(
+                "https://github.com/GuillaumeGomez/browser-UI-test",
+                &builder.out.join("browser-UI-test"),
+                Some("e47b1a8697f628429cb684e0b3716737a3a0fa78"),
+            );
+            util::clone_repository(
+                "https://github.com/GuillaumeGomez/test-rust-docs-ui",
+                &builder.out.join("test-rust-docs-ui"),
+                Some("d74abb60d1bd1e3fbf647d2d2eb521f2aa357f27"),
+            );
+            // Second step: install npm dependencies.
+            let mut cmd = Command::new("npm");
+            cmd.arg("install").current_dir(builder.out.join("browser-UI-test"));
+            if !try_run(builder, &mut cmd) {
+                panic!("failed to install browser-UI-test node package");
+            }
+
+            // Third step: building documentation with lastest rustdoc version.
+            let mut cmd = builder.rustdoc_cmd(self.compiler);
+            cmd.current_dir(builder.out.join("test-rust-docs-ui/test-docs")).arg("src/lib.rs");
+            if !try_run(builder, &mut cmd) {
+                panic!("Failed to build docs for rustdoc-gui test!");
+            }
+
+            // Last step: running tests.
+            let mut command = Command::new(nodejs);
+            command
+                .arg("../browser-UI-test/src/index.js")
+                .arg("--no-sandbox")
+                .arg("--test-folder")
+                .arg("ui-tests")
+                .arg("--failure-folder")
+                .arg("failures")
+                .arg("--variable")
+                .arg("DOC_PATH")
+                .arg(
+                    builder
+                        .out
+                        .canonicalize()
+                        .unwrap()
+                        .join("test-rust-docs-ui/test-docs/doc/test_docs"),
+                )
+                .arg("--show-text")
+                .arg("--generate-images")
+                .current_dir(builder.out.join("test-rust-docs-ui"));
+            builder.run(&mut command);
+        } else {
+            builder.info("No nodejs found, skipping \"src/test/rustdoc-gui\" tests");
+        }
+    }
+}
+
 #[derive(Debug, Copy, Clone, PartialEq, Eq, Hash)]
 pub struct Tidy;
 

src/bootstrap/util.rs

@@ -7,7 +7,7 @@ use std::env;
 use std::fs;
 use std::io;
 use std::path::{Path, PathBuf};
-use std::process::Command;
+use std::process::{Command, ExitStatus};
 use std::str;
 use std::time::Instant;
 
@@ -302,3 +302,61 @@ pub fn use_host_linker(target: TargetSelection) -> bool {
         || target.contains("fortanix")
         || target.contains("fuchsia"))
 }
+
+fn check_command_status(cmd_status: io::Result<ExitStatus>, error_msg: &str) {
+    let success = match cmd_status {
+        Ok(s) => s.success(),
+        Err(_) => false,
+    };
+    if !success {
+        panic!("{} unsuccessful (status: {:?})", error_msg, cmd_status);
+    }
+}
+
+pub fn clone_repository(
+    repository_url: &str,
+    target_dir: &Path,
+    specific_commit_hash: Option<&str>,
+) {
+    if target_dir.is_dir() {
+        // First fetch to have latest remote repository data.
+        check_command_status(
+            Command::new("git")
+                .arg("fetch")
+                .arg("origin")
+                .current_dir(target_dir.to_str().unwrap())
+                .status(),
+            "git fetch",
+        );
+        // In case there is no specific commit hash, we need to get the last version.
+        if specific_commit_hash.is_none() {
+            check_command_status(
+                Command::new("git")
+                    .arg("checkout")
+                    .arg("origin/master")
+                    .current_dir(target_dir.to_str().unwrap())
+                    .status(),
+                "git checkout on origin/master",
+            );
+        }
+    } else {
+        check_command_status(
+            Command::new("git")
+                .arg("clone")
+                .arg(repository_url)
+                .arg(target_dir.as_os_str())
+                .status(),
+            "git clone",
+        );
+    }
+    if let Some(specific_commit_hash) = specific_commit_hash {
+        check_command_status(
+            Command::new("git")
+                .arg("checkout")
+                .arg(specific_commit_hash)
+                .current_dir(target_dir.to_str().unwrap())
+                .status(),
+            "git checkout",
+        );
+    }
+}

src/ci/docker/host-x86_64/x86_64-gnu-aux/Dockerfile

@@ -16,10 +16,44 @@ RUN apt-get update && apt-get install -y --no-install-recommends \
   libgl1-mesa-dev \
   llvm-dev \
   libfreetype6-dev \
-  libexpat1-dev
+  libexpat1-dev \
+  gnupg \
+  apt-utils \
+  wget \
+  fonts-ipafont-gothic \
+  fonts-wqy-zenhei \
+  fonts-thai-tlwg \
+  fonts-kacst \
+  fonts-freefont-ttf \
+  libxss1 \
+  libxtst6
+
+RUN curl -sL https://deb.nodesource.com/setup_12.x | sudo -E bash -
+RUN apt install -y nodejs
+
+# This part is to install the requirements to run rustdoc GUI tests
+#
+# To do so we need:
+#  * a chrome instance (we can extend it to firefox)
+#  * cloning the browser-UI-test repository from https://github.com/GuillaumeGomez/browser-UI-test/
+#  * cloning the test-rust-docs-ui repository from https://github.com/GuillaumeGomez/test-rust-docs-ui
+#
+# The browser-UI-test repository is a framework to make it as easy as possible to write GUI tests
+# where the test-rust-docs-ui repository contains the tests specific to rustdoc GUI.
+#
+# NOte that the two repositories clone part is handled by bootstrap.
+#
+# To prevent unwanted changes breaking tests for everyone, both repositories will target a specfic
+# commit that will need to be updated when the rustdoc GUI will be updated.
+RUN wget -q -O - https://dl-ssl.google.com/linux/linux_signing_key.pub | apt-key add - \
+    && sh -c 'echo "deb [arch=amd64] http://dl.google.com/linux/chrome/deb/ stable main" >> /etc/apt/sources.list.d/google.list'
+
+RUN apt-get update && apt-get install -y --no-install-recommends \
+  google-chrome-unstable \
+  && rm -rf /var/lib/apt/lists/*
 
 COPY scripts/sccache.sh /scripts/
 RUN sh /scripts/sccache.sh
 
 ENV RUST_CONFIGURE_ARGS --build=x86_64-unknown-linux-gnu
-ENV RUST_CHECK_TARGET check-aux
+ENV RUST_CHECK_TARGET check-aux-and-gui

src/ci/github-actions/ci.yml

@@ -293,7 +293,7 @@ jobs:
     strategy:
       matrix:
         include:
-          - name: dist-x86_64-linux
+          - name: x86_64-gnu-aux
             <<: *job-linux-xl
 
   auto: