rfc#3662 changes under unstable flags

* All new functionality is under unstable options
* Missing (will add later) functionality for --merge=finalize without a
  crate root
* Adds `--merge=shared|none|finalize` flags
* Adds `--parts-out-dir=<crate specific directory>` for `--merge=none`
to write cross-crate info file for a single crate
* Adds `--include-parts-dir=<previously specified directory>` for
`--merge=finalize` to write cross-crate info files
* update tests/run-make/rustdoc-default-output/rmake.rs golden

Author: EtomicBomb

Diff for: src/librustdoc/config.rs

@@ -289,6 +289,12 @@ pub(crate) struct RenderOptions {
     /// This field is only used for the JSON output. If it's set to true, no file will be created
     /// and content will be displayed in stdout directly.
     pub(crate) output_to_stdout: bool,
+    /// Whether we should read or write rendered cross-crate info in the doc root.
+    pub(crate) should_merge: ShouldMerge,
+    /// Path to crate-info for external crates.
+    pub(crate) include_parts_dir: Vec<PathToParts>,
+    /// Where to write crate-info
+    pub(crate) parts_out_dir: Option<PathToParts>,
 }
 
 #[derive(Copy, Clone, Debug, PartialEq, Eq)]
@@ -494,6 +500,16 @@ impl Options {
             Err(err) => dcx.fatal(err),
         };
 
+        let parts_out_dir =
+            match matches.opt_str("parts-out-dir").map(|p| PathToParts::from_flag(p)).transpose() {
+                Ok(parts_out_dir) => parts_out_dir,
+                Err(e) => dcx.fatal(e),
+            };
+        let include_parts_dir = match parse_include_parts_dir(matches) {
+            Ok(include_parts_dir) => include_parts_dir,
+            Err(e) => dcx.fatal(e),
+        };
+
         let default_settings: Vec<Vec<(String, String)>> = vec![
             matches
                 .opt_str("default-theme")
@@ -735,6 +751,10 @@ impl Options {
         let extern_html_root_takes_precedence =
             matches.opt_present("extern-html-root-takes-precedence");
         let html_no_source = matches.opt_present("html-no-source");
+        let should_merge = match parse_merge(matches) {
+            Ok(result) => result,
+            Err(e) => dcx.fatal(format!("--merge option error: {e}")),
+        };
 
         if generate_link_to_definition && (show_coverage || output_format != OutputFormat::Html) {
             dcx.struct_warn(
@@ -823,6 +843,9 @@ impl Options {
             no_emit_shared: false,
             html_no_source,
             output_to_stdout,
+            should_merge,
+            include_parts_dir,
+            parts_out_dir,
         };
         Some((options, render_options))
     }
@@ -900,3 +923,70 @@ fn parse_extern_html_roots(
     }
     Ok(externs)
 }
+
+/// Path directly to crate-info file.
+///
+/// For example, `/home/user/project/target/doc.parts/<crate>/crate-info`.
+#[derive(Clone, Debug)]
+pub(crate) struct PathToParts(pub(crate) PathBuf);
+
+impl PathToParts {
+    fn from_flag(path: String) -> Result<PathToParts, String> {
+        let mut path = PathBuf::from(path);
+        // check here is for diagnostics
+        if path.exists() && !path.is_dir() {
+            Err(format!(
+                "must provide a directory to --parts-out-dir and --include-parts-dir, got: {path:?}"
+            ))
+        } else {
+            // if it doesn't exist, we'll create it. worry about that in write_shared
+            path.push("crate-info");
+            Ok(PathToParts(path))
+        }
+    }
+}
+
+/// Reports error if --include-parts-dir / crate-info is not a file
+fn parse_include_parts_dir(m: &getopts::Matches) -> Result<Vec<PathToParts>, String> {
+    let mut ret = Vec::new();
+    for p in m.opt_strs("include-parts-dir") {
+        let p = PathToParts::from_flag(p)?;
+        // this is just for diagnostic
+        if !p.0.is_file() {
+            return Err(format!("--include-parts-dir expected {p:?} to be a file"));
+        }
+        ret.push(p);
+    }
+    Ok(ret)
+}
+
+/// Controls merging of cross-crate information
+#[derive(Debug, Clone)]
+pub(crate) struct ShouldMerge {
+    /// Should we append to existing cci in the doc root
+    pub(crate) read_rendered_cci: bool,
+    /// Should we write cci to the doc root
+    pub(crate) write_rendered_cci: bool,
+}
+
+/// Extracts read_rendered_cci and write_rendered_cci from command line arguments, or
+/// reports an error if an invalid option was provided
+fn parse_merge(m: &getopts::Matches) -> Result<ShouldMerge, &'static str> {
+    match m.opt_str("merge").as_deref() {
+        // default = read-write
+        None => Ok(ShouldMerge { read_rendered_cci: true, write_rendered_cci: true }),
+        Some("none") if m.opt_present("include-parts-dir") => {
+            Err("--include-parts-dir not allowed if --merge=none")
+        }
+        Some("none") => Ok(ShouldMerge { read_rendered_cci: false, write_rendered_cci: false }),
+        Some("shared") if m.opt_present("parts-out-dir") || m.opt_present("include-parts-dir") => {
+            Err("--parts-out-dir and --include-parts-dir not allowed if --merge=shared")
+        }
+        Some("shared") => Ok(ShouldMerge { read_rendered_cci: true, write_rendered_cci: true }),
+        Some("finalize") if m.opt_present("parts-out-dir") => {
+            Err("--parts-out-dir not allowed if --merge=finalize")
+        }
+        Some("finalize") => Ok(ShouldMerge { read_rendered_cci: false, write_rendered_cci: true }),
+        Some(_) => Err("argument to --merge must be `none`, `shared`, or `finalize`"),
+    }
+}

Diff for: src/librustdoc/html/render/context.rs

@@ -20,7 +20,7 @@ use super::{collect_spans_and_sources, scrape_examples_help, AllTypes, LinkFromS
 use crate::clean::types::ExternalLocation;
 use crate::clean::utils::has_doc_flag;
 use crate::clean::{self, ExternalCrate};
-use crate::config::{ModuleSorting, RenderOptions};
+use crate::config::{ModuleSorting, RenderOptions, ShouldMerge};
 use crate::docfs::{DocFS, PathError};
 use crate::error::Error;
 use crate::formats::cache::Cache;
@@ -127,8 +127,10 @@ pub(crate) struct SharedContext<'tcx> {
     pub(crate) span_correspondence_map: FxHashMap<rustc_span::Span, LinkFromSrc>,
     /// The [`Cache`] used during rendering.
     pub(crate) cache: Cache,
-
     pub(crate) call_locations: AllCallLocations,
+    /// Controls whether we read / write to cci files in the doc root. Defaults read=true,
+    /// write=true
+    should_merge: ShouldMerge,
 }
 
 impl SharedContext<'_> {
@@ -550,6 +552,7 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
             span_correspondence_map: matches,
             cache,
             call_locations,
+            should_merge: options.should_merge,
         };
 
         let dst = output;
@@ -637,92 +640,96 @@ impl<'tcx> FormatRenderer<'tcx> for Context<'tcx> {
         );
         shared.fs.write(final_file, v)?;
 
-        // Generating settings page.
-        page.title = "Settings";
-        page.description = "Settings of Rustdoc";
-        page.root_path = "./";
-        page.rust_logo = true;
+        // if to avoid writing help, settings files to doc root unless we're on the final invocation
+        if shared.should_merge.write_rendered_cci {
+            // Generating settings page.
+            page.title = "Settings";
+            page.description = "Settings of Rustdoc";
+            page.root_path = "./";
+            page.rust_logo = true;
 
-        let sidebar = "<h2 class=\"location\">Settings</h2><div class=\"sidebar-elems\"></div>";
-        let v = layout::render(
-            &shared.layout,
-            &page,
-            sidebar,
-            |buf: &mut Buffer| {
-                write!(
-                    buf,
-                    "<div class=\"main-heading\">\
-                     <h1>Rustdoc settings</h1>\
-                     <span class=\"out-of-band\">\
-                         <a id=\"back\" href=\"javascript:void(0)\" onclick=\"history.back();\">\
-                            Back\
-                        </a>\
-                     </span>\
-                     </div>\
-                     <noscript>\
-                        <section>\
-                            You need to enable JavaScript be able to update your settings.\
-                        </section>\
-                     </noscript>\
-                     <script defer src=\"{static_root_path}{settings_js}\"></script>",
-                    static_root_path = page.get_static_root_path(),
-                    settings_js = static_files::STATIC_FILES.settings_js,
-                );
-                // Pre-load all theme CSS files, so that switching feels seamless.
-                //
-                // When loading settings.html as a popover, the equivalent HTML is
-                // generated in main.js.
-                for file in &shared.style_files {
-                    if let Ok(theme) = file.basename() {
-                        write!(
-                            buf,
-                            "<link rel=\"preload\" href=\"{root_path}{theme}{suffix}.css\" \
-                                as=\"style\">",
-                            root_path = page.static_root_path.unwrap_or(""),
-                            suffix = page.resource_suffix,
-                        );
+            let sidebar = "<h2 class=\"location\">Settings</h2><div class=\"sidebar-elems\"></div>";
+            let v = layout::render(
+                &shared.layout,
+                &page,
+                sidebar,
+                |buf: &mut Buffer| {
+                    write!(
+                        buf,
+                        "<div class=\"main-heading\">\
+                         <h1>Rustdoc settings</h1>\
+                         <span class=\"out-of-band\">\
+                             <a id=\"back\" href=\"javascript:void(0)\" onclick=\"history.back();\">\
+                                Back\
+                            </a>\
+                         </span>\
+                         </div>\
+                         <noscript>\
+                            <section>\
+                                You need to enable JavaScript be able to update your settings.\
+                            </section>\
+                         </noscript>\
+                         <script defer src=\"{static_root_path}{settings_js}\"></script>",
+                        static_root_path = page.get_static_root_path(),
+                        settings_js = static_files::STATIC_FILES.settings_js,
+                    );
+                    // Pre-load all theme CSS files, so that switching feels seamless.
+                    //
+                    // When loading settings.html as a popover, the equivalent HTML is
+                    // generated in main.js.
+                    for file in &shared.style_files {
+                        if let Ok(theme) = file.basename() {
+                            write!(
+                                buf,
+                                "<link rel=\"preload\" href=\"{root_path}{theme}{suffix}.css\" \
+                                    as=\"style\">",
+                                root_path = page.static_root_path.unwrap_or(""),
+                                suffix = page.resource_suffix,
+                            );
+                        }
                     }
-                }
-            },
-            &shared.style_files,
-        );
-        shared.fs.write(settings_file, v)?;
+                },
+                &shared.style_files,
+            );
+            shared.fs.write(settings_file, v)?;
 
-        // Generating help page.
-        page.title = "Help";
-        page.description = "Documentation for Rustdoc";
-        page.root_path = "./";
-        page.rust_logo = true;
+            // Generating help page.
+            page.title = "Help";
+            page.description = "Documentation for Rustdoc";
+            page.root_path = "./";
+            page.rust_logo = true;
 
-        let sidebar = "<h2 class=\"location\">Help</h2><div class=\"sidebar-elems\"></div>";
-        let v = layout::render(
-            &shared.layout,
-            &page,
-            sidebar,
-            |buf: &mut Buffer| {
-                write!(
-                    buf,
-                    "<div class=\"main-heading\">\
-                     <h1>Rustdoc help</h1>\
-                     <span class=\"out-of-band\">\
-                         <a id=\"back\" href=\"javascript:void(0)\" onclick=\"history.back();\">\
-                            Back\
-                        </a>\
-                     </span>\
-                     </div>\
-                     <noscript>\
-                        <section>\
-                            <p>You need to enable JavaScript to use keyboard commands or search.</p>\
-                            <p>For more information, browse the <a href=\"https://doc.rust-lang.org/rustdoc/\">rustdoc handbook</a>.</p>\
-                        </section>\
-                     </noscript>",
-                )
-            },
-            &shared.style_files,
-        );
-        shared.fs.write(help_file, v)?;
+            let sidebar = "<h2 class=\"location\">Help</h2><div class=\"sidebar-elems\"></div>";
+            let v = layout::render(
+                &shared.layout,
+                &page,
+                sidebar,
+                |buf: &mut Buffer| {
+                    write!(
+                        buf,
+                        "<div class=\"main-heading\">\
+                         <h1>Rustdoc help</h1>\
+                         <span class=\"out-of-band\">\
+                             <a id=\"back\" href=\"javascript:void(0)\" onclick=\"history.back();\">\
+                                Back\
+                            </a>\
+                         </span>\
+                         </div>\
+                         <noscript>\
+                            <section>\
+                                <p>You need to enable JavaScript to use keyboard commands or search.</p>\
+                                <p>For more information, browse the <a href=\"https://doc.rust-lang.org/rustdoc/\">rustdoc handbook</a>.</p>\
+                            </section>\
+                         </noscript>",
+                    )
+                },
+                &shared.style_files,
+            );
+            shared.fs.write(help_file, v)?;
+        }
 
-        if shared.layout.scrape_examples_extension {
+        // if to avoid writing files to doc root unless we're on the final invocation
+        if shared.layout.scrape_examples_extension && shared.should_merge.write_rendered_cci {
             page.title = "About scraped examples";
             page.description = "How the scraped examples feature works in Rustdoc";
             let v = layout::render(