rustdoc: Fix generating documentation from json

src/compiletest/header.rs

@@ -39,6 +39,8 @@ pub struct TestProps {
     pub check_lines: Vec<String> ,
     // Build documentation for all specified aux-builds as well
     pub build_aux_docs: bool,
+    // Skip the check that emitting and building docs from json works correctly
+    pub skip_json_docs: bool,
     // Flag to force a crate to be built with the host architecture
     pub force_host: bool,
     // Check stdout for error-pattern output as well as stderr
@@ -66,6 +68,7 @@ pub fn load_props(testfile: &Path) -> TestProps {
     let pp_exact = None;
     let check_lines = Vec::new();
     let build_aux_docs = false;
+    let skip_json_docs = false;
     let force_host = false;
     let check_stdout = false;
     let no_prefer_dynamic = false;
@@ -83,6 +86,7 @@ pub fn load_props(testfile: &Path) -> TestProps {
         exec_env: exec_env,
         check_lines: check_lines,
         build_aux_docs: build_aux_docs,
+        skip_json_docs: skip_json_docs,
         force_host: force_host,
         check_stdout: check_stdout,
         no_prefer_dynamic: no_prefer_dynamic,
@@ -128,6 +132,10 @@ pub fn load_props_into(props: &mut TestProps, testfile: &Path, cfg: Option<&str>
             props.build_aux_docs = parse_build_aux_docs(ln);
         }
 
+        if !props.skip_json_docs {
+            props.skip_json_docs = parse_skip_json_docs(ln);
+        }
+
         if !props.force_host {
             props.force_host = parse_force_host(ln);
         }
@@ -359,6 +367,10 @@ fn parse_build_aux_docs(line: &str) -> bool {
     parse_name_directive(line, "build-aux-docs")
 }
 
+fn parse_skip_json_docs(line: &str) -> bool {
+    parse_name_directive(line, "skip-json-docs")
+}
+
 fn parse_check_stdout(line: &str) -> bool {
     parse_name_directive(line, "check-stdout")
 }

src/compiletest/runtest.rs

@@ -1274,13 +1274,21 @@ fn compile_test(config: &Config, props: &TestProps,
 fn document(config: &Config,
             props: &TestProps,
             testpaths: &TestPaths,
-            out_dir: &Path)
-            -> ProcRes {
+            out_dir: &Path,
+            json_trip: bool) -> ProcRes {
+    let run_rustdoc = |args: Vec<String>| -> ProcRes {
+        let args = ProcArgs {
+            prog: config.rustdoc_path.to_str().unwrap().to_owned(),
+            args: args.iter().map(|s|s.to_owned()).collect(),
+        };
+        compose_and_run_compiler(config, props, testpaths, args, None)
+    };
+
     if props.build_aux_docs {
         for rel_ab in &props.aux_builds {
             let aux_testpaths = compute_aux_test_paths(config, testpaths, rel_ab);
             let aux_props = header::load_props(&aux_testpaths.file);
-            let auxres = document(config, &aux_props, &aux_testpaths, out_dir);
+            let auxres = document(config, &aux_props, &aux_testpaths, out_dir, json_trip);
             if !auxres.status.success() {
                 return auxres;
             }
@@ -1290,15 +1298,31 @@ fn document(config: &Config,
     let aux_dir = aux_output_dir_name(config, testpaths);
     let mut args = vec!["-L".to_owned(),
                         aux_dir.to_str().unwrap().to_owned(),
-                        "-o".to_owned(),
-                        out_dir.to_str().unwrap().to_owned(),
                         testpaths.file.to_str().unwrap().to_owned()];
+    let mut out = PathBuf::from(out_dir);
+    if json_trip {
+        args.push("-w".to_owned());
+        args.push("json".to_owned());
+        out = out.join("doc.json");
+    }
+    args.push("-o".to_owned());
+    args.push(out.to_str().unwrap().to_owned());
     args.extend(props.compile_flags.iter().cloned());
-    let args = ProcArgs {
-        prog: config.rustdoc_path.to_str().unwrap().to_owned(),
-        args: args,
-    };
-    compose_and_run_compiler(config, props, testpaths, args, None)
+
+    let res = run_rustdoc(args);
+
+    if json_trip {
+        if !res.status.success() {
+            return res
+        }
+        run_rustdoc(vec!["-r".to_owned(),
+                         "json".to_owned(),
+                         "-o".to_owned(),
+                         out_dir.to_str().unwrap().to_owned(),
+                         out.to_str().unwrap().to_owned()])
+    } else {
+        res
+    }
 }
 
 fn exec_compiled_test(config: &Config, props: &TestProps,
@@ -1902,26 +1926,38 @@ fn charset() -> &'static str {
 
 fn run_rustdoc_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {
     assert!(props.revisions.is_empty(), "revisions not relevant here");
-
     let out_dir = output_base_name(config, testpaths);
-    let _ = fs::remove_dir_all(&out_dir);
-    ensure_dir(&out_dir);
-
-    let proc_res = document(config, props, testpaths, &out_dir);
-    if !proc_res.status.success() {
-        fatal_proc_rec(None, "rustdoc failed!", &proc_res);
-    }
     let root = find_rust_src_root(config).unwrap();
 
-    let res = cmd2procres(config,
-                          testpaths,
-                          Command::new(&config.python)
-                                  .arg(root.join("src/etc/htmldocck.py"))
-                                  .arg(out_dir)
-                                  .arg(&testpaths.file));
-    if !res.status.success() {
-        fatal_proc_rec(None, "htmldocck failed!", &res);
+    let check = |json_trip: bool| {
+        let _ = fs::remove_dir_all(&out_dir);
+        ensure_dir(&out_dir);
+
+        let proc_res = document(config, props, testpaths, &out_dir, json_trip);
+        if !proc_res.status.success() {
+            fatal_proc_rec(None, "rustdoc failed!", &proc_res);
+        }
+
+        let res = cmd2procres(config,
+                              testpaths,
+                              Command::new(&config.python)
+                                      .arg(root.join("src/etc/htmldocck.py"))
+                                      .arg(out_dir.clone())
+                                      .arg(&testpaths.file));
+        if !res.status.success() {
+            let msg = if json_trip {
+                "htmldocck failed! (documentation generated from json)"
+            } else {
+                "htmldocck failed!"
+            };
+            fatal_proc_rec(None, msg, &res);
+        }
+    };
+
+    if !props.skip_json_docs {
+        check(true);
     }
+    check(false);
 }
 
 fn run_codegen_units_test(config: &Config, props: &TestProps, testpaths: &TestPaths) {

src/librustdoc/clean/mod.rs

@@ -44,19 +44,22 @@ use rustc::middle::stability;
 
 use rustc_front::hir;
 
+use serialize::{Encoder, Decoder, Encodable, Decodable};
+
 use std::collections::{HashMap, HashSet};
 use std::path::PathBuf;
 use std::rc::Rc;
 use std::u32;
 use std::env::current_dir;
+use std::str::FromStr;
 
 use core::DocContext;
 use doctree;
 use visit_ast;
 
 /// A stable identifier to the particular version of JSON output.
 /// Increment this when the `Crate` and related structures change.
-pub const SCHEMA_VERSION: &'static str = "0.8.3";
+pub const SCHEMA_VERSION: &'static str = "0.8.4";
 
 mod inline;
 mod simplify;
@@ -116,14 +119,54 @@ impl<T: Clean<U>, U> Clean<Vec<U>> for P<[T]> {
     }
 }
 
+#[derive(Clone, Debug)]
+pub struct ExternalTraits {
+    pub map: HashMap<DefId, Trait>,
+}
+
+impl Encodable for ExternalTraits {
+    fn encode<S: Encoder>(&self, e: &mut S) -> Result<(), S::Error> {
+        e.emit_map(self.map.len(), |e| {
+            for (i, (did, val)) in self.map.iter().enumerate() {
+                let key = format!("{:?}@{:?}", did.krate, did.index.as_u32());
+                e.emit_map_elt_key(i, |e| key.encode(e))?;
+                e.emit_map_elt_val(i, |e| val.encode(e))?;
+            }
+            Ok(())
+        })
+    }
+}
+
+impl Decodable for ExternalTraits {
+    fn decode<D: Decoder>(d: &mut D) -> Result<ExternalTraits, D::Error> {
+        d.read_map(|d, len| {
+            let state = Default::default();
+            let mut map = HashMap::with_capacity_and_hasher(len, state);
+            for i in 0..len {
+                let key: String = d.read_map_elt_key(i, |d| Decodable::decode(d))?;
+                let val = d.read_map_elt_val(i, |d| Decodable::decode(d))?;
+
+                let mut fragments = key.split('@').map(|f| u32::from_str(f).unwrap());
+                let krate = fragments.next().unwrap();
+                let index = DefIndex::from_u32(fragments.next().unwrap());
+                assert!(fragments.next().is_none());
+
+                let did = DefId { krate: krate, index: index };
+                map.insert(did, val);
+            }
+            Ok(ExternalTraits { map: map })
+        })
+    }
+}
+
 #[derive(Clone, RustcEncodable, RustcDecodable, Debug)]
 pub struct Crate {
     pub name: String,
     pub src: PathBuf,
     pub module: Option<Item>,
     pub externs: Vec<(ast::CrateNum, ExternalCrate)>,
     pub primitives: Vec<PrimitiveType>,
-    pub external_traits: HashMap<DefId, Trait>,
+    pub external_traits: ExternalTraits,
 }
 
 struct CrateNum(ast::CrateNum);
@@ -214,8 +257,10 @@ impl<'a, 'tcx> Clean<Crate> for visit_ast::RustdocVisitor<'a, 'tcx> {
             module: Some(module),
             externs: externs,
             primitives: primitives,
-            external_traits: cx.external_traits.borrow_mut().take()
-                               .unwrap_or(HashMap::new()),
+            external_traits: ExternalTraits {
+                map: cx.external_traits.borrow_mut().take()
+                       .unwrap_or(HashMap::new()),
+            }
         }
     }
 }

src/librustdoc/fold.rs

@@ -73,7 +73,7 @@ pub trait DocFolder : Sized {
         c.module = c.module.and_then(|module| {
             self.fold_item(module)
         });
-        c.external_traits = c.external_traits.into_iter().map(|(k, mut v)| {
+        c.external_traits.map = c.external_traits.map.into_iter().map(|(k, mut v)| {
             v.items = v.items.into_iter().filter_map(|i| self.fold_item(i)).collect();
             (k, v)
         }).collect();

src/librustdoc/html/format.rs

@@ -298,7 +298,8 @@ pub fn href(did: DefId) -> Option<(String, ItemType, Vec<String>)> {
     let mut url = if did.is_local() || cache.inlined.contains(&did) {
         repeat("../").take(loc.len()).collect::<String>()
     } else {
-        match cache.extern_locations[&did.krate] {
+        match *cache.extern_locations.get(&did.krate)
+                                     .expect("extern location not found!") {
             (_, render::Remote(ref s)) => s.to_string(),
             (_, render::Local) => repeat("../").take(loc.len()).collect(),
             (_, render::Unknown) => return None,
@@ -384,27 +385,25 @@ fn primitive_link(f: &mut fmt::Formatter,
             needs_termination = true;
         }
         Some(&cnum) => {
-            let path = &m.paths[&DefId {
+            let path = m.paths.get(&DefId {
                 krate: cnum,
                 index: CRATE_DEF_INDEX,
-            }];
-            let loc = match m.extern_locations[&cnum] {
+            }).expect("path not found");
+            let loc = match *m.extern_locations.get(&cnum)
+                                               .expect("extern location not found!") {
                 (_, render::Remote(ref s)) => Some(s.to_string()),
                 (_, render::Local) => {
                     let len = CURRENT_LOCATION_KEY.with(|s| s.borrow().len());
                     Some(repeat("../").take(len).collect::<String>())
                 }
                 (_, render::Unknown) => None,
             };
-            match loc {
-                Some(root) => {
-                    write!(f, "<a class='primitive' href='{}{}/primitive.{}.html'>",
-                           root,
-                           path.0.first().unwrap(),
-                           prim.to_url_str())?;
-                    needs_termination = true;
-                }
-                None => {}
+            if let Some(root) = loc {
+                write!(f, "<a class='primitive' href='{}{}/primitive.{}.html'>",
+                       root,
+                       path.0.first().unwrap(),
+                       prim.to_url_str())?;
+                needs_termination = true;
             }
         }
         None => {}

src/librustdoc/html/render.rs

@@ -496,7 +496,7 @@ pub fn run(mut krate: clean::Crate,
         privmod: false,
         access_levels: access_levels,
         orphan_methods: Vec::new(),
-        traits: mem::replace(&mut krate.external_traits, HashMap::new()),
+        traits: mem::replace(&mut krate.external_traits.map, HashMap::new()),
         deref_trait_did: analysis.as_ref().and_then(|a| a.deref_trait_did),
         typarams: analysis.as_ref().map(|a| {
             a.external_typarams.borrow_mut().take().unwrap()
@@ -1477,17 +1477,19 @@ impl<'a> Item<'a> {
         // located, then we return `None`.
         } else {
             let cache = cache();
-            let path = &cache.external_paths[&self.item.def_id];
-            let root = match cache.extern_locations[&self.item.def_id.krate] {
+            let root = match *cache.extern_locations.get(&self.item.def_id.krate)
+                                                    .expect("extern location not found!") {
                 (_, Remote(ref s)) => s.to_string(),
                 (_, Local) => self.cx.root_path.clone(),
                 (_, Unknown) => return None,
             };
-            Some(format!("{root}{path}/{file}?gotosrc={goto}",
-                         root = root,
-                         path = path[..path.len() - 1].join("/"),
-                         file = item_path(self.item),
-                         goto = self.item.def_id.index.as_usize()))
+            cache.external_paths.get(&self.item.def_id).map(|path| {
+                format!("{root}{path}/{file}?gotosrc={goto}",
+                        root = root,
+                        path = path[..path.len() - 1].join("/"),
+                        file = item_path(self.item),
+                        goto = self.item.def_id.index.as_usize())
+            })
         }
     }
 }
@@ -2012,18 +2014,23 @@ fn item_trait(w: &mut fmt::Formatter, cx: &Context, it: &clean::Item,
         None => {}
     }
     write!(w, "</ul>")?;
-    write!(w, r#"<script type="text/javascript" async
-                         src="{root_path}/implementors/{path}/{ty}.{name}.js">
-                 </script>"#,
-           root_path = vec![".."; cx.current.len()].join("/"),
-           path = if it.def_id.is_local() {
-               cx.current.join("/")
-           } else {
-               let path = &cache.external_paths[&it.def_id];
-               path[..path.len() - 1].join("/")
-           },
-           ty = shortty(it).to_static_str(),
-           name = *it.name.as_ref().unwrap())?;
+
+    let path_opt = if it.def_id.is_local() {
+       Some(cx.current.join("/"))
+    } else {
+        cache.external_paths.get(&it.def_id).map(|path| {
+            path[..path.len() - 1].join("/")
+        })
+    };
+    if let Some(path) = path_opt {
+        write!(w, r#"<script type="text/javascript" async
+                             src="{root_path}/implementors/{path}/{ty}.{name}.js">
+                     </script>"#,
+               root_path = vec![".."; cx.current.len()].join("/"),
+               path = path,
+               ty = shortty(it).to_static_str(),
+               name = *it.name.as_ref().unwrap())?;
+    }
     Ok(())
 }
 

src/test/rustdoc/impl-parts-crosscrate.rs

@@ -8,6 +8,8 @@
 // option. This file may not be copied, modified, or distributed
 // except according to those terms.
 
+// skip-json-docs
+
 // aux-build:rustdoc-impl-parts-crosscrate.rs
 // ignore-cross-compile