rustdoc: Implement cross-crate searching

A major discoverability issue with rustdoc is that all crates have their
documentation built in isolation, so it's difficult when looking at the
documentation for libstd to learn that there's a libcollections crate with a
HashMap in it.

This commit moves rustdoc a little closer to improving the multiple crate
experience. This unifies all search indexes for all crates into one file so all
pages share the same search index. This allows searching to work across crates
in the same documentation directory (as the standard distribution is currently
built).

This strategy involves updating a shared file amongst many rustdoc processes, so
I implemented a simple file locking API for handling synchronization for updates
to the shared files.

cc #12554

Author: alexcrichton

src/librustdoc/flock.rs

@@ -0,0 +1,191 @@
+// Copyright 2014 The Rust Project Developers. See the COPYRIGHT
+// file at the top-level directory of this distribution and at
+// http://rust-lang.org/COPYRIGHT.
+//
+// Licensed under the Apache License, Version 2.0 <LICENSE-APACHE or
+// http://www.apache.org/licenses/LICENSE-2.0> or the MIT license
+// <LICENSE-MIT or http://opensource.org/licenses/MIT>, at your
+// option. This file may not be copied, modified, or distributed
+// except according to those terms.
+
+//! Simple file-locking apis for each OS.
+//!
+//! This is not meant to be in the standard library, it does nothing with
+//! green/native threading. This is just a bare-bones enough solution for
+//! librustdoc, it is not production quality at all.
+
+#[allow(non_camel_case_types)];
+
+pub use self::imp::Lock;
+
+#[cfg(unix)]
+mod imp {
+    use std::libc;
+
+    #[cfg(target_os = "linux")]
+    mod os {
+        use std::libc;
+
+        pub struct flock {
+            l_type: libc::c_short,
+            l_whence: libc::c_short,
+            l_start: libc::off_t,
+            l_len: libc::off_t,
+            l_pid: libc::pid_t,
+
+            // not actually here, but brings in line with freebsd
+            l_sysid: libc::c_int,
+        }
+
+        pub static F_WRLCK: libc::c_short = 1;
+        pub static F_UNLCK: libc::c_short = 2;
+        pub static F_SETLK: libc::c_int = 6;
+        pub static F_SETLKW: libc::c_int = 7;
+    }
+
+    #[cfg(target_os = "freebsd")]
+    mod os {
+        use std::libc;
+
+        pub struct flock {
+            l_start: libc::off_t,
+            l_len: libc::off_t,
+            l_pid: libc::pid_t,
+            l_type: libc::c_short,
+            l_whence: libc::c_short,
+            l_sysid: libc::c_int,
+        }
+
+        pub static F_UNLCK: libc::c_short = 2;
+        pub static F_WRLCK: libc::c_short = 3;
+        pub static F_SETLK: libc::c_int = 12;
+        pub static F_SETLKW: libc::c_int = 13;
+    }
+
+    #[cfg(target_os = "macos")]
+    mod os {
+        use std::libc;
+
+        pub struct flock {
+            l_start: libc::off_t,
+            l_len: libc::off_t,
+            l_pid: libc::pid_t,
+            l_type: libc::c_short,
+            l_whence: libc::c_short,
+
+            // not actually here, but brings in line with freebsd
+            l_sysid: libc::c_int,
+        }
+
+        pub static F_UNLCK: libc::c_short = 2;
+        pub static F_WRLCK: libc::c_short = 3;
+        pub static F_SETLK: libc::c_int = 8;
+        pub static F_SETLKW: libc::c_int = 9;
+    }
+
+    pub struct Lock {
+        priv fd: libc::c_int,
+    }
+
+    impl Lock {
+        pub fn new(p: &Path) -> Lock {
+            let fd = p.with_c_str(|s| unsafe {
+                libc::open(s, libc::O_RDWR | libc::O_CREAT, libc::S_IRWXU)
+            });
+            assert!(fd > 0);
+            let flock = os::flock {
+                l_start: 0,
+                l_len: 0,
+                l_pid: 0,
+                l_whence: libc::SEEK_SET as libc::c_short,
+                l_type: os::F_WRLCK,
+                l_sysid: 0,
+            };
+            let ret = unsafe {
+                libc::fcntl(fd, os::F_SETLKW, &flock as *os::flock)
+            };
+            if ret == -1 {
+                unsafe { libc::close(fd); }
+                fail!("could not lock `{}`", p.display())
+            }
+            Lock { fd: fd }
+        }
+    }
+
+    impl Drop for Lock {
+        fn drop(&mut self) {
+            let flock = os::flock {
+                l_start: 0,
+                l_len: 0,
+                l_pid: 0,
+                l_whence: libc::SEEK_SET as libc::c_short,
+                l_type: os::F_UNLCK,
+                l_sysid: 0,
+            };
+            unsafe {
+                libc::fcntl(self.fd, os::F_SETLK, &flock as *os::flock);
+                libc::close(self.fd);
+            }
+        }
+    }
+}
+
+#[cfg(windows)]
+mod imp {
+    use std::libc;
+    use std::mem;
+    use std::os::win32::as_utf16_p;
+    use std::ptr;
+
+    static LOCKFILE_EXCLUSIVE_LOCK: libc::DWORD = 0x00000002;
+
+    extern "system" {
+        fn LockFileEx(hFile: libc::HANDLE,
+                      dwFlags: libc::DWORD,
+                      dwReserved: libc::DWORD,
+                      nNumberOfBytesToLockLow: libc::DWORD,
+                      nNumberOfBytesToLockHigh: libc::DWORD,
+                      lpOverlapped: libc::LPOVERLAPPED) -> libc::BOOL;
+        fn UnlockFileEx(hFile: libc::HANDLE,
+                        dwReserved: libc::DWORD,
+                        nNumberOfBytesToLockLow: libc::DWORD,
+                        nNumberOfBytesToLockHigh: libc::DWORD,
+                        lpOverlapped: libc::LPOVERLAPPED) -> libc::BOOL;
+    }
+
+    pub struct Lock {
+        priv handle: libc::HANDLE,
+    }
+
+    impl Lock {
+        pub fn new(p: &Path) -> Lock {
+            let handle = as_utf16_p(p.as_str().unwrap(), |p| unsafe {
+                libc::CreateFileW(p, libc::GENERIC_READ, 0, ptr::mut_null(),
+                                  libc::CREATE_ALWAYS,
+                                  libc::FILE_ATTRIBUTE_NORMAL,
+                                  ptr::mut_null())
+            });
+            assert!(handle as uint != libc::INVALID_HANDLE_VALUE as uint);
+            let mut overlapped: libc::OVERLAPPED = unsafe { mem::init() };
+            let ret = unsafe {
+                LockFileEx(handle, LOCKFILE_EXCLUSIVE_LOCK, 0, 100, 0,
+                           &mut overlapped)
+            };
+            if ret == 0 {
+                unsafe { libc::CloseHandle(handle); }
+                fail!("could not lock `{}`", p.display())
+            }
+            Lock { handle: handle }
+        }
+    }
+
+    impl Drop for Lock {
+        fn drop(&mut self) {
+            let mut overlapped: libc::OVERLAPPED = unsafe { mem::init() };
+            unsafe {
+                UnlockFileEx(self.handle, 0, 100, 0, &mut overlapped);
+                libc::CloseHandle(self.handle);
+            }
+        }
+    }
+}

src/librustdoc/html/layout.rs

@@ -37,7 +37,7 @@ pub fn render<T: fmt::Show, S: fmt::Show>(
 
     <link href='http://fonts.googleapis.com/css?family=Oswald:700|Inconsolata:400,700'
           rel='stylesheet' type='text/css'>
-    <link rel=\"stylesheet\" type=\"text/css\" href=\"{root_path}{krate}/main.css\">
+    <link rel=\"stylesheet\" type=\"text/css\" href=\"{root_path}main.css\">
 
     {favicon, select, none{} other{<link rel=\"shortcut icon\" href=\"#\" />}}
 </head>
@@ -74,13 +74,6 @@ pub fn render<T: fmt::Show, S: fmt::Show>(
 
     <section class=\"footer\"></section>
 
-    <script>
-        var rootPath = \"{root_path}\";
-    </script>
-    <script src=\"{root_path}{krate}/jquery.js\"></script>
-    <script src=\"{root_path}{krate}/search-index.js\"></script>
-    <script src=\"{root_path}{krate}/main.js\"></script>
-
     <div id=\"help\" class=\"hidden\">
         <div class=\"shortcuts\">
             <h1>Keyboard shortcuts</h1>
@@ -111,6 +104,14 @@ pub fn render<T: fmt::Show, S: fmt::Show>(
             </p>
         </div>
     </div>
+
+    <script>
+        var rootPath = \"{root_path}\";
+        var currentCrate = \"{krate}\";
+    </script>
+    <script src=\"{root_path}jquery.js\"></script>
+    <script src=\"{root_path}main.js\"></script>
+    <script async src=\"{root_path}search-index.js\"></script>
 </body>
 </html>
 ",

src/librustdoc/html/render.rs

@@ -36,7 +36,7 @@
 use std::fmt;
 use std::local_data;
 use std::io;
-use std::io::{fs, File, BufferedWriter};
+use std::io::{fs, File, BufferedWriter, MemWriter, BufferedReader};
 use std::str;
 use std::vec;
 use std::vec_ng::Vec;
@@ -283,48 +283,75 @@ pub fn run(mut krate: clean::Crate, dst: Path) -> io::IoResult<()> {
         };
     }
 
-    // Add all the static files
-    let mut dst = cx.dst.join(krate.name.as_slice());
-    try!(mkdir(&dst));
-    try!(write(dst.join("jquery.js"),
-                 include_str!("static/jquery-2.1.0.min.js")));
-    try!(write(dst.join("main.js"), include_str!("static/main.js")));
-    try!(write(dst.join("main.css"), include_str!("static/main.css")));
-    try!(write(dst.join("normalize.css"),
-                 include_str!("static/normalize.css")));
-
     // Publish the search index
-    {
-        dst.push("search-index.js");
-        let mut w = BufferedWriter::new(File::create(&dst).unwrap());
-        let w = &mut w as &mut Writer;
-        try!(write!(w, "var searchIndex = ["));
+    let index = {
+        let mut w = MemWriter::new();
+        try!(write!(&mut w, "searchIndex['{}'] = [", krate.name));
         for (i, item) in cache.search_index.iter().enumerate() {
             if i > 0 {
-                try!(write!(w, ","));
+                try!(write!(&mut w, ","));
             }
-            try!(write!(w, "\\{ty:\"{}\",name:\"{}\",path:\"{}\",desc:{}",
-                          item.ty, item.name, item.path,
-                          item.desc.to_json().to_str()));
+            try!(write!(&mut w, "\\{ty:\"{}\",name:\"{}\",path:\"{}\",desc:{}",
+                        item.ty, item.name, item.path,
+                        item.desc.to_json().to_str()));
             match item.parent {
                 Some(id) => {
-                    try!(write!(w, ",parent:'{}'", id));
+                    try!(write!(&mut w, ",parent:'{}'", id));
                 }
                 None => {}
             }
-            try!(write!(w, "\\}"));
+            try!(write!(&mut w, "\\}"));
         }
-        try!(write!(w, "];"));
-        try!(write!(w, "var allPaths = \\{"));
+        try!(write!(&mut w, "];"));
+        try!(write!(&mut w, "allPaths['{}'] = \\{", krate.name));
         for (i, (&id, &(ref fqp, short))) in cache.paths.iter().enumerate() {
             if i > 0 {
-                try!(write!(w, ","));
+                try!(write!(&mut w, ","));
             }
-            try!(write!(w, "'{}':\\{type:'{}',name:'{}'\\}",
-                          id, short, *fqp.last().unwrap()));
+            try!(write!(&mut w, "'{}':\\{type:'{}',name:'{}'\\}",
+                        id, short, *fqp.last().unwrap()));
         }
-        try!(write!(w, "\\};"));
-        try!(w.flush());
+        try!(write!(&mut w, "\\};"));
+
+        str::from_utf8_owned(w.unwrap()).unwrap()
+    };
+
+    // Write out the shared files. Note that these are shared among all rustdoc
+    // docs placed in the output directory, so this needs to be a synchronized
+    // operation with respect to all other rustdocs running around.
+    {
+        try!(mkdir(&cx.dst));
+        let _lock = ::flock::Lock::new(&cx.dst.join(".lock"));
+
+        // Add all the static files. These may already exist, but we just
+        // overwrite them anyway to make sure that they're fresh and up-to-date.
+        try!(write(cx.dst.join("jquery.js"),
+                   include_str!("static/jquery-2.1.0.min.js")));
+        try!(write(cx.dst.join("main.js"), include_str!("static/main.js")));
+        try!(write(cx.dst.join("main.css"), include_str!("static/main.css")));
+        try!(write(cx.dst.join("normalize.css"),
+                   include_str!("static/normalize.css")));
+
+        // Update the search index
+        let dst = cx.dst.join("search-index.js");
+        let mut all_indexes = Vec::new();
+        all_indexes.push(index);
+        if dst.exists() {
+            for line in BufferedReader::new(File::open(&dst)).lines() {
+                let line = try!(line);
+                if !line.starts_with("searchIndex") { continue }
+                if line.starts_with(format!("searchIndex['{}']", krate.name)) {
+                    continue
+                }
+                all_indexes.push(line);
+            }
+        }
+        let mut w = try!(File::create(&dst));
+        try!(writeln!(&mut w, r"var searchIndex = \{\}; var allPaths = \{\};"));
+        for index in all_indexes.iter() {
+            try!(writeln!(&mut w, "{}", *index));
+        }
+        try!(writeln!(&mut w, "initSearch(searchIndex);"));
     }
 
     // Render all source files (this may turn into a giant no-op)