rustbuild: Tweak for vendored dependencies

A few changes are included here:

* The `winapi` and `url` dependencies were dropped. The source code for these
  projects is pretty weighty, and we're about to vendor them, so let's not
  commit to that intake just yet. If necessary we can vendor them later but for
  now it shouldn't be necessary.

* The `--frozen` flag is now always passed to Cargo, obviating the need for
  tidy's `cargo_lock` check.

* Tidy was updated to not check the vendor directory

Closes rust-lang#34687

.gitattributes

@@ -7,3 +7,4 @@
 src/etc/pkg/rust-logo.ico binary
 src/etc/pkg/rust-logo.png binary
 *.woff binary
+src/vendor/* binary

.gitignore

@@ -98,3 +98,4 @@ tmp.*.rs
 version.md
 version.ml
 version.texi
+.cargo

.travis.yml

@@ -15,7 +15,7 @@ before_install:
 script:
   - docker run -v `pwd`:/build rust
     sh -c "
-      ./configure --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
+      ./configure --enable-vendor --enable-rustbuild --llvm-root=/usr/lib/llvm-3.7 --enable-quiet-tests &&
       make tidy &&
       make check -j4
     "

configure

@@ -634,6 +634,7 @@ opt rustbuild 0 "use the rust and cargo based build system"
 opt codegen-tests 1 "run the src/test/codegen tests"
 opt option-checking 1 "complain about unrecognized options in this configure script"
 opt ninja 0 "build LLVM using the Ninja generator (for MSVC, requires building in the correct environment)"
+opt vendor 0 "enable usage of vendored Rust crates"
 
 # Optimization and debugging options. These may be overridden by the release channel, etc.
 opt_nosave optimize 1 "build optimized rust code"

mk/dist.mk

@@ -65,7 +65,8 @@ PKG_FILES := \
       stage0.txt                               \
       rust-installer                           \
       tools                                    \
-      test)                                    \
+      test                                     \
+      vendor)                                  \
     $(PKG_GITMODULES)                          \
     $(filter-out config.stamp, \
                  $(MKFILES_FOR_TARBALL))

src/bootstrap/Cargo.toml

@@ -27,10 +27,6 @@ num_cpus = "0.2"
 toml = "0.1"
 getopts = "0.2"
 rustc-serialize = "0.3"
-gcc = "0.3.36"
+gcc = "0.3.38"
 libc = "0.2"
 md5 = "0.1"
-
-[target.'cfg(windows)'.dependencies]
-winapi = "0.2"
-kernel32-sys = "0.2"

src/bootstrap/bootstrap.py

@@ -259,9 +259,11 @@ def build_bootstrap(self):
         env["DYLD_LIBRARY_PATH"] = os.path.join(self.bin_root(), "lib")
         env["PATH"] = os.path.join(self.bin_root(), "bin") + \
                       os.pathsep + env["PATH"]
-        self.run([self.cargo(), "build", "--manifest-path",
-                  os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")],
-                 env)
+        args = [self.cargo(), "build", "--manifest-path",
+                os.path.join(self.rust_root, "src/bootstrap/Cargo.toml")]
+        if self.use_vendored_sources:
+            args.append("--frozen")
+        self.run(args, env)
 
     def run(self, args, env):
         proc = subprocess.Popen(args, env=env)
@@ -384,6 +386,25 @@ def main():
     except:
         pass
 
+    rb.use_vendored_sources = '\nvendor = true' in rb.config_toml or \
+                              'CFG_ENABLE_VENDOR' in rb.config_mk
+
+    if rb.use_vendored_sources:
+        if not os.path.exists('.cargo'):
+            os.makedirs('.cargo')
+        f = open('.cargo/config','w')
+        f.write("""
+            [source.crates-io]
+            replace-with = 'vendored-sources'
+            registry = 'https://example.com'
+
+            [source.vendored-sources]
+            directory = '{}/src/vendor'
+        """.format(rb.rust_root))
+        f.close()
+    else:
+        if os.path.exists('.cargo'):
+            shutil.rmtree('.cargo')
     data = stage0_data(rb.rust_root)
     rb._rustc_channel, rb._rustc_date = data['rustc'].split('-', 1)
     rb._cargo_channel, rb._cargo_date = data['cargo'].split('-', 1)

src/bootstrap/config.rs

@@ -44,6 +44,7 @@ pub struct Config {
     pub submodules: bool,
     pub compiler_docs: bool,
     pub docs: bool,
+    pub vendor: bool,
     pub target_config: HashMap<String, Target>,
 
     // llvm codegen options
@@ -126,6 +127,7 @@ struct Build {
     docs: Option<bool>,
     submodules: Option<bool>,
     gdb: Option<String>,
+    vendor: Option<bool>,
 }
 
 /// TOML representation of how the LLVM build is configured.
@@ -234,6 +236,7 @@ impl Config {
         set(&mut config.compiler_docs, build.compiler_docs);
         set(&mut config.docs, build.docs);
         set(&mut config.submodules, build.submodules);
+        set(&mut config.vendor, build.vendor);
 
         if let Some(ref llvm) = toml.llvm {
             set(&mut config.ccache, llvm.ccache);
@@ -347,6 +350,7 @@ impl Config {
                 ("LOCAL_REBUILD", self.local_rebuild),
                 ("NINJA", self.ninja),
                 ("CODEGEN_TESTS", self.codegen_tests),
+                ("VENDOR", self.vendor),
             }
 
             match key {

src/bootstrap/config.toml.example

@@ -82,6 +82,9 @@
 # The path to (or name of) the GDB executable to use
 #gdb = "gdb"
 
+# Indicate whether the vendored sources are used for Rust dependencies or not
+#vendor = false
+
 # =============================================================================
 # Options for compiling Rust code itself
 # =============================================================================

src/bootstrap/job.rs

@@ -37,15 +37,82 @@
 //! Note that this module has a #[cfg(windows)] above it as none of this logic
 //! is required on Unix.
 
-extern crate kernel32;
-extern crate winapi;
+#![allow(bad_style, dead_code)]
 
 use std::env;
 use std::io;
 use std::mem;
 
-use self::winapi::*;
-use self::kernel32::*;
+type HANDLE = *mut u8;
+type BOOL = i32;
+type DWORD = u32;
+type LPHANDLE = *mut HANDLE;
+type LPVOID = *mut u8;
+type JOBOBJECTINFOCLASS = i32;
+type SIZE_T = usize;
+type LARGE_INTEGER = i64;
+type ULONG_PTR = usize;
+type ULONGLONG = u64;
+
+const FALSE: BOOL = 0;
+const DUPLICATE_SAME_ACCESS: DWORD = 0x2;
+const PROCESS_DUP_HANDLE: DWORD = 0x40;
+const JobObjectExtendedLimitInformation: JOBOBJECTINFOCLASS = 9;
+const JOB_OBJECT_LIMIT_KILL_ON_JOB_CLOSE: DWORD = 0x2000;
+
+extern "system" {
+    fn CreateJobObjectW(lpJobAttributes: *mut u8, lpName: *const u8) -> HANDLE;
+    fn CloseHandle(hObject: HANDLE) -> BOOL;
+    fn GetCurrentProcess() -> HANDLE;
+    fn OpenProcess(dwDesiredAccess: DWORD,
+                   bInheritHandle: BOOL,
+                   dwProcessId: DWORD) -> HANDLE;
+    fn DuplicateHandle(hSourceProcessHandle: HANDLE,
+                       hSourceHandle: HANDLE,
+                       hTargetProcessHandle: HANDLE,
+                       lpTargetHandle: LPHANDLE,
+                       dwDesiredAccess: DWORD,
+                       bInheritHandle: BOOL,
+                       dwOptions: DWORD) -> BOOL;
+    fn AssignProcessToJobObject(hJob: HANDLE, hProcess: HANDLE) -> BOOL;
+    fn SetInformationJobObject(hJob: HANDLE,
+                               JobObjectInformationClass: JOBOBJECTINFOCLASS,
+                               lpJobObjectInformation: LPVOID,
+                               cbJobObjectInformationLength: DWORD) -> BOOL;
+}
+
+#[repr(C)]
+struct JOBOBJECT_EXTENDED_LIMIT_INFORMATION {
+    BasicLimitInformation: JOBOBJECT_BASIC_LIMIT_INFORMATION,
+    IoInfo: IO_COUNTERS,
+    ProcessMemoryLimit: SIZE_T,
+    JobMemoryLimit: SIZE_T,
+    PeakProcessMemoryUsed: SIZE_T,
+    PeakJobMemoryUsed: SIZE_T,
+}
+
+#[repr(C)]
+struct IO_COUNTERS {
+    ReadOperationCount: ULONGLONG,
+    WriteOperationCount: ULONGLONG,
+    OtherOperationCount: ULONGLONG,
+    ReadTransferCount: ULONGLONG,
+    WriteTransferCount: ULONGLONG,
+    OtherTransferCount: ULONGLONG,
+}
+
+#[repr(C)]
+struct JOBOBJECT_BASIC_LIMIT_INFORMATION {
+    PerProcessUserTimeLimit: LARGE_INTEGER,
+    PerJobUserTimeLimit: LARGE_INTEGER,
+    LimitFlags: DWORD,
+    MinimumWorkingsetSize: SIZE_T,
+    MaximumWorkingsetSize: SIZE_T,
+    ActiveProcessLimit: DWORD,
+    Affinity: ULONG_PTR,
+    PriorityClass: DWORD,
+    SchedulingClass: DWORD,
+}
 
 pub unsafe fn setup() {
     // Create a new job object for us to use

src/bootstrap/lib.rs

@@ -460,6 +460,9 @@ impl Build {
         if self.config.rust_optimize {
             cargo.arg("--release");
         }
+        if self.config.vendor {
+            cargo.arg("--frozen");
+        }
         return cargo
     }
 

src/tools/linkchecker/Cargo.toml

@@ -3,9 +3,6 @@ name = "linkchecker"
 version = "0.1.0"
 authors = ["Alex Crichton <alex@alexcrichton.com>"]
 
-[dependencies]
-url = "1.2"
-
 [[bin]]
 name = "linkchecker"
 path = "main.rs"