Start using subtrees for external dependencies

mk/crates.mk

@@ -29,6 +29,11 @@
 #	the HOST_CRATES set, but the HOST_CRATES set can depend on target
 #	crates.
 #
+#   EXTERNAL_CRATES
+#   	These crates are all imported using the git-subtree command and have a
+#   	slightly different structure as they're primarily intended to be built
+#   	with Cargo, but we build them manually here.
+#
 #   TOOLS
 #	A list of all tools which will be built as part of the compilation
 #	process. It is currently assumed that most tools are built through
@@ -49,15 +54,16 @@
 # automatically generated for all stage/host/target combinations.
 ################################################################################
 
-TARGET_CRATES := libc std flate arena term \
-                 serialize getopts collections test rand \
+TARGET_CRATES := libc std flate arena \
+                 collections test rand \
                  log graphviz core rbml alloc \
                  rustc_unicode rustc_bitflags
 RUSTC_CRATES := rustc rustc_typeck rustc_borrowck rustc_resolve rustc_driver \
                 rustc_trans rustc_back rustc_llvm rustc_privacy rustc_lint \
                 rustc_data_structures
 HOST_CRATES := syntax $(RUSTC_CRATES) rustdoc fmt_macros
-CRATES := $(TARGET_CRATES) $(HOST_CRATES)
+EXTERNAL_CRATES := term rustc_serialize getopts
+CRATES := $(TARGET_CRATES) $(HOST_CRATES) $(EXTERNAL_CRATES)
 TOOLS := compiletest rustdoc rustc rustbook error-index-generator
 
 DEPS_core :=
@@ -68,37 +74,37 @@ DEPS_std := core libc rand alloc collections rustc_unicode \
 	native:rust_builtin native:backtrace native:rustrt_native \
 	rustc_bitflags
 DEPS_graphviz := std
-DEPS_syntax := std term serialize log fmt_macros arena libc
+DEPS_syntax := std term rustc_serialize log fmt_macros arena libc
 DEPS_rustc_driver := arena flate getopts graphviz libc rustc rustc_back rustc_borrowck \
-                     rustc_typeck rustc_resolve log syntax serialize rustc_llvm \
+                     rustc_typeck rustc_resolve log syntax rustc_serialize rustc_llvm \
 		     rustc_trans rustc_privacy rustc_lint
 
 DEPS_rustc_trans := arena flate getopts graphviz libc rustc rustc_back \
-	                log syntax serialize rustc_llvm
+	                log syntax rustc_serialize rustc_llvm
 DEPS_rustc_typeck := rustc syntax
 DEPS_rustc_borrowck := rustc log graphviz syntax
 DEPS_rustc_resolve := rustc log syntax
 DEPS_rustc_privacy := rustc log syntax
 DEPS_rustc_lint := rustc log syntax
-DEPS_rustc := syntax flate arena serialize getopts rbml \
+DEPS_rustc := syntax flate arena rustc_serialize getopts rbml \
               log graphviz rustc_llvm rustc_back rustc_data_structures
 DEPS_rustc_llvm := native:rustllvm libc std
 DEPS_rustc_back := std syntax rustc_llvm flate log libc
-DEPS_rustc_data_structures := std log serialize
-DEPS_rustdoc := rustc rustc_driver native:hoedown serialize getopts \
+DEPS_rustc_data_structures := std log rustc_serialize
+DEPS_rustdoc := rustc rustc_driver native:hoedown rustc_serialize getopts \
                 test rustc_lint
 DEPS_rustc_bitflags := core
 DEPS_flate := std native:miniz
 DEPS_arena := std
 DEPS_graphviz := std
 DEPS_glob := std
-DEPS_serialize := std log
-DEPS_rbml := std log serialize
-DEPS_term := std log
+DEPS_rustc_serialize := std
+DEPS_rbml := std log rustc_serialize
+DEPS_term := std
 DEPS_getopts := std
 DEPS_collections := core alloc rustc_unicode
 DEPS_num := std
-DEPS_test := std getopts serialize rbml term native:rust_test_helpers
+DEPS_test := std getopts rustc_serialize rbml term native:rust_test_helpers
 DEPS_rand := core
 DEPS_log := std
 DEPS_fmt_macros = std
@@ -107,7 +113,7 @@ TOOL_DEPS_compiletest := test getopts
 TOOL_DEPS_rustdoc := rustdoc
 TOOL_DEPS_rustc := rustc_driver
 TOOL_DEPS_rustbook := std rustdoc
-TOOL_DEPS_error-index-generator := rustdoc syntax serialize
+TOOL_DEPS_error-index-generator := rustdoc syntax rustc_serialize
 TOOL_SOURCE_compiletest := $(S)src/compiletest/compiletest.rs
 TOOL_SOURCE_rustdoc := $(S)src/driver/driver.rs
 TOOL_SOURCE_rustc := $(S)src/driver/driver.rs
@@ -136,11 +142,33 @@ DOC_CRATES := std alloc collections core libc rustc_unicode
 define RUST_CRATE
 CRATEFILE_$(1) := $$(S)src/lib$(1)/lib.rs
 RSINPUTS_$(1) := $$(call rwildcard,$(S)src/lib$(1)/,*.rs)
+endef
+
+$(foreach crate,$(TARGET_CRATES),$(eval $(call RUST_CRATE,$(crate))))
+$(foreach crate,$(HOST_CRATES),$(eval $(call RUST_CRATE,$(crate))))
+
+# Distinct from the above macro, this generates the variables needed for
+# external crates (located in the src/external folder). These crates are
+# typically intended to be built with Cargo so we need to pass some extra flags
+# and use a different source location.
+#
+# $(1) is the crate to generate variables for
+define EXTERNAL_CRATE
+CRATEFILE_$(1) := $$(S)src/external/$(1)/src/lib.rs
+RUSTFLAGS_$(1) += --crate-type rlib,dylib --crate-name $(1) --cfg rust_build
+RSINPUTS_$(1) := $$(call rwildcard,$(S)src/external/$(1)/src/,*.rs)
+endef
+
+$(foreach crate,$(EXTERNAL_CRATES),$(eval $(call EXTERNAL_CRATE,$(crate))))
+
+# Build the dependencies array for all crates
+#
+# $(1) is the crate to generate variables for
+define BUILD_CRATE_DEPS
 RUST_DEPS_$(1) := $$(filter-out native:%,$$(DEPS_$(1)))
 NATIVE_DEPS_$(1) := $$(patsubst native:%,%,$$(filter native:%,$$(DEPS_$(1))))
 endef
-
-$(foreach crate,$(CRATES),$(eval $(call RUST_CRATE,$(crate))))
+$(foreach crate,$(CRATES),$(eval $(call BUILD_CRATE_DEPS,$(crate))))
 
 # Similar to the macro above for crates, this macro is for tools
 #

mk/dist.mk

@@ -53,7 +53,9 @@ PKG_FILES := \
       driver                                   \
       etc                                      \
       error-index-generator                    \
-      $(foreach crate,$(CRATES),lib$(crate))   \
+      external                                 \
+      $(foreach crate,$(TARGET_CRATES),lib$(crate))  \
+      $(foreach crate,$(HOST_CRATES),lib$(crate))    \
       libcollectionstest                       \
       libcoretest                              \
       libbacktrace                             \

src/compiletest/compiletest.rs

@@ -31,7 +31,6 @@ extern crate log;
 use std::env;
 use std::fs;
 use std::path::{Path, PathBuf};
-use getopts::{optopt, optflag, reqopt};
 use common::Config;
 use common::{Pretty, DebugInfoGdb, DebugInfoLldb};
 use util::logv;
@@ -55,60 +54,60 @@ pub fn main() {
     run_tests(&config);
 }
 
-pub fn parse_config(args: Vec<String> ) -> Config {
-
-    let groups : Vec<getopts::OptGroup> =
-        vec!(reqopt("", "compile-lib-path", "path to host shared libraries", "PATH"),
-          reqopt("", "run-lib-path", "path to target shared libraries", "PATH"),
-          reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH"),
-          reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH"),
-          reqopt("", "python", "path to python to use for doc tests", "PATH"),
-          optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM"),
-          optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind"),
-          optopt("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR"),
-          reqopt("", "src-base", "directory to scan for test files", "PATH"),
-          reqopt("", "build-base", "directory to deposit test outputs", "PATH"),
-          reqopt("", "aux-base", "directory to find auxiliary test files", "PATH"),
-          reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET"),
-          reqopt("", "mode", "which sort of compile tests to run",
-                 "(compile-fail|parse-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)"),
-          optflag("", "ignored", "run tests marked as ignored"),
-          optopt("", "runtool", "supervisor program to run tests under \
-                                 (eg. emulator, valgrind)", "PROGRAM"),
-          optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS"),
-          optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS"),
-          optflag("", "verbose", "run tests verbosely, showing all output"),
-          optopt("", "logfile", "file to log test execution to", "FILE"),
-          optflag("", "jit", "run tests under the JIT"),
-          optopt("", "target", "the target to build for", "TARGET"),
-          optopt("", "host", "the host to build for", "HOST"),
-          optopt("", "gdb-version", "the version of GDB used", "VERSION STRING"),
-          optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING"),
-          optopt("", "android-cross-path", "Android NDK standalone path", "PATH"),
-          optopt("", "adb-path", "path to the android debugger", "PATH"),
-          optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH"),
-          optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH"),
-          optflag("h", "help", "show this message"));
+pub fn parse_config(args: Vec<String>) -> Config {
+
+    let mut options = getopts::Options::new();
+    options
+        .reqopt("", "compile-lib-path", "path to host shared libraries", "PATH")
+        .reqopt("", "run-lib-path", "path to target shared libraries", "PATH")
+        .reqopt("", "rustc-path", "path to rustc to use for compiling", "PATH")
+        .reqopt("", "rustdoc-path", "path to rustdoc to use for compiling", "PATH")
+        .reqopt("", "python", "path to python to use for doc tests", "PATH")
+        .optopt("", "valgrind-path", "path to Valgrind executable for Valgrind tests", "PROGRAM")
+        .optflag("", "force-valgrind", "fail if Valgrind tests cannot be run under Valgrind")
+        .optopt("", "llvm-bin-path", "path to directory holding llvm binaries", "DIR")
+        .reqopt("", "src-base", "directory to scan for test files", "PATH")
+        .reqopt("", "build-base", "directory to deposit test outputs", "PATH")
+        .reqopt("", "aux-base", "directory to find auxiliary test files", "PATH")
+        .reqopt("", "stage-id", "the target-stage identifier", "stageN-TARGET")
+        .reqopt("", "mode", "which sort of compile tests to run",
+                "(compile-fail|parse-fail|run-fail|run-pass|run-pass-valgrind|pretty|debug-info)")
+        .optflag("", "ignored", "run tests marked as ignored")
+        .optopt("", "runtool", "supervisor program to run tests under \
+                                (eg. emulator, valgrind)", "PROGRAM")
+        .optopt("", "host-rustcflags", "flags to pass to rustc for host", "FLAGS")
+        .optopt("", "target-rustcflags", "flags to pass to rustc for target", "FLAGS")
+        .optflag("", "verbose", "run tests verbosely, showing all output")
+        .optopt("", "logfile", "file to log test execution to", "FILE")
+        .optflag("", "jit", "run tests under the JIT")
+        .optopt("", "target", "the target to build for", "TARGET")
+        .optopt("", "host", "the host to build for", "HOST")
+        .optopt("", "gdb-version", "the version of GDB used", "VERSION STRING")
+        .optopt("", "lldb-version", "the version of LLDB used", "VERSION STRING")
+        .optopt("", "android-cross-path", "Android NDK standalone path", "PATH")
+        .optopt("", "adb-path", "path to the android debugger", "PATH")
+        .optopt("", "adb-test-dir", "path to tests for the android debugger", "PATH")
+        .optopt("", "lldb-python-dir", "directory containing LLDB's python module", "PATH")
+        .optflag("h", "help", "show this message");
 
     assert!(!args.is_empty());
     let argv0 = args[0].clone();
     let args_ = args.tail();
     if args[1] == "-h" || args[1] == "--help" {
         let message = format!("Usage: {} [OPTIONS] [TESTNAME...]", argv0);
-        println!("{}", getopts::usage(&message, &groups));
+        println!("{}", options.usage(&message));
         println!("");
         panic!()
     }
 
-    let matches =
-        &match getopts::getopts(args_, &groups) {
-          Ok(m) => m,
-          Err(f) => panic!("{:?}", f)
-        };
+    let matches = &match options.parse(args_) {
+        Ok(m) => m,
+        Err(f) => panic!("{:?}", f)
+    };
 
     if matches.opt_present("h") || matches.opt_present("help") {
         let message = format!("Usage: {} [OPTIONS]  [TESTNAME...]", argv0);
-        println!("{}", getopts::usage(&message, &groups));
+        println!("{}", options.usage(&message));
         println!("");
         panic!()
     }

src/etc/tidy.py

@@ -108,6 +108,7 @@ def interesting_file(f):
             'src/rustllvm',
             'src/rt/valgrind',
             'src/rt/msvc',
+            'src/external',
             'src/rust-installer'
         }
 

src/external/README.md

@@ -0,0 +1,42 @@
+## External crates
+
+All crates in this directory are hosted externally from this repository and are
+imported via the standard `git-subtree` command. These crates **should not** be
+edited directly, but instead changes to should go upstream and then be pulled
+into these crates.
+
+Crates here are listed in the `EXTERNAL_CRATES` array in `mk/crates.mk` and are
+built via the standard build system.
+
+### Adding a new external crate
+
+1. Make sure the crate has the appropriate `#![cfg_attr]` annotations to make
+   the crate unstable in the distribution with a message pointing to crates.io.
+   See the existing crates in the `src/external` folder for examples.
+
+2. To add a new crate `foo` to this folder, first execute the following:
+
+   ```sh
+   git subtree add -P src/external/foo https://github.com/bar/foo master --squash
+   ```
+
+  This will check out the crate into this folder, squashing the entire history
+  into one commit (the rust-lang/rust repo doesn't need the whole history of the
+  crate).
+
+3. Next, edit `mk/crates.mk` appropriately by modifying `EXTERNAL_CRATES` and
+   possibly some other crates and/or dependency lists.
+
+4. Add the crate to `src/test/compile-fail-fulldeps/unstable-crates.rs` to
+   ensure that it is unstable in the distribution.
+
+### Updating an external crate
+
+To pull in upstream changes to a library `foo`, execute the following
+
+```sh
+git subtree pull -P src/external/foo https://github.com/bar/foo master --squash
+```
+
+Similar to the addition process the `--squash` argument is provided to squash
+all changes into one commit.

src/external/getopts/.gitignore

@@ -0,0 +1,2 @@
+/target
+/Cargo.lock

src/external/getopts/.travis.yml

@@ -0,0 +1,24 @@
+language: rust
+rust:
+  - 1.0.0
+  - beta
+  - nightly
+script:
+  - cargo build --verbose
+  - cargo test --verbose
+  - cargo doc
+after_success: |
+  [ $TRAVIS_BRANCH = master ] &&
+  [ $TRAVIS_PULL_REQUEST = false ] &&
+  [ $TRAVIS_RUST_VERSION = nightly ] &&
+  echo '<meta http-equiv=refresh content=0;url=getopts/index.html>' > target/doc/index.html &&
+  sudo pip install ghp-import &&
+  ghp-import -n target/doc &&
+  git push -fq https://${TOKEN}@github.com/${TRAVIS_REPO_SLUG}.git gh-pages
+
+env:
+  global:
+    secure: H6SBXFQ/i/xRTCkEgKwk9WTSKXXaaEMqrASLbzKqhqJHR3ed/vp5YBWzlCJkAI3+2j3iVmbCfHqtpzmF9f1GAn+apg3mDI5GfFRJAfncHhZoN332wcBLvvNuRvJS9wRk3j3kOLKjBJmPa7+TCFafyXfCAutzfUlQPK8dgxG8WEI=
+notifications:
+  email:
+    on_success: never

src/external/getopts/Cargo.toml

@@ -0,0 +1,16 @@
+[package]
+
+name = "getopts"
+version = "0.2.11"
+authors = ["The Rust Project Developers"]
+license = "MIT/Apache-2.0"
+readme = "README.md"
+repository = "https://github.com/rust-lang/getopts"
+documentation = "http://doc.rust-lang.org/getopts"
+homepage = "https://github.com/rust-lang/getopts"
+description = """
+getopts-like option parsing.
+"""
+
+[dependencies]
+log = "0.3"