improve crate vendoring (#110)

Summary:
Pull Request resolved: facebook/sapling#110

Pull Request resolved: facebookexperimental/rust-shed#27

Make it so that changes to rust-shed or other common rust source are used locally vendored, so they don't need to be pushed to github before they are visible in a build.

There was already some support for cargo vendoring in getdeps, but it was limited to dependencies between manifests built with cargo builder.  This wasn't enough to build something like eden (cmake is main entry point, with later calls cargo) or eden_scm (make is main entry point, with later calls to cargo), so this diff adds a cargo prepare step for getdeps other primary build systems.

The cargo vendoring is done by using a cargo config file to point to the source files used by getdeps.  It has two modes:

1. per crate, existing mode which is already automatic for cargo to cargo manifest dependencies.  To use it for a non cargo build manifest, add crate.pathmap
2. per git url, existing mode which was only use for crates.io third-party crates, now can be enabled by setting cargo.cargo_config_file

Reviewed By: yancouto

Differential Revision: D33895469

fbshipit-source-id: 7b13c0b679532492a336ce217de875c25fe1be90

build/fbcode_builder/CMake/RustStaticLibrary.cmake

@@ -8,7 +8,14 @@ set(
 )
 set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF)
 
+set(
+  GENERATE_CARGO_VENDOR_CONFIG AUTO CACHE STRING
+  "Whether to generate Rust cargo vendor config or use existing"
+)
+set_property(CACHE GENERATE_CARGO_VENDOR_CONFIG PROPERTY STRINGS AUTO ON OFF)
+
 set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}")
+
 if("${USE_CARGO_VENDOR}" STREQUAL "AUTO")
   if(EXISTS "${RUST_VENDORED_CRATES_DIR}")
     set(USE_CARGO_VENDOR ON)
@@ -17,7 +24,11 @@ if("${USE_CARGO_VENDOR}" STREQUAL "AUTO")
   endif()
 endif()
 
-if(USE_CARGO_VENDOR)
+if("${GENERATE_CARGO_VENDOR_CONFIG}" STREQUAL "AUTO")
+  set(GENERATE_CARGO_VENDOR_CONFIG "${USE_CARGO_VENDOR}")
+endif() 
+
+if(GENERATE_CARGO_VENDOR_CONFIG)
   if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}")
     message(
       FATAL "vendored rust crates not present: "

build/fbcode_builder/getdeps.py

@@ -599,6 +599,18 @@ def run_project_cmd(self, args, loader, manifest):
                     if os.path.exists(built_marker):
                         os.unlink(built_marker)
                     src_dir = fetcher.get_src_dir()
+                    # Prepare builders write out config before the main builder runs
+                    prepare_builders = m.create_prepare_builders(
+                        loader.build_opts,
+                        ctx,
+                        src_dir,
+                        build_dir,
+                        inst_dir,
+                        loader,
+                    )
+                    for preparer in prepare_builders:
+                        preparer.prepare(install_dirs, reconfigure=reconfigure)
+
                     builder = m.create_builder(
                         loader.build_opts,
                         src_dir,

build/fbcode_builder/getdeps/builder.py

@@ -86,14 +86,22 @@ def _run_cmd(
             allow_fail=allow_fail,
         )
 
-    def build(self, install_dirs, reconfigure: bool) -> None:
-        print("Building %s..." % self.manifest.name)
-
+    def _reconfigure(self, reconfigure):
         if self.build_dir is not None:
             if not os.path.isdir(self.build_dir):
                 os.makedirs(self.build_dir)
                 reconfigure = True
+        return reconfigure
 
+    def prepare(self, install_dirs, reconfigure):
+        print("Preparing %s..." % self.manifest.name)
+        reconfigure = self._reconfigure(reconfigure)
+        self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
+
+    def build(self, install_dirs, reconfigure) -> None:
+        print("Building %s..." % self.manifest.name)
+        reconfigure = self._reconfigure(reconfigure)
+        self._prepare(install_dirs=install_dirs, reconfigure=reconfigure)
         self._build(install_dirs=install_dirs, reconfigure=reconfigure)
 
         # On Windows, emit a wrapper script that can be used to run build artifacts
@@ -135,6 +143,12 @@ def run_tests(
         raise an exception."""
         pass
 
+    def _prepare(self, install_dirs, reconfigure):
+        """Prepare the build. Useful when need to generate config,
+        but builder is not the primary build system.
+        e.g. cargo when called from cmake"""
+        pass
+
     def _build(self, install_dirs, reconfigure) -> None:
         """Perform the build.
         install_dirs contains the list of installation directories for