speed up file copy on windows

build/fbcode_builder/getdeps/builder.py

@@ -18,6 +18,7 @@
 from shlex import quote as shellquote
 from typing import Optional
 
+from .copytree import simple_copytree
 from .dyndeps import create_dyn_dep_munger
 from .envfuncs import add_path_entry, Env, path_search
 from .fetcher import copy_if_different
@@ -1165,9 +1166,14 @@ def _build(self, reconfigure) -> None:
         perl = typing.cast(str, path_search(env, "perl", "perl"))
 
         make_j_args = []
+        extra_args = []
         if self.build_opts.is_windows():
-            make = "nmake.exe"
+            # jom is compatible with nmake, adds the /j argument for parallel build
+            make = "jom.exe"
+            make_j_args = ["/j%s" % self.num_jobs]
             args = ["VC-WIN64A-masm", "-utf-8"]
+            # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS"
+            extra_args = ["/FS"]
         elif self.build_opts.is_darwin():
             make = "make"
             make_j_args = ["-j%s" % self.num_jobs]
@@ -1200,11 +1206,14 @@ def _build(self, reconfigure) -> None:
                 "no-unit-test",
                 "no-tests",
             ]
+            + extra_args
         )
+        # show the config produced
+        self._run_cmd([perl, "configdata.pm", "--dump"], env=env)
         make_build = [make] + make_j_args
-        self._run_cmd(make_build)
+        self._run_cmd(make_build, env=env)
         make_install = [make, "install_sw", "install_ssldirs"]
-        self._run_cmd(make_install)
+        self._run_cmd(make_install, env=env)
 
 
 class Boost(BuilderBase):
@@ -1321,7 +1330,7 @@ def build(self, reconfigure: bool) -> None:
                     os.makedirs(dest_parent)
                 if os.path.isdir(full_src):
                     if not os.path.exists(full_dest):
-                        shutil.copytree(full_src, full_dest)
+                        simple_copytree(full_src, full_dest)
                 else:
                     shutil.copyfile(full_src, full_dest)
                     shutil.copymode(full_src, full_dest)
@@ -1333,7 +1342,7 @@ def build(self, reconfigure: bool) -> None:
                         os.chmod(full_dest, st.st_mode | stat.S_IXUSR)
         else:
             if not os.path.exists(self.inst_dir):
-                shutil.copytree(self.src_dir, self.inst_dir)
+                simple_copytree(self.src_dir, self.inst_dir)
 
 
 class SqliteBuilder(BuilderBase):

build/fbcode_builder/getdeps/cargo.py

@@ -13,6 +13,7 @@
 import typing
 
 from .builder import BuilderBase
+from .copytree import simple_copytree
 
 if typing.TYPE_CHECKING:
     from .buildopts import BuildOptions
@@ -79,7 +80,7 @@ def recreate_dir(self, src, dst) -> None:
                 os.remove(dst)
             else:
                 shutil.rmtree(dst)
-        shutil.copytree(src, dst)
+        simple_copytree(src, dst)
 
     def cargo_config_file(self):
         build_source_dir = self.build_dir

build/fbcode_builder/getdeps/copytree.py

@@ -10,6 +10,7 @@
 import subprocess
 
 from .platform import is_windows
+from .runcmd import run_cmd
 
 
 PREFETCHED_DIRS = set()
@@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None:
     PREFETCHED_DIRS.add(dirpath)
 
 
-# pyre-fixme[9]: ignore has type `bool`; used as `None`.
-def copytree(src_dir, dest_dir, ignore: bool = None):
-    """Recursively copy the src_dir to the dest_dir, filtering
-    out entries using the ignore lambda.  The behavior of the
-    ignore lambda must match that described by `shutil.copytree`.
-    This `copytree` function knows how to prefetch data when
-    running in an eden repo.
-    TODO: I'd like to either extend this or add a variant that
-    uses watchman to mirror src_dir into dest_dir.
-    """
-    prefetch_dir_if_eden(src_dir)
-    # pyre-fixme[6]: For 3rd param expected
-    #  `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]],
-    #  typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`.
-    return shutil.copytree(src_dir, dest_dir, ignore=ignore)
+def simple_copytree(src_dir, dest_dir, symlinks=False):
+    """A simple version of shutil.copytree() that can delegate to native tools if faster"""
+    if is_windows():
+        os.makedirs(dest_dir, exist_ok=True)
+        cmd = [
+            "robocopy.exe",
+            src_dir,
+            dest_dir,
+            # copy directories, including empty ones
+            "/E",
+            # Ignore Extra files in destination
+            "/XX",
+            # enable parallel copy
+            "/MT",
+            # be quiet
+            "/NFL",
+            "/NDL",
+            "/NJH",
+            "/NJS",
+            "/NP",
+        ]
+        if symlinks:
+            cmd.append("/SL")
+        # robocopy exits with code 1 if it copied ok, hence allow_fail
+        # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility
+        exit_code = run_cmd(cmd, allow_fail=True)
+        if exit_code > 1:
+            raise subprocess.CalledProcessError(exit_code, cmd)
+        return dest_dir
+    else:
+        return shutil.copytree(src_dir, dest_dir, symlinks=symlinks)

build/fbcode_builder/manifests/jom

@@ -0,0 +1,15 @@
+# jom is compatible with MSVC nmake, but adds the /j<number of jobs> argment which 
+# speeds up openssl build a lot
+[manifest]
+name = jom
+
+# see https://download.qt.io/official_releases/jom/changelog.txt for latest version
+[download.os=windows]
+url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip
+sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b
+
+[build.os=windows]
+builder = nop
+
+[install.files.os=windows]
+. = bin

build/fbcode_builder/manifests/openssl

@@ -31,4 +31,5 @@ builder = openssl
 subdir = openssl-3.0.15
 
 [dependencies.os=windows]
+jom
 perl