diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 10bd2a62a6..83916fde94 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -18,6 +18,7 @@ from shlex import quote as shellquote from typing import Optional +from .copytree import simple_copytree from .dyndeps import create_dyn_dep_munger from .envfuncs import add_path_entry, Env, path_search from .fetcher import copy_if_different @@ -1165,9 +1166,14 @@ def _build(self, reconfigure) -> None: perl = typing.cast(str, path_search(env, "perl", "perl")) make_j_args = [] + extra_args = [] if self.build_opts.is_windows(): - make = "nmake.exe" + # jom is compatible with nmake, adds the /j argument for parallel build + make = "jom.exe" + make_j_args = ["/j%s" % self.num_jobs] args = ["VC-WIN64A-masm", "-utf-8"] + # fixes "if multiple CL.EXE write to the same .PDB file, please use /FS" + extra_args = ["/FS"] elif self.build_opts.is_darwin(): make = "make" make_j_args = ["-j%s" % self.num_jobs] @@ -1200,11 +1206,14 @@ def _build(self, reconfigure) -> None: "no-unit-test", "no-tests", ] + + extra_args ) + # show the config produced + self._run_cmd([perl, "configdata.pm", "--dump"], env=env) make_build = [make] + make_j_args - self._run_cmd(make_build) + self._run_cmd(make_build, env=env) make_install = [make, "install_sw", "install_ssldirs"] - self._run_cmd(make_install) + self._run_cmd(make_install, env=env) class Boost(BuilderBase): @@ -1321,7 +1330,7 @@ def build(self, reconfigure: bool) -> None: os.makedirs(dest_parent) if os.path.isdir(full_src): if not os.path.exists(full_dest): - shutil.copytree(full_src, full_dest) + simple_copytree(full_src, full_dest) else: shutil.copyfile(full_src, full_dest) shutil.copymode(full_src, full_dest) @@ -1333,7 +1342,7 @@ def build(self, reconfigure: bool) -> None: os.chmod(full_dest, st.st_mode | stat.S_IXUSR) else: if not os.path.exists(self.inst_dir): - shutil.copytree(self.src_dir, self.inst_dir) + simple_copytree(self.src_dir, self.inst_dir) class SqliteBuilder(BuilderBase): diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 0e0e0ddfe0..5bb2ada85c 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -13,6 +13,7 @@ import typing from .builder import BuilderBase +from .copytree import simple_copytree if typing.TYPE_CHECKING: from .buildopts import BuildOptions @@ -79,7 +80,7 @@ def recreate_dir(self, src, dst) -> None: os.remove(dst) else: shutil.rmtree(dst) - shutil.copytree(src, dst) + simple_copytree(src, dst) def cargo_config_file(self): build_source_dir = self.build_dir diff --git a/build/fbcode_builder/getdeps/copytree.py b/build/fbcode_builder/getdeps/copytree.py index 2297bd3aa8..6815f74c89 100644 --- a/build/fbcode_builder/getdeps/copytree.py +++ b/build/fbcode_builder/getdeps/copytree.py @@ -10,6 +10,7 @@ import subprocess from .platform import is_windows +from .runcmd import run_cmd PREFETCHED_DIRS = set() @@ -65,18 +66,34 @@ def prefetch_dir_if_eden(dirpath) -> None: PREFETCHED_DIRS.add(dirpath) -# pyre-fixme[9]: ignore has type `bool`; used as `None`. -def copytree(src_dir, dest_dir, ignore: bool = None): - """Recursively copy the src_dir to the dest_dir, filtering - out entries using the ignore lambda. The behavior of the - ignore lambda must match that described by `shutil.copytree`. - This `copytree` function knows how to prefetch data when - running in an eden repo. - TODO: I'd like to either extend this or add a variant that - uses watchman to mirror src_dir into dest_dir. - """ - prefetch_dir_if_eden(src_dir) - # pyre-fixme[6]: For 3rd param expected - # `Union[typing.Callable[[Union[PathLike[str], str], List[str]], Iterable[str]], - # typing.Callable[[str, List[str]], Iterable[str]], None]` but got `bool`. - return shutil.copytree(src_dir, dest_dir, ignore=ignore) +def simple_copytree(src_dir, dest_dir, symlinks=False): + """A simple version of shutil.copytree() that can delegate to native tools if faster""" + if is_windows(): + os.makedirs(dest_dir, exist_ok=True) + cmd = [ + "robocopy.exe", + src_dir, + dest_dir, + # copy directories, including empty ones + "/E", + # Ignore Extra files in destination + "/XX", + # enable parallel copy + "/MT", + # be quiet + "/NFL", + "/NDL", + "/NJH", + "/NJS", + "/NP", + ] + if symlinks: + cmd.append("/SL") + # robocopy exits with code 1 if it copied ok, hence allow_fail + # https://learn.microsoft.com/en-us/troubleshoot/windows-server/backup-and-storage/return-codes-used-robocopy-utility + exit_code = run_cmd(cmd, allow_fail=True) + if exit_code > 1: + raise subprocess.CalledProcessError(exit_code, cmd) + return dest_dir + else: + return shutil.copytree(src_dir, dest_dir, symlinks=symlinks) diff --git a/build/fbcode_builder/manifests/jom b/build/fbcode_builder/manifests/jom new file mode 100644 index 0000000000..effecab67a --- /dev/null +++ b/build/fbcode_builder/manifests/jom @@ -0,0 +1,15 @@ +# jom is compatible with MSVC nmake, but adds the /j argment which +# speeds up openssl build a lot +[manifest] +name = jom + +# see https://download.qt.io/official_releases/jom/changelog.txt for latest version +[download.os=windows] +url = https://download.qt.io/official_releases/jom/jom_1_1_4.zip +sha256 = d533c1ef49214229681e90196ed2094691e8c4a0a0bef0b2c901debcb562682b + +[build.os=windows] +builder = nop + +[install.files.os=windows] +. = bin diff --git a/build/fbcode_builder/manifests/openssl b/build/fbcode_builder/manifests/openssl index 7dd40727cc..ebd680e7e1 100644 --- a/build/fbcode_builder/manifests/openssl +++ b/build/fbcode_builder/manifests/openssl @@ -31,4 +31,5 @@ builder = openssl subdir = openssl-3.0.15 [dependencies.os=windows] +jom perl