diff --git a/build/fbcode_builder/CMake/RustStaticLibrary.cmake b/build/fbcode_builder/CMake/RustStaticLibrary.cmake index 8546fe2fb..d35f208cf 100644 --- a/build/fbcode_builder/CMake/RustStaticLibrary.cmake +++ b/build/fbcode_builder/CMake/RustStaticLibrary.cmake @@ -8,7 +8,14 @@ set( ) set_property(CACHE USE_CARGO_VENDOR PROPERTY STRINGS AUTO ON OFF) +set( + GENERATE_CARGO_VENDOR_CONFIG AUTO CACHE STRING + "Whether to generate Rust cargo vendor config or use existing" +) +set_property(CACHE GENERATE_CARGO_VENDOR_CONFIG PROPERTY STRINGS AUTO ON OFF) + set(RUST_VENDORED_CRATES_DIR "$ENV{RUST_VENDORED_CRATES_DIR}") + if("${USE_CARGO_VENDOR}" STREQUAL "AUTO") if(EXISTS "${RUST_VENDORED_CRATES_DIR}") set(USE_CARGO_VENDOR ON) @@ -17,7 +24,11 @@ if("${USE_CARGO_VENDOR}" STREQUAL "AUTO") endif() endif() -if(USE_CARGO_VENDOR) +if("${GENERATE_CARGO_VENDOR_CONFIG}" STREQUAL "AUTO") + set(GENERATE_CARGO_VENDOR_CONFIG "${USE_CARGO_VENDOR}") +endif() + +if(GENERATE_CARGO_VENDOR_CONFIG) if(NOT EXISTS "${RUST_VENDORED_CRATES_DIR}") message( FATAL "vendored rust crates not present: " diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index 2da326ef3..f88a3972b 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -599,6 +599,18 @@ def run_project_cmd(self, args, loader, manifest): if os.path.exists(built_marker): os.unlink(built_marker) src_dir = fetcher.get_src_dir() + # Prepare builders write out config before the main builder runs + prepare_builders = m.create_prepare_builders( + loader.build_opts, + ctx, + src_dir, + build_dir, + inst_dir, + loader, + ) + for preparer in prepare_builders: + preparer.prepare(install_dirs, reconfigure=reconfigure) + builder = m.create_builder( loader.build_opts, src_dir, diff --git a/build/fbcode_builder/getdeps/builder.py b/build/fbcode_builder/getdeps/builder.py index 2f43cb933..e1ea29186 100644 --- a/build/fbcode_builder/getdeps/builder.py +++ b/build/fbcode_builder/getdeps/builder.py @@ -86,14 +86,22 @@ def _run_cmd( allow_fail=allow_fail, ) - def build(self, install_dirs, reconfigure: bool) -> None: - print("Building %s..." % self.manifest.name) - + def _reconfigure(self, reconfigure): if self.build_dir is not None: if not os.path.isdir(self.build_dir): os.makedirs(self.build_dir) reconfigure = True + return reconfigure + def prepare(self, install_dirs, reconfigure): + print("Preparing %s..." % self.manifest.name) + reconfigure = self._reconfigure(reconfigure) + self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) + + def build(self, install_dirs, reconfigure) -> None: + print("Building %s..." % self.manifest.name) + reconfigure = self._reconfigure(reconfigure) + self._prepare(install_dirs=install_dirs, reconfigure=reconfigure) self._build(install_dirs=install_dirs, reconfigure=reconfigure) # On Windows, emit a wrapper script that can be used to run build artifacts @@ -135,6 +143,12 @@ def run_tests( raise an exception.""" pass + def _prepare(self, install_dirs, reconfigure): + """Prepare the build. Useful when need to generate config, + but builder is not the primary build system. + e.g. cargo when called from cmake""" + pass + def _build(self, install_dirs, reconfigure) -> None: """Perform the build. install_dirs contains the list of installation directories for diff --git a/build/fbcode_builder/getdeps/cargo.py b/build/fbcode_builder/getdeps/cargo.py index 346fb5127..3a96f177b 100644 --- a/build/fbcode_builder/getdeps/cargo.py +++ b/build/fbcode_builder/getdeps/cargo.py @@ -5,6 +5,7 @@ # LICENSE file in the root directory of this source tree. import os +import re import shutil from .builder import BuilderBase @@ -23,6 +24,7 @@ def __init__( workspace_dir, manifests_to_build, loader, + cargo_config_file, ) -> None: super(CargoBuilder, self).__init__( build_opts, ctx, manifest, src_dir, build_dir, inst_dir @@ -31,6 +33,7 @@ def __init__( self.ws_dir = workspace_dir self.manifests_to_build = manifests_to_build and manifests_to_build.split(",") self.loader = loader + self.cargo_config_file_subdir = cargo_config_file def run_cargo(self, install_dirs, operation, args=None) -> None: args = args or [] @@ -60,17 +63,24 @@ def recreate_dir(self, src, dst) -> None: shutil.rmtree(dst) shutil.copytree(src, dst) - def _build(self, install_dirs, reconfigure) -> None: - build_source_dir = self.build_source_dir() - self.recreate_dir(self.src_dir, build_source_dir) + def cargo_config_file(self): + build_source_dir = self.build_dir + if self.cargo_config_file_subdir: + return os.path.join(build_source_dir, self.cargo_config_file_subdir) + else: + return os.path.join(build_source_dir, ".cargo", "config") - dot_cargo_dir = os.path.join(build_source_dir, ".cargo") - if not os.path.isdir(dot_cargo_dir): - os.mkdir(dot_cargo_dir) + def _create_cargo_config(self): + cargo_config_file = self.cargo_config_file() + cargo_config_dir = os.path.dirname(cargo_config_file) + if not os.path.isdir(cargo_config_dir): + os.mkdir(cargo_config_dir) - with open(os.path.join(dot_cargo_dir, "config"), "w+") as f: + print(f"Writing cargo config for {self.manifest.name} to {cargo_config_file}") + with open(cargo_config_file, "w+") as f: f.write( """\ +# Generated by getdeps.py [build] target-dir = '''{}''' @@ -85,18 +95,45 @@ def _build(self, install_dirs, reconfigure) -> None: ) ) - if self.ws_dir is not None: - self._patchup_workspace() + # Point to vendored sources from getdeps manifests + dep_to_git = self._resolve_dep_to_git() + for _dep, git_conf in dep_to_git.items(): + if "cargo_vendored_sources" in git_conf: + with open(cargo_config_file, "a") as f: + vendored_dir = git_conf["cargo_vendored_sources"].replace( + "\\", "\\\\" + ) + f.write( + f""" +[source."{git_conf["repo_url"]}"] +directory = "{vendored_dir}" +""" + ) + # Point to vendored crates.io if possible try: from .facebook.rust import vendored_crates - vendored_crates(self.build_opts, build_source_dir) + vendored_crates(self.build_opts, cargo_config_file) except ImportError: # This FB internal module isn't shippped to github, # so just rely on cargo downloading crates on it's own pass + return dep_to_git + + def _prepare(self, install_dirs, reconfigure): + build_source_dir = self.build_source_dir() + self.recreate_dir(self.src_dir, build_source_dir) + + dep_to_git = self._create_cargo_config() + + if self.ws_dir is not None: + self._patchup_workspace(dep_to_git) + + def _build(self, install_dirs, reconfigure) -> None: + # _prepare has been run already. Actually do the build + build_source_dir = self.build_source_dir() if self.manifests_to_build is None: self.run_cargo( install_dirs, @@ -138,7 +175,7 @@ def run_tests( if self.build_doc: self.run_cargo(install_dirs, "doc", ["--no-deps"] + margs) - def _patchup_workspace(self) -> None: + def _patchup_workspace(self, dep_to_git) -> None: """ This method makes some assumptions about the state of the project and its cargo dependendies: @@ -159,9 +196,11 @@ def _patchup_workspace(self) -> None: producing bad results. """ workspace_dir = self.workspace_dir() - config = self._resolve_config() + config = self._resolve_config(dep_to_git) if config: - with open(os.path.join(workspace_dir, "Cargo.toml"), "r+") as f: + patch_cargo = os.path.join(workspace_dir, "Cargo.toml") + print(f"writing patch to {patch_cargo}") + with open(patch_cargo, "r+") as f: manifest_content = f.read() if "[package]" not in manifest_content: # A fake manifest has to be crated to change the virtual @@ -169,52 +208,63 @@ def _patchup_workspace(self) -> None: # in many ways and the inability to define patches on them is # one. Check https://github.com/rust-lang/cargo/issues/4934 to # see if it is resolved. + null_file = "/dev/null" + if self.build_opts.is_windows(): + null_file = "nul" f.write( - """ - [package] - name = "fake_manifest_of_{}" - version = "0.0.0" - [lib] - path = "/dev/null" - """.format( - self.manifest.name - ) + f""" +[package] +name = "fake_manifest_of_{self.manifest.name}" +version = "0.0.0" + +[lib] +path = "{null_file}" +""" ) else: f.write("\n") f.write(config) - def _resolve_config(self) -> str: + def _resolve_config(self, dep_to_git) -> str: """ Returns a configuration to be put inside root Cargo.toml file which patches the dependencies git code with local getdeps versions. See https://doc.rust-lang.org/cargo/reference/manifest.html#the-patch-section """ - dep_to_git = self._resolve_dep_to_git() - dep_to_crates = CargoBuilder._resolve_dep_to_crates( - self.build_source_dir(), dep_to_git - ) + dep_to_crates = self._resolve_dep_to_crates(self.build_source_dir(), dep_to_git) config = [] - for name in sorted(dep_to_git.keys()): - git_conf = dep_to_git[name] - crates = sorted(dep_to_crates.get(name, [])) - if not crates: + + git_url_to_crates_and_paths = {} + for dep_name in sorted(dep_to_git.keys()): + git_conf = dep_to_git[dep_name] + req_crates = sorted(dep_to_crates.get(dep_name, [])) + if not req_crates: continue # nothing to patch, move along + + git_url = git_conf.get("repo_url", None) + crate_source_map = git_conf["crate_source_map"] + if git_url and crate_source_map: + crates_to_patch_path = git_url_to_crates_and_paths.get(git_url, {}) + for c in req_crates: + if c in crate_source_map and c not in crates_to_patch_path: + crates_to_patch_path[c] = crate_source_map[c] + print( + f"{self.manifest.name}: Patching crate {c} via virtual manifest in {self.workspace_dir()}" + ) + if crates_to_patch_path: + git_url_to_crates_and_paths[git_url] = crates_to_patch_path + + for git_url, crates_to_patch_path in git_url_to_crates_and_paths.items(): crates_patches = [ '{} = {{ path = "{}" }}'.format( crate, - CargoBuilder._resolve_crate_to_path(crate, git_conf).replace( - "\\", "\\\\" - ), + crates_to_patch_path[crate].replace("\\", "\\\\"), ) - for crate in crates + for crate in sorted(crates_to_patch_path.keys()) ] + config.append(f'\n[patch."{git_url}"]\n' + "\n".join(crates_patches)) - config.append( - '[patch."{0}"]\n'.format(git_conf["repo_url"]) - + "\n".join(crates_patches) - ) return "\n".join(config) def _resolve_dep_to_git(self): @@ -231,27 +281,74 @@ def _resolve_dep_to_git(self): for dep in dependencies: dep_manifest = self.loader.load_manifest(dep) dep_builder = dep_manifest.get("build", "builder", ctx=self.ctx) - if dep_builder not in ["cargo", "nop"] or dep == "rust": - # This is a direct dependency, but it is not build with cargo - # and it is not simply copying files with nop, so ignore it. + + dep_cargo_conf = dep_manifest.get_section_as_dict("cargo", self.ctx) + dep_crate_map = dep_manifest.get_section_as_dict("crate.pathmap", self.ctx) + + if ( + not (dep_crate_map or dep_cargo_conf) + and dep_builder not in ["cargo"] + or dep == "rust" + ): + # This dependency has no cargo rust content so ignore it. # The "rust" dependency is an exception since it contains the # toolchain. continue git_conf = dep_manifest.get_section_as_dict("git", self.ctx) - if "repo_url" not in git_conf: + if dep != "rust" and "repo_url" not in git_conf: raise Exception( - "A cargo dependency requires git.repo_url to be defined." + f"{dep}: A cargo dependency requires git.repo_url to be defined." ) - source_dir = self.loader.get_project_install_dir(dep_manifest) + if dep_builder == "cargo": - source_dir = os.path.join(source_dir, "source") - git_conf["source_dir"] = source_dir + dep_source_dir = self.loader.get_project_install_dir(dep_manifest) + dep_source_dir = os.path.join(dep_source_dir, "source") + else: + fetcher = self.loader.create_fetcher(dep_manifest) + dep_source_dir = fetcher.get_src_dir() + + crate_source_map = {} + if dep_crate_map: + for (crate, subpath) in dep_crate_map.items(): + if crate not in crate_source_map: + if self.build_opts.is_windows(): + subpath = subpath.replace("/", "\\") + crate_path = os.path.join(dep_source_dir, subpath) + print( + f"{self.manifest.name}: Mapped crate {crate} to dep {dep} dir {crate_path}" + ) + crate_source_map[crate] = crate_path + elif dep_cargo_conf: + # We don't know what crates are defined buy the dep, look for them + search_pattern = re.compile('\\[package\\]\nname = "(.*)"') + for crate_root, _, files in os.walk(dep_source_dir): + if "Cargo.toml" in files: + with open(os.path.join(crate_root, "Cargo.toml"), "r") as f: + content = f.read() + match = search_pattern.search(content) + if match: + crate = match.group(1) + if crate: + print( + f"{self.manifest.name}: Discovered crate {crate} in dep {dep} dir {crate_root}" + ) + crate_source_map[crate] = crate_root + + git_conf["crate_source_map"] = crate_source_map + + if not dep_crate_map and dep_cargo_conf: + dep_cargo_dir = self.loader.get_project_build_dir(dep_manifest) + dep_cargo_dir = os.path.join(dep_cargo_dir, "source") + dep_ws_dir = dep_cargo_conf.get("workspace_dir", None) + if dep_ws_dir: + dep_cargo_dir = os.path.join(dep_cargo_dir, dep_ws_dir) + git_conf["cargo_vendored_sources"] = dep_cargo_dir + dep_to_git[dep] = git_conf return dep_to_git - @staticmethod - def _resolve_dep_to_crates(build_source_dir, dep_to_git): + def _resolve_dep_to_crates(self, build_source_dir, dep_to_git): """ This function traverse the build_source_dir in search of Cargo.toml files, extracts the crate names from them using _extract_crates @@ -262,18 +359,33 @@ def _resolve_dep_to_crates(build_source_dir, dep_to_git): return {} # no deps, so don't waste time traversing files dep_to_crates = {} + + # First populate explicit crate paths from depedencies + for name, git_conf in dep_to_git.items(): + crates = git_conf["crate_source_map"].keys() + if crates: + dep_to_crates.setdefault(name, set()).update(crates) + + # Now find from Cargo.tomls for root, _, files in os.walk(build_source_dir): for f in files: if f == "Cargo.toml": - more_dep_to_crates = CargoBuilder._extract_crates( + more_dep_to_crates = CargoBuilder._extract_crates_used( os.path.join(root, f), dep_to_git ) - for name, crates in more_dep_to_crates.items(): - dep_to_crates.setdefault(name, set()).update(crates) + for dep_name, crates in more_dep_to_crates.items(): + existing_crates = dep_to_crates.get(dep_name, set()) + for c in crates: + if c not in existing_crates: + print( + f"Patch {self.manifest.name} uses {dep_name} crate {crates}" + ) + existing_crates.insert(c) + dep_to_crates.setdefault(name, set()).update(existing_crates) return dep_to_crates @staticmethod - def _extract_crates(cargo_toml_file, dep_to_git): + def _extract_crates_used(cargo_toml_file, dep_to_git): """ This functions reads content of provided cargo toml file and extracts crate names per each dependency. The extraction is done by a heuristic @@ -284,7 +396,8 @@ def _extract_crates(cargo_toml_file, dep_to_git): for line in f.readlines(): if line.startswith("#") or "git = " not in line: continue # filter out commented lines and ones without git deps - for name, conf in dep_to_git.items(): + for dep_name, conf in dep_to_git.items(): + # Only redirect deps that point to git URLS if 'git = "{}"'.format(conf["repo_url"]) in line: pkg_template = ' package = "' if pkg_template in line: @@ -293,23 +406,26 @@ def _extract_crates(cargo_toml_file, dep_to_git): ].partition('"') else: crate_name, _, _ = line.partition("=") - deps_to_crates.setdefault(name, set()).add(crate_name.strip()) + deps_to_crates.setdefault(dep_name, set()).add( + crate_name.strip() + ) return deps_to_crates - @staticmethod - def _resolve_crate_to_path(crate, git_conf): + def _resolve_crate_to_path(self, crate, crate_source_map): """ - Tries to find in git_conf["inst_dir"] by searching a [package] + Tries to find in source_dir by searching a [package] keyword followed by name = "". """ - source_dir = git_conf["source_dir"] search_pattern = '[package]\nname = "{}"'.format(crate) - for root, _, files in os.walk(source_dir): - for fname in files: - if fname == "Cargo.toml": - with open(os.path.join(root, fname), "r") as f: - if search_pattern in f.read(): - return root + for (_crate, crate_source_dir) in crate_source_map.items(): + for crate_root, _, files in os.walk(crate_source_dir): + if "Cargo.toml" in files: + with open(os.path.join(crate_root, "Cargo.toml"), "r") as f: + content = f.read() + if search_pattern in content: + return crate_root - raise Exception("Failed to found crate {} in path {}".format(crate, source_dir)) + raise Exception( + f"{self.manifest.name}: Failed to find dep crate {crate} in paths {crate_source_map}" + ) diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 99c47bb2f..705c2af4c 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -75,6 +75,8 @@ "build_doc": OPTIONAL, "workspace_dir": OPTIONAL, "manifests_to_build": OPTIONAL, + # Where to write cargo config (defaults to build_dir/.cargo/config) + "cargo_config_file": OPTIONAL, }, }, "github.actions": { @@ -83,6 +85,7 @@ "run_tests": OPTIONAL, }, }, + "crate.pathmap": {"optional_section": True}, "cmake.defines": {"optional_section": True}, "autoconf.args": {"optional_section": True}, "autoconf.envcmd.LDFLAGS": {"optional_section": True}, @@ -447,6 +450,12 @@ def create_fetcher(self, build_options, ctx): "project %s has no fetcher configuration matching %s" % (self.name, ctx) ) + def get_builder_name(self, ctx): + builder = self.get("build", "builder", ctx=ctx) + if not builder: + raise Exception("project %s has no builder for %r" % (self.name, ctx)) + return builder + def create_builder( # noqa:C901 self, build_options, @@ -458,9 +467,7 @@ def create_builder( # noqa:C901 final_install_prefix=None, extra_cmake_defines=None, ): - builder = self.get("build", "builder", ctx=ctx) - if not builder: - raise Exception("project %s has no builder for %r" % (self.name, ctx)) + builder = self.get_builder_name(ctx) build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx) if build_in_src_dir == "true": # Some scripts don't work when they are configured and build in @@ -574,20 +581,8 @@ def create_builder( # noqa:C901 ) if builder == "cargo": - build_doc = self.get("cargo", "build_doc", False, ctx) - workspace_dir = self.get("cargo", "workspace_dir", None, ctx) - manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) - return CargoBuilder( - build_options, - ctx, - self, - src_dir, - build_dir, - inst_dir, - build_doc, - workspace_dir, - manifests_to_build, - loader, + return self.create_cargo_builder( + build_options, ctx, src_dir, build_dir, inst_dir, loader ) if builder == "OpenNSA": @@ -595,6 +590,41 @@ def create_builder( # noqa:C901 raise KeyError("project %s has no known builder" % (self.name)) + def create_prepare_builders( + self, build_options, ctx, src_dir, build_dir, inst_dir, loader + ): + """Create builders that have a prepare step run, e.g. to write config files""" + prepare_builders = [] + builder = self.get_builder_name(ctx) + cargo = self.get_section_as_dict("cargo", ctx) + if not builder == "cargo" and cargo: + cargo_builder = self.create_cargo_builder( + build_options, ctx, src_dir, build_dir, inst_dir, loader + ) + prepare_builders.append(cargo_builder) + return prepare_builders + + def create_cargo_builder( + self, build_options, ctx, src_dir, build_dir, inst_dir, loader + ): + build_doc = self.get("cargo", "build_doc", False, ctx) + workspace_dir = self.get("cargo", "workspace_dir", None, ctx) + manifests_to_build = self.get("cargo", "manifests_to_build", None, ctx) + cargo_config_file = self.get("cargo", "cargo_config_file", None, ctx) + return CargoBuilder( + build_options, + ctx, + self, + src_dir, + build_dir, + inst_dir, + build_doc, + workspace_dir, + manifests_to_build, + loader, + cargo_config_file, + ) + class ManifestContext(object): """ProjectContext contains a dictionary of values to use when evaluating boolean diff --git a/build/fbcode_builder/manifests/eden b/build/fbcode_builder/manifests/eden index 6509270f6..ec31df0db 100644 --- a/build/fbcode_builder/manifests/eden +++ b/build/fbcode_builder/manifests/eden @@ -10,6 +10,9 @@ repo_url = https://github.com/facebookexperimental/eden.git [github.actions] run_tests = off +[cargo] +cargo_config_file = _cargo_home/config + [build] builder = cmake @@ -26,6 +29,7 @@ lz4 pexpect python-toml python-filelock +rust-shed [dependencies.fbsource=on] rust @@ -100,6 +104,8 @@ INSTALL_PYTHON_LIB=ON ENABLE_GIT=OFF [cmake.defines.fbsource=on] +# OFF as getdeps will write the cargo config file +GENERATE_CARGO_VENDOR_CONFIG=OFF USE_CARGO_VENDOR=ON [cmake.defines.fb=on] diff --git a/build/fbcode_builder/manifests/eden_scm b/build/fbcode_builder/manifests/eden_scm index 362e97dbb..e381fda68 100644 --- a/build/fbcode_builder/manifests/eden_scm +++ b/build/fbcode_builder/manifests/eden_scm @@ -53,9 +53,11 @@ fbcode/tools/lfs = tools/lfs [dependencies] fb303 fbthrift -python rust-shed +[dependencies.not(os=windows)] +python + # We use the system openssl on linux [dependencies.not(os=linux)] openssl diff --git a/build/fbcode_builder/manifests/fb303 b/build/fbcode_builder/manifests/fb303 index b19254913..86ba61ffd 100644 --- a/build/fbcode_builder/manifests/fb303 +++ b/build/fbcode_builder/manifests/fb303 @@ -7,6 +7,12 @@ shipit_fbcode_builder = true [git] repo_url = https://github.com/facebook/fb303.git +[cargo] +cargo_config_file = source/fb303/thrift/.cargo/config + +[crate.pathmap] +fb303_core = fb303/thrift + [build] builder = cmake diff --git a/build/fbcode_builder/manifests/fbthrift b/build/fbcode_builder/manifests/fbthrift index 38640433a..80e74962a 100644 --- a/build/fbcode_builder/manifests/fbthrift +++ b/build/fbcode_builder/manifests/fbthrift @@ -7,6 +7,12 @@ shipit_fbcode_builder = true [git] repo_url = https://github.com/facebook/fbthrift.git +[cargo] +cargo_config_file = source/thrift/lib/rust/.cargo/config + +[crate.pathmap] +fbthrift = thrift/lib/rust + [build] builder = cmake job_weight_mib = 2048 diff --git a/build/fbcode_builder/manifests/rust-shed b/build/fbcode_builder/manifests/rust-shed index a15335d48..31e2b61d9 100644 --- a/build/fbcode_builder/manifests/rust-shed +++ b/build/fbcode_builder/manifests/rust-shed @@ -25,6 +25,7 @@ tools/rust/ossconfigs = . [dependencies] fbthrift +fb303 # We use the system openssl on linux [dependencies.not(os=linux)]