From 63b0bdcbfc73e85a917a70c6794ef34cd0d51800 Mon Sep 17 00:00:00 2001 From: David Greenberg Date: Fri, 11 Mar 2022 13:55:14 -0800 Subject: [PATCH 1/3] getdeps optionally can get hg info from env var Summary: Adds an environment variable to getdeps to provide `hg` info to avoid calling `hg` directly. When using `getdeps` inside a containerized environment (which we need to build Research Super Cluster tooling with the correct linker attributes), `getdeps` fails because of unregistered mercurial extensions in the `hgrc`. This allows `getdeps` to be useable in an environment where the mercurial extensions used in a project aren't installed/available. Differential Revision: D34732506 fbshipit-source-id: e5b921b7a9c62fe2f59e0b28366f7b6601fb50ff --- build/fbcode_builder/getdeps/fetcher.py | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index a42ae58b96..3f45716d1b 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -525,12 +525,15 @@ def get_fbsource_repo_data(build_options) -> FbsourceRepoData: if cached_data: return cached_data - cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"] - env = Env() - env.set("HGPLAIN", "1") - log_data = subprocess.check_output( - cmd, cwd=build_options.fbsource_dir, env=dict(env.items()) - ).decode("ascii") + if "GETDEPS_HG_REPO_DATA" in os.environ: + log_data = os.environ["GETDEPS_HG_REPO_DATA"] + else: + cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"] + env = Env() + env.set("HGPLAIN", "1") + log_data = subprocess.check_output( + cmd, cwd=build_options.fbsource_dir, env=dict(env.items()) + ).decode("ascii") (hash, datestr) = log_data.split("\n") From 6042c1567c2304b0866857eee4fb963d63ed0daf Mon Sep 17 00:00:00 2001 From: David Greenberg Date: Fri, 11 Mar 2022 13:55:14 -0800 Subject: [PATCH 2/3] use libcurl in getdeps Summary: When using getdeps inside of a container, Python's urllib isn't able to download from dewey lfs (see this post for details https://fb.workplace.com/groups/systemd.and.friends/permalink/2747692278870647/). This allows for getdeps to use `libcurl` to fetch dependencies, which allows for a getdeps build to work inside the container environment. Differential Revision: D34696330 fbshipit-source-id: 051a91b504c341c3b212c8d21bce4a7a121c3a37 --- build/fbcode_builder/getdeps/fetcher.py | 33 +++++++++++++++++++++---- 1 file changed, 28 insertions(+), 5 deletions(-) diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py index 3f45716d1b..eeed36dda0 100644 --- a/build/fbcode_builder/getdeps/fetcher.py +++ b/build/fbcode_builder/getdeps/fetcher.py @@ -654,15 +654,14 @@ def get_src_dir(self): def download_url_to_file_with_progress(url: str, file_name) -> None: - print("Download %s -> %s ..." % (url, file_name)) + print("Download with %s -> %s ..." % (url, file_name)) class Progress(object): last_report = 0 - def progress(self, count, block, total): + def write_update(self, total, amount): if total == -1: total = "(Unknown)" - amount = count * block if sys.stdout.isatty(): sys.stdout.write("\r downloading %s of %s " % (amount, total)) @@ -675,10 +674,33 @@ def progress(self, count, block, total): self.last_report = now sys.stdout.flush() + def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount): + self.write_update(total, amount) + + def progress_urllib(self, count, block, total): + amount = count * block + self.write_update(total, amount) + progress = Progress() start = time.time() try: - (_filename, headers) = urlretrieve(url, file_name, reporthook=progress.progress) + if os.environ.get("GETDEPS_USE_LIBCURL") is not None: + import pycurl + + with open(file_name, "wb") as f: + c = pycurl.Curl() + c.setopt(pycurl.URL, url) + c.setopt(pycurl.WRITEDATA, f) + # display progress + c.setopt(pycurl.NOPROGRESS, False) + c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl) + c.perform() + c.close() + headers = None + else: + (_filename, headers) = urlretrieve( + url, file_name, reporthook=progress.progress_urllib + ) except (OSError, IOError) as exc: # noqa: B014 raise TransientFailure( "Failed to download %s to %s: %s" % (url, file_name, str(exc)) @@ -687,7 +709,8 @@ def progress(self, count, block, total): end = time.time() sys.stdout.write(" [Complete in %f seconds]\n" % (end - start)) sys.stdout.flush() - print(f"{headers}") + if headers is not None: + print(f"{headers}") class ArchiveFetcher(Fetcher): From 5978dc026c4f87929b15d51f87423f091bfe6495 Mon Sep 17 00:00:00 2001 From: David Greenberg Date: Fri, 11 Mar 2022 13:55:34 -0800 Subject: [PATCH 3/3] Add argument to pass extra arguments to boost b2 build tool Summary: This adds a way to pass arguments to the `b2` build tool, used by Boost. This is needed in order to link a getdeps built boost into an relocatable `.so`. The motivating use case is that we need to statically link Boost into a native library used by a python wheel, which must be relocatable. This functionality already exists for CMake-based projects. Reviewed By: mackorone Differential Revision: D34796774 fbshipit-source-id: e63446a8ee8b835514518419713eefdba1ef6f72 --- build/fbcode_builder/getdeps.py | 12 ++++++++++++ build/fbcode_builder/getdeps/manifest.py | 3 +++ 2 files changed, 15 insertions(+) diff --git a/build/fbcode_builder/getdeps.py b/build/fbcode_builder/getdeps.py index a4ae0cbfbb..58387232e9 100755 --- a/build/fbcode_builder/getdeps.py +++ b/build/fbcode_builder/getdeps.py @@ -595,6 +595,8 @@ def run_project_cmd(self, args, loader, manifest): else {} ) + extra_b2_args = args.extra_b2_args or [] + if sources_changed or reconfigure or not os.path.exists(built_marker): if os.path.exists(built_marker): os.unlink(built_marker) @@ -620,6 +622,7 @@ def run_project_cmd(self, args, loader, manifest): loader, final_install_prefix=loader.get_project_install_prefix(m), extra_cmake_defines=extra_cmake_defines, + extra_b2_args=extra_b2_args, ) builder.build(install_dirs, reconfigure=reconfigure) @@ -760,6 +763,15 @@ def setup_project_cmd_parser(self, parser): 'e.g: \'{"CMAKE_CXX_FLAGS": "--bla"}\'' ), ) + parser.add_argument( + "--extra-b2-args", + help=( + "Repeatable argument that contains extra arguments to pass " + "to b2, which compiles boost. " + "e.g.: 'cxxflags=-fPIC' 'cflags=-fPIC'" + ), + action="append", + ) parser.add_argument( "--shared-libs", help="Build shared libraries if possible", diff --git a/build/fbcode_builder/getdeps/manifest.py b/build/fbcode_builder/getdeps/manifest.py index 705c2af4c7..b9e29fa4ec 100644 --- a/build/fbcode_builder/getdeps/manifest.py +++ b/build/fbcode_builder/getdeps/manifest.py @@ -466,6 +466,7 @@ def create_builder( # noqa:C901 loader, final_install_prefix=None, extra_cmake_defines=None, + extra_b2_args=None, ): builder = self.get_builder_name(ctx) build_in_src_dir = self.get("build", "build_in_src_dir", "false", ctx=ctx) @@ -527,6 +528,8 @@ def create_builder( # noqa:C901 if builder == "boost": args = self.get_section_as_args("b2.args", ctx) + if extra_b2_args is not None: + args += extra_b2_args return Boost(build_options, ctx, self, src_dir, build_dir, inst_dir, args) if builder == "bistro":