From 108a7cd3421c58edde3e9719475a69e55a3579b3 Mon Sep 17 00:00:00 2001
From: David Greenberg <dgrnbrg@fb.com>
Date: Fri, 11 Mar 2022 13:18:03 -0800
Subject: [PATCH 1/2] getdeps optionally can get hg info from env var

Summary:
Adds an environment variable to getdeps to provide `hg` info to avoid calling `hg` directly.

When using `getdeps` inside a containerized environment (which we need to build Research Super Cluster tooling with the correct linker attributes), `getdeps` fails because of unregistered mercurial extensions in the `hgrc`.

This allows `getdeps` to be useable in an environment where the mercurial extensions used in a project aren't installed/available.

Differential Revision: D34732506

fbshipit-source-id: 2b0fc664cb4afe1315ff62d35c617ae1e9a126e3
---
 build/fbcode_builder/getdeps/fetcher.py | 15 +++++++++------
 1 file changed, 9 insertions(+), 6 deletions(-)

diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index a42ae58b9..3f45716d1 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -525,12 +525,15 @@ def get_fbsource_repo_data(build_options) -> FbsourceRepoData:
     if cached_data:
         return cached_data
 
-    cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"]
-    env = Env()
-    env.set("HGPLAIN", "1")
-    log_data = subprocess.check_output(
-        cmd, cwd=build_options.fbsource_dir, env=dict(env.items())
-    ).decode("ascii")
+    if "GETDEPS_HG_REPO_DATA" in os.environ:
+        log_data = os.environ["GETDEPS_HG_REPO_DATA"]
+    else:
+        cmd = ["hg", "log", "-r.", "-T{node}\n{date|hgdate}"]
+        env = Env()
+        env.set("HGPLAIN", "1")
+        log_data = subprocess.check_output(
+            cmd, cwd=build_options.fbsource_dir, env=dict(env.items())
+        ).decode("ascii")
 
     (hash, datestr) = log_data.split("\n")
 

From 355eed677eba76d513179dfefc5036d34f54a75a Mon Sep 17 00:00:00 2001
From: David Greenberg <dgrnbrg@fb.com>
Date: Fri, 11 Mar 2022 13:18:18 -0800
Subject: [PATCH 2/2] use libcurl in getdeps

Summary:
When using getdeps inside of a container, Python's urllib isn't able to download from dewey lfs (see this post for details https://fb.workplace.com/groups/systemd.and.friends/permalink/2747692278870647/).

This allows for getdeps to use `libcurl` to fetch dependencies, which allows for a getdeps build to work inside the container environment.

Reviewed By: mackorone

Differential Revision: D34696330

fbshipit-source-id: 0dcc935ae7273e1d56ff413d45c738b693ac4fc5
---
 build/fbcode_builder/getdeps/fetcher.py | 33 +++++++++++++++++++++----
 1 file changed, 28 insertions(+), 5 deletions(-)

diff --git a/build/fbcode_builder/getdeps/fetcher.py b/build/fbcode_builder/getdeps/fetcher.py
index 3f45716d1..eeed36dda 100644
--- a/build/fbcode_builder/getdeps/fetcher.py
+++ b/build/fbcode_builder/getdeps/fetcher.py
@@ -654,15 +654,14 @@ def get_src_dir(self):
 
 
 def download_url_to_file_with_progress(url: str, file_name) -> None:
-    print("Download %s -> %s ..." % (url, file_name))
+    print("Download with %s -> %s ..." % (url, file_name))
 
     class Progress(object):
         last_report = 0
 
-        def progress(self, count, block, total):
+        def write_update(self, total, amount):
             if total == -1:
                 total = "(Unknown)"
-            amount = count * block
 
             if sys.stdout.isatty():
                 sys.stdout.write("\r downloading %s of %s " % (amount, total))
@@ -675,10 +674,33 @@ def progress(self, count, block, total):
                     self.last_report = now
             sys.stdout.flush()
 
+        def progress_pycurl(self, total, amount, _uploadtotal, _uploadamount):
+            self.write_update(total, amount)
+
+        def progress_urllib(self, count, block, total):
+            amount = count * block
+            self.write_update(total, amount)
+
     progress = Progress()
     start = time.time()
     try:
-        (_filename, headers) = urlretrieve(url, file_name, reporthook=progress.progress)
+        if os.environ.get("GETDEPS_USE_LIBCURL") is not None:
+            import pycurl
+
+            with open(file_name, "wb") as f:
+                c = pycurl.Curl()
+                c.setopt(pycurl.URL, url)
+                c.setopt(pycurl.WRITEDATA, f)
+                # display progress
+                c.setopt(pycurl.NOPROGRESS, False)
+                c.setopt(pycurl.XFERINFOFUNCTION, progress.progress_pycurl)
+                c.perform()
+                c.close()
+            headers = None
+        else:
+            (_filename, headers) = urlretrieve(
+                url, file_name, reporthook=progress.progress_urllib
+            )
     except (OSError, IOError) as exc:  # noqa: B014
         raise TransientFailure(
             "Failed to download %s to %s: %s" % (url, file_name, str(exc))
@@ -687,7 +709,8 @@ def progress(self, count, block, total):
     end = time.time()
     sys.stdout.write(" [Complete in %f seconds]\n" % (end - start))
     sys.stdout.flush()
-    print(f"{headers}")
+    if headers is not None:
+        print(f"{headers}")
 
 
 class ArchiveFetcher(Fetcher):