Download and use goma client from cipd

Author: zanderso

DEPS

@@ -39,7 +39,13 @@ vars = {
   # The list of revisions for these tools comes from Fuchsia, here:
   # https://fuchsia.googlesource.com/integration/+/HEAD/toolchain
   # If there are problems with the toolchain, contact fuchsia-toolchain@.
-  'clang_version': 'git_revision:e8cfbfd05a951b85f80156dffc8eeecb34c7271c',
+  'clang_version': 'git_revision:20d06c833d833ef6b2d0f519cc4a7998d49a2803',
+
+  # The goma version and the clang version can be tightly coupled. If goma
+  # stops working on a clang roll, this may need to be updated using the value
+  # from the 'integration' tag of
+  # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client
+  'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
 
   # When updating the Dart revision, ensure that all entries that are
   # dependencies of Dart are also updated to match the entries in the
@@ -98,7 +104,12 @@ vars = {
   "checkout_llvm": False,
 
   # Setup Git hooks by default.
-  "setup_githooks": True,
+  'setup_githooks': True,
+
+  # When this is true, the goma client will be downloaded from cipd, and
+  # the engine build will prefer to use this client over a client that is
+  # specified by GOMA_DIR, or installed in the default goma install location.
+  'use_cipd_goma': False,
 
   # This is not downloaded be default because it increases the
   # `gclient sync` time by between 1 and 3 minutes. This option is enabled
@@ -830,6 +841,40 @@ deps = {
     'dep_type': 'cipd',
   },
 
+  # GOMA
+  'src/buildtools/mac-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/mac-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and host_os == "mac"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/linux-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/linux-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and host_os == "linux"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/windows-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/windows-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'use_cipd_goma and download_windows_deps',
+    'dep_type': 'cipd',
+  },
+
   # Get the SDK from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core at the 'latest' tag
   # Get the toolchain from https://chrome-infra-packages.appspot.com/p/fuchsia/clang at the 'goma' tag
    'src/fuchsia/sdk/mac': {
@@ -965,6 +1010,36 @@ hooks = [
       'src/flutter/tools/activate_emsdk.py',
     ]
   },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and host_os == "mac"',
+    'action': [
+      'python3',
+      'src/buildtools/mac-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and host_os == "linux"',
+    'action': [
+      'python3',
+      'src/buildtools/linux-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
+  {
+    'name': 'Start compiler proxy',
+    'pattern': '.',
+    'condition': 'use_cipd_goma and download_windows_deps',
+    'action': [
+      'python3',
+      'src/buildtools/windows-x64/goma/goma_ctl.py',
+      'ensure_start'
+    ]
+  },
   {
     'name': 'Setup githooks',
     'pattern': '.',

tools/gn

@@ -199,12 +199,32 @@ def get_target_cpu(args):
   return 'x64'
 
 
+def buildtools_dir():
+  host_os = get_host_os()
+  host_cpu = get_host_cpu()
+  if host_os == 'win':
+    host_os = 'windows'
+  if host_os == 'mac' and host_cpu == 'arm64':
+    host_cpu = 'x64'
+  return '%s-%s' % (host_os, host_cpu)
+
+
 def setup_goma(args):
   goma_gn_args = {}
 
+  # When running in CI, the recipes use their own goma install, and take
+  # care of starting and stopping the compiler proxy.
+  running_on_luci = os.environ.get('LUCI_CONTEXT') is not None
+
+  # Prefer the goma fetched by gclient if it exists.
+  cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
+
+  # Next, if GOMA_DIR is set, use that install.
   goma_dir = os.environ.get('GOMA_DIR')
-  goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma')
 
+  # Finally, look for goma in the install location recommended in our
+  # documentation.
+  goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma')
   # GOMA has a different default (home) path on gWindows.
   if not os.path.exists(goma_home_dir) and sys.platform.startswith(
       ('cygwin', 'win')):
@@ -214,6 +234,9 @@ def setup_goma(args):
     goma_gn_args['use_goma'] = False
     goma_gn_args['goma_dir'] = None
     print('Disabling GOMA for wasm builds, it is not supported yet.')
+  elif args.goma and not running_on_luci and os.path.exists(cipd_goma_dir):
+    goma_gn_args['use_goma'] = True
+    goma_gn_args['goma_dir'] = cipd_goma_dir
   elif args.goma and goma_dir and os.path.exists(goma_dir):
     goma_gn_args['use_goma'] = True
     goma_gn_args['goma_dir'] = goma_dir
@@ -233,7 +256,7 @@ def setup_goma(args):
     goma_gn_args['goma_dir'] = None
 
   if goma_gn_args['use_goma'] and sys.platform == 'darwin':
-    if (os.environ.get('LUCI_CONTEXT') is None or args.xcode_symlinks or
+    if (not running_on_luci or args.xcode_symlinks or
         os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
       goma_gn_args['create_xcode_symlinks'] = True