Download and use goma client from cipd

Author: zanderso

DEPS

@@ -39,7 +39,11 @@ vars = {
   # The list of revisions for these tools comes from Fuchsia, here:
   # https://fuchsia.googlesource.com/integration/+/HEAD/toolchain
   # If there are problems with the toolchain, contact fuchsia-toolchain@.
-  'clang_version': 'git_revision:e8cfbfd05a951b85f80156dffc8eeecb34c7271c',
+  'clang_version': 'git_revision:20d06c833d833ef6b2d0f519cc4a7998d49a2803',
+
+  # The goma version and the clang version can be tightly coupled. If goma
+  # stops working on a clang roll, this may need to be updated.
+  'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56',
 
   # When updating the Dart revision, ensure that all entries that are
   # dependencies of Dart are also updated to match the entries in the
@@ -98,7 +102,12 @@ vars = {
   "checkout_llvm": False,
 
   # Setup Git hooks by default.
-  "setup_githooks": True,
+  'setup_githooks': True,
+
+  # When this is true, the goma client will be downloaded from cipd, and
+  # the engine build will prefer to use this client over a client that is
+  # specified by GOMA_DIR, or installed in the default goma install location.
+  'download_goma': False,
 
   # This is not downloaded be default because it increases the
   # `gclient sync` time by between 1 and 3 minutes. This option is enabled
@@ -830,6 +839,41 @@ deps = {
     'dep_type': 'cipd',
   },
 
+
+  # GOMA
+  'src/buildtools/mac-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/mac-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'download_goma and host_os == "mac"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/linux-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/linux-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'download_goma and host_os == "linux"',
+    'dep_type': 'cipd',
+  },
+
+  'src/buildtools/windows-x64/goma': {
+    'packages': [
+      {
+        'package': 'fuchsia/third_party/goma/client/windows-amd64',
+        'version': Var('goma_version'),
+      }
+    ],
+    'condition': 'download_goma and download_windows_deps',
+    'dep_type': 'cipd',
+  },
+
   # Get the SDK from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core at the 'latest' tag
   # Get the toolchain from https://chrome-infra-packages.appspot.com/p/fuchsia/clang at the 'goma' tag
    'src/fuchsia/sdk/mac': {

tools/gn

@@ -199,9 +199,35 @@ def get_target_cpu(args):
   return 'x64'
 
 
+def buildtools_dir():
+  host_os = get_host_os()
+  host_cpu = get_host_cpu()
+  if host_os == 'win':
+    host_os = 'windows'
+  if host_os == 'mac' and host_cpu == 'arm64':
+    host_cpu = 'x64'
+  return '%s-%s' % (host_os, host_cpu)
+
+
+def goma_ensure_start(goma_dir):
+  goma_ctl_path = os.path.join(goma_dir, 'goma_ctl.py')
+  command = ['python3', goma_ctl_path, 'ensure_start']
+  try:
+    output = subprocess.check_output(command, stderr=subprocess.STDOUT)
+    print(output)
+  except subprocess.CalledProcessError:
+    print(
+        'Failed to start goma:\n%s\nEnsure it is started by manually running:\n'
+        '$ %s\n' % (output.decode('utf-8'), ' '.join(command))
+    )
+
+
 def setup_goma(args):
   goma_gn_args = {}
 
+  # Prefer the goma fetched by gclient.
+  cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma')
+
   goma_dir = os.environ.get('GOMA_DIR')
   goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma')
 
@@ -214,6 +240,9 @@ def setup_goma(args):
     goma_gn_args['use_goma'] = False
     goma_gn_args['goma_dir'] = None
     print('Disabling GOMA for wasm builds, it is not supported yet.')
+  elif args.goma and os.path.exists(cipd_goma_dir):
+    goma_gn_args['use_goma'] = True
+    goma_gn_args['goma_dir'] = cipd_goma_dir
   elif args.goma and goma_dir and os.path.exists(goma_dir):
     goma_gn_args['use_goma'] = True
     goma_gn_args['goma_dir'] = goma_dir
@@ -237,6 +266,9 @@ def setup_goma(args):
         os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'):
       goma_gn_args['create_xcode_symlinks'] = True
 
+  if goma_gn_args['use_goma'] and args.goma_ensure_start:
+    goma_ensure_start(goma_gn_args['goma_dir'])
+
   return goma_gn_args
 
 
@@ -1070,6 +1102,19 @@ def parse_args(args):
       help='Do not run GN. Instead configure the Impeller cmake example build.',
   )
 
+  parser.add_argument(
+      '--goma-ensure-start',
+      default=True,
+      action='store_true',
+      help='Ensure that the goma compiler proxy is running.',
+  )
+  parser.add_argument(
+      '--no-goma-ensure-start',
+      dest='goma_ensure_start',
+      action='store_false',
+      help='Do not ensure that the goma compiler proxy is running.',
+  )
+
   # Sanitizers.
   parser.add_argument('--asan', default=False, action='store_true')
   parser.add_argument('--lsan', default=False, action='store_true')