From e1e07daa5b5515c91fa9f0f87b894f4ea37de21e Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Sat, 22 Apr 2023 06:35:08 -0700 Subject: [PATCH 1/2] Manual Clang roll --- DEPS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/DEPS b/DEPS index 1874d0fe8b983..6b860fc76e549 100644 --- a/DEPS +++ b/DEPS @@ -39,7 +39,7 @@ vars = { # The list of revisions for these tools comes from Fuchsia, here: # https://fuchsia.googlesource.com/integration/+/HEAD/toolchain # If there are problems with the toolchain, contact fuchsia-toolchain@. - 'clang_version': 'git_revision:20d06c833d833ef6b2d0f519cc4a7998d49a2803', + 'clang_version': 'git_revision:e8cfbfd05a951b85f80156dffc8eeecb34c7271c', # When updating the Dart revision, ensure that all entries that are # dependencies of Dart are also updated to match the entries in the From 0f5dcd2c56d7b3f07a0764fd42a458a8deb2a9f8 Mon Sep 17 00:00:00 2001 From: Zach Anderson Date: Tue, 25 Apr 2023 12:23:07 -0700 Subject: [PATCH 2/2] Download and use goma client from cipd --- DEPS | 79 ++++++++++++++++++++++++++++++++++++++++++++++++++++++-- tools/gn | 27 +++++++++++++++++-- 2 files changed, 102 insertions(+), 4 deletions(-) diff --git a/DEPS b/DEPS index 6b860fc76e549..79252ea6c177a 100644 --- a/DEPS +++ b/DEPS @@ -39,7 +39,13 @@ vars = { # The list of revisions for these tools comes from Fuchsia, here: # https://fuchsia.googlesource.com/integration/+/HEAD/toolchain # If there are problems with the toolchain, contact fuchsia-toolchain@. - 'clang_version': 'git_revision:e8cfbfd05a951b85f80156dffc8eeecb34c7271c', + 'clang_version': 'git_revision:20d06c833d833ef6b2d0f519cc4a7998d49a2803', + + # The goma version and the clang version can be tightly coupled. If goma + # stops working on a clang roll, this may need to be updated using the value + # from the 'integration' tag of + # https://chrome-infra-packages.appspot.com/p/fuchsia/third_party/goma/client + 'goma_version': ' git_revision:41b3bcb64014144a844153fd5588c36411fffb56', # When updating the Dart revision, ensure that all entries that are # dependencies of Dart are also updated to match the entries in the @@ -98,7 +104,12 @@ vars = { "checkout_llvm": False, # Setup Git hooks by default. - "setup_githooks": True, + 'setup_githooks': True, + + # When this is true, the goma client will be downloaded from cipd, and + # the engine build will prefer to use this client over a client that is + # specified by GOMA_DIR, or installed in the default goma install location. + 'use_cipd_goma': False, # This is not downloaded be default because it increases the # `gclient sync` time by between 1 and 3 minutes. This option is enabled @@ -830,6 +841,40 @@ deps = { 'dep_type': 'cipd', }, + # GOMA + 'src/buildtools/mac-x64/goma': { + 'packages': [ + { + 'package': 'fuchsia/third_party/goma/client/mac-amd64', + 'version': Var('goma_version'), + } + ], + 'condition': 'use_cipd_goma and host_os == "mac"', + 'dep_type': 'cipd', + }, + + 'src/buildtools/linux-x64/goma': { + 'packages': [ + { + 'package': 'fuchsia/third_party/goma/client/linux-amd64', + 'version': Var('goma_version'), + } + ], + 'condition': 'use_cipd_goma and host_os == "linux"', + 'dep_type': 'cipd', + }, + + 'src/buildtools/windows-x64/goma': { + 'packages': [ + { + 'package': 'fuchsia/third_party/goma/client/windows-amd64', + 'version': Var('goma_version'), + } + ], + 'condition': 'use_cipd_goma and download_windows_deps', + 'dep_type': 'cipd', + }, + # Get the SDK from https://chrome-infra-packages.appspot.com/p/fuchsia/sdk/core at the 'latest' tag # Get the toolchain from https://chrome-infra-packages.appspot.com/p/fuchsia/clang at the 'goma' tag 'src/fuchsia/sdk/mac': { @@ -965,6 +1010,36 @@ hooks = [ 'src/flutter/tools/activate_emsdk.py', ] }, + { + 'name': 'Start compiler proxy', + 'pattern': '.', + 'condition': 'use_cipd_goma and host_os == "mac"', + 'action': [ + 'python3', + 'src/buildtools/mac-x64/goma/goma_ctl.py', + 'ensure_start' + ] + }, + { + 'name': 'Start compiler proxy', + 'pattern': '.', + 'condition': 'use_cipd_goma and host_os == "linux"', + 'action': [ + 'python3', + 'src/buildtools/linux-x64/goma/goma_ctl.py', + 'ensure_start' + ] + }, + { + 'name': 'Start compiler proxy', + 'pattern': '.', + 'condition': 'use_cipd_goma and download_windows_deps', + 'action': [ + 'python3', + 'src/buildtools/windows-x64/goma/goma_ctl.py', + 'ensure_start' + ] + }, { 'name': 'Setup githooks', 'pattern': '.', diff --git a/tools/gn b/tools/gn index 29f2eec504a16..cbc33afdc0bd1 100755 --- a/tools/gn +++ b/tools/gn @@ -199,12 +199,32 @@ def get_target_cpu(args): return 'x64' +def buildtools_dir(): + host_os = get_host_os() + host_cpu = get_host_cpu() + if host_os == 'win': + host_os = 'windows' + if host_os == 'mac' and host_cpu == 'arm64': + host_cpu = 'x64' + return '%s-%s' % (host_os, host_cpu) + + def setup_goma(args): goma_gn_args = {} + # When running in CI, the recipes use their own goma install, and take + # care of starting and stopping the compiler proxy. + running_on_luci = os.environ.get('LUCI_CONTEXT') is not None + + # Prefer the goma fetched by gclient if it exists. + cipd_goma_dir = os.path.join(SRC_ROOT, 'buildtools', buildtools_dir(), 'goma') + + # Next, if GOMA_DIR is set, use that install. goma_dir = os.environ.get('GOMA_DIR') - goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma') + # Finally, look for goma in the install location recommended in our + # documentation. + goma_home_dir = os.path.join(os.getenv('HOME', ''), 'goma') # GOMA has a different default (home) path on gWindows. if not os.path.exists(goma_home_dir) and sys.platform.startswith( ('cygwin', 'win')): @@ -214,6 +234,9 @@ def setup_goma(args): goma_gn_args['use_goma'] = False goma_gn_args['goma_dir'] = None print('Disabling GOMA for wasm builds, it is not supported yet.') + elif args.goma and not running_on_luci and os.path.exists(cipd_goma_dir): + goma_gn_args['use_goma'] = True + goma_gn_args['goma_dir'] = cipd_goma_dir elif args.goma and goma_dir and os.path.exists(goma_dir): goma_gn_args['use_goma'] = True goma_gn_args['goma_dir'] = goma_dir @@ -233,7 +256,7 @@ def setup_goma(args): goma_gn_args['goma_dir'] = None if goma_gn_args['use_goma'] and sys.platform == 'darwin': - if (os.environ.get('LUCI_CONTEXT') is None or args.xcode_symlinks or + if (not running_on_luci or args.xcode_symlinks or os.getenv('FLUTTER_GOMA_CREATE_XCODE_SYMLINKS', '0') == '1'): goma_gn_args['create_xcode_symlinks'] = True