Skip to content
This repository was archived by the owner on Apr 21, 2025. It is now read-only.

RBE #758

Merged
zanderso merged 1 commit into from
Aug 31, 2023
Merged

RBE #758

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions build/rbe.cfg
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,9 @@
service=remotebuildexecution.googleapis.com:443
instance=projects/flutter-rbe-prod/instances/default
use_application_default_credentials=true
enable_deps_cache=true
xattr_digest=user.flutter.rbe.digest.sha256
server_address=unix:///tmp/reproxy.sock
log_dir=/tmp
Copy link

@keyonghan keyonghan Aug 30, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Nit: can we use a dir based on timestamp?

Copy link
Member Author

@zanderso zanderso Aug 30, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

You tell me =) I just copied this from your doc.

Copy link

@keyonghan keyonghan Aug 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This file here is used for local build, and the dir should not be a problem.
I will make sure recipes to use a timestamp based dir when executed from CI/LED.

Copy link
Member Author

@zanderso zanderso Aug 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you tell me how you plan to do that?

Copy link

@keyonghan keyonghan Aug 31, 2023

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

One way we can do is have recipes read this cfg file and override the log_file value before starting the bootstrap.

proxy_log_dir=/tmp
log_format=reducedtext
31 changes: 28 additions & 3 deletions build/toolchain/android/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import("//build/toolchain/ccache.gni")
import("//build/toolchain/clang.gni")
import("//build/toolchain/gcc_toolchain.gni")
import("//build/toolchain/goma.gni")
import("//build/toolchain/rbe.gni")

# The Android GCC toolchains share most of the same parameters, so we have this
# wrapper around gcc_toolchain to avoid duplication of logic.
Expand Down Expand Up @@ -35,10 +36,31 @@ template("android_gcc_toolchain") {
if (use_goma) {
assert(!use_ccache, "Goma and ccache can't be used together.")
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_rbe) {
assert(!use_goma)
rewrapper_args = [
"$rbe_dir/rewrapper",
"--server_address=$rbe_server_address",
"--exec_strategy=$rbe_exec_strategy",
"--dial_timeout=$rbe_dial_timeout",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]
compiler_args = rewrapper_args + [
"--labels=type=compile,compiler=clang,lang=cpp ",
]
link_args = rewrapper_args + [
"--labels=type=link,tool=clang ",
]
compiler_prefix = string_join(" ", compiler_args)
link_prefix = string_join(" ", link_args)
} else if (use_ccache) {
compiler_prefix = "ccache "
link_prefix = "ccache "
} else {
compiler_prefix = ""
link_prefix = ""
}

is_clang = invoker.is_clang
Expand All @@ -54,18 +76,21 @@ template("android_gcc_toolchain") {

prefix = rebase_path("//buildtools/$host_dir/clang/bin", root_build_dir)

asm = prefix + "/clang"
cc = compiler_prefix + prefix + "/clang"
cxx = compiler_prefix + prefix + "/clang++"
ar = prefix + "/llvm-ar"
ld = link_prefix + prefix + "/clang++"
} else {
asm = tool_prefix + "gcc"
cc = compiler_prefix + tool_prefix + "gcc"
cxx = compiler_prefix + tool_prefix + "g++"
ar = tool_prefix + "ar"
ld = link_prefix + tool_prefix + "g++"
}

ld = cxx
readelf = compiler_prefix + tool_prefix + "readelf"
nm = compiler_prefix + tool_prefix + "nm"
readelf = tool_prefix + "readelf"
nm = tool_prefix + "nm"

toolchain_os = "android"
toolchain_cpu = invoker.toolchain_cpu
Expand Down
19 changes: 17 additions & 2 deletions build/toolchain/gcc_toolchain.gni
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@

import("//build/toolchain/clang.gni")
import("//build/toolchain/clang_static_analyzer.gni")
import("//build/toolchain/rbe.gni")

# Path to the Clang static analysis wrapper script.
analyzer_wrapper =
Expand All @@ -14,6 +15,7 @@ analyzer_wrapper =
# (including clang).
#
# It requires the following variables specifying the executables to run:
# - asm
# - cc
# - cxx
# - ar
Expand Down Expand Up @@ -50,6 +52,7 @@ analyzer_wrapper =
# when specified.
template("gcc_toolchain") {
toolchain(target_name) {
assert(defined(invoker.asm), "gcc_toolchain() must specify a \"asm\" value")
assert(defined(invoker.cc), "gcc_toolchain() must specify a \"cc\" value")
assert(defined(invoker.cxx), "gcc_toolchain() must specify a \"cxx\" value")
assert(defined(invoker.ar), "gcc_toolchain() must specify a \"ar\" value")
Expand All @@ -76,6 +79,7 @@ template("gcc_toolchain") {
# We can't do string interpolation ($ in strings) on things with dots in
# them. To allow us to use $cc below, for example, we create copies of
# these values in our scope.
asm = invoker.asm
cc = compiler_prefix + invoker.cc
cxx = compiler_prefix + invoker.cxx
ar = invoker.ar
Expand Down Expand Up @@ -120,6 +124,12 @@ template("gcc_toolchain") {
tool("cc") {
depfile = "{{output}}.d"
command = "$cc -MD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}"
if (use_rbe) {
# The depfile generated on an RBE worker can contain absolute paths.
# This sed command strips off the prefix, and rewrites to be
# relative to the buildroot.
command += " && sed -i 's@/b/f/w@../..@g' $depfile"
}
depsformat = "gcc"
description = "CC {{output}}"
outputs = [
Expand All @@ -130,6 +140,9 @@ template("gcc_toolchain") {
tool("cxx") {
depfile = "{{output}}.d"
command = "$cxx -MD -MF $depfile {{defines}} {{include_dirs}} {{cflags}} {{cflags_cc}} $coverage_flags -c {{source}} -o {{output}}"
if (use_rbe) {
command += " && sed -i 's@/b/f/w@../..@g' $depfile"
}
depsformat = "gcc"
description = "CXX {{output}}"
outputs = [
Expand All @@ -138,9 +151,11 @@ template("gcc_toolchain") {
}

tool("asm") {
# For GCC we can just use the C compiler to compile assembly.
depfile = "{{output}}.d"
command = "$cc -MD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}"
command = "$asm -MD -MF $depfile {{defines}} {{include_dirs}} {{asmflags}} {{cflags}} {{cflags_c}} $coverage_flags -c {{source}} -o {{output}}"
if (use_rbe) {
command += " && sed -i 's@/b/f/w@../..@g' $depfile"
}
depsformat = "gcc"
description = "ASM {{output}}"
outputs = [
Expand Down
59 changes: 46 additions & 13 deletions build/toolchain/linux/BUILD.gn
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,19 +6,40 @@ import("//build/config/sysroot.gni")
import("//build/toolchain/ccache.gni")
import("//build/toolchain/gcc_toolchain.gni")
import("//build/toolchain/goma.gni")
import("//build/toolchain/rbe.gni")

declare_args() {
toolchain_prefix = ""
# TODO(zra): Add an argument for overriding the host toolchain.
}

if (use_goma) {
assert(!use_ccache, "Goma and ccache can't be used together.")
compiler_prefix = "$goma_dir/gomacc "
link_prefix = "$goma_dir/gomacc "
} else if (use_rbe) {
assert(!use_goma)
rewrapper_args = [
"$rbe_dir/rewrapper",
"--server_address=$rbe_server_address",
"--exec_strategy=$rbe_exec_strategy",
"--dial_timeout=$rbe_dial_timeout",
"--exec_root=$rbe_exec_root",
"--platform=$rbe_platform",
]
compiler_args = rewrapper_args + [
"--labels=type=compile,compiler=clang,lang=cpp ",
]
link_args = rewrapper_args + [
"--labels=type=link,tool=clang ",
]
compiler_prefix = string_join(" ", compiler_args)
link_prefix = string_join(" ", link_args)
} else if (use_ccache) {
compiler_prefix = "ccache "
link_prefix = "ccache "
} else {
compiler_prefix = ""
link_prefix = ""
}

if (host_cpu == "arm64") {
Expand All @@ -35,11 +56,12 @@ gcc_toolchain("arm") {
prefix = toolchain_prefix
}

asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
Expand All @@ -51,13 +73,14 @@ gcc_toolchain("arm") {

gcc_toolchain("clang_arm") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "arm"
Expand All @@ -71,11 +94,12 @@ gcc_toolchain("arm64") {
prefix = toolchain_prefix
}

asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
Expand All @@ -87,13 +111,14 @@ gcc_toolchain("arm64") {

gcc_toolchain("clang_arm64") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "arm64"
Expand All @@ -103,13 +128,14 @@ gcc_toolchain("clang_arm64") {

gcc_toolchain("clang_x86") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "x86"
Expand All @@ -119,13 +145,14 @@ gcc_toolchain("clang_x86") {

gcc_toolchain("x86") {
prefix = ""
asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

readelf = "${prefix}readelf"
nm = "${prefix}nm"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
strip = "${prefix}strip"

toolchain_cpu = "x86"
Expand All @@ -135,13 +162,14 @@ gcc_toolchain("x86") {

gcc_toolchain("clang_x64") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "${prefix}/llvm-readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "x64"
Expand All @@ -151,13 +179,14 @@ gcc_toolchain("clang_x64") {

gcc_toolchain("x64") {
prefix = ""
asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

readelf = "${prefix}readelf"
nm = "${prefix}nm"
ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
strip = "${prefix}strip"

toolchain_cpu = "x64"
Expand All @@ -171,11 +200,12 @@ gcc_toolchain("riscv32") {
prefix = toolchain_prefix
}

asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
Expand All @@ -187,13 +217,14 @@ gcc_toolchain("riscv32") {

gcc_toolchain("clang_riscv32") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "riscv32"
Expand All @@ -207,11 +238,12 @@ gcc_toolchain("riscv64") {
prefix = toolchain_prefix
}

asm = "${prefix}gcc"
cc = "${compiler_prefix}${prefix}gcc"
cxx = "${compiler_prefix}${prefix}g++"

ar = "${prefix}ar"
ld = cxx
ld = "${link_prefix}${prefix}g++"
readelf = "${prefix}readelf"
nm = "${prefix}nm"
strip = "${prefix}strip"
Expand All @@ -223,13 +255,14 @@ gcc_toolchain("riscv64") {

gcc_toolchain("clang_riscv64") {
prefix = rebased_clang_dir
asm = "${prefix}/clang"
cc = "${compiler_prefix}${prefix}/clang"
cxx = "${compiler_prefix}${prefix}/clang++"

readelf = "readelf"
nm = "${prefix}/llvm-nm"
ar = "${prefix}/llvm-ar"
ld = cxx
ld = "${link_prefix}${prefix}/clang++"
llvm_objcopy = "${prefix}/llvm-objcopy"

toolchain_cpu = "riscv64"
Expand Down
Loading