Skip to content

Commit

Permalink
Use XCode from CIPD. Include xSAN dylibs in Build task isolated output.
Browse files Browse the repository at this point in the history
Change-Id: I0eb3dcc4d38eb2aeab22d1df1515e499cf21dc62
Reviewed-on: https://skia-review.googlesource.com/144608
Commit-Queue: Eric Boren <borenet@google.com>
Auto-Submit: Ben Wagner <benjaminwagner@google.com>
Reviewed-by: Eric Boren <borenet@google.com>
  • Loading branch information
dogben authored and Skia Commit-Bot committed Aug 3, 2018
1 parent 057ad70 commit cdfa16d
Show file tree
Hide file tree
Showing 13 changed files with 542 additions and 42 deletions.
30 changes: 23 additions & 7 deletions infra/bots/gen_tasks.go
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,16 @@ var (
},
}

CIPD_PKGS_XCODE = []*specs.CipdPackage{
// https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#317
// This package is really just an installer for XCode.
&specs.CipdPackage{
Name: "infra/tools/mac_toolchain/${platform}",
Path: "mac_toolchain",
Version: "git_revision:796d2b92cff93fc2059623ce0a66284373ceea0a",
},
}

// Flags.
builderNameSchemaFile = flag.String("builder_name_schema", "", "Path to the builder_name_schema.json file. If not specified, uses infra/bots/recipe_modules/builder_name_schema/builder_name_schema.json from this repo.")
assetsDir = flag.String("assets_dir", "", "Directory containing assets.")
Expand Down Expand Up @@ -301,7 +311,7 @@ func kitchenTask(name, recipe, isolate, serviceAccount string, dimensions []stri
Dependencies: []string{BUNDLE_RECIPES_NAME},
Dimensions: dimensions,
EnvPrefixes: map[string][]string{
"PATH": []string{"cipd_bin_packages", "cipd_bin_packages/bin"},
"PATH": []string{"cipd_bin_packages", "cipd_bin_packages/bin"},
"VPYTHON_VIRTUALENV_ROOT": []string{"${cache_dir}/vpython"},
},
ExtraTags: map[string]string{
Expand Down Expand Up @@ -798,8 +808,6 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("armhf_sysroot"))
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("chromebook_arm_gles"))
}
} else if strings.Contains(name, "CommandBuffer") {
timeout(task, 2*time.Hour)
} else if strings.Contains(name, "Debian") {
if strings.Contains(name, "Clang") {
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("clang_linux"))
Expand Down Expand Up @@ -830,10 +838,18 @@ func compile(b *specs.TasksCfgBuilder, name string, parts map[string]string) str
if strings.Contains(name, "Vulkan") {
task.Dependencies = append(task.Dependencies, isolateCIPDAsset(b, ISOLATE_WIN_VULKAN_SDK_NAME))
}
}

if strings.Contains(name, "MoltenVK") {
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("moltenvk"))
} else if strings.Contains(name, "Mac") {
task.CipdPackages = append(task.CipdPackages, CIPD_PKGS_XCODE...)
task.Caches = append(task.Caches, &specs.Cache{
Name: "xcode",
Path: "cache/Xcode.app",
})
if strings.Contains(name, "CommandBuffer") {
timeout(task, 2*time.Hour)
}
if strings.Contains(name, "MoltenVK") {
task.CipdPackages = append(task.CipdPackages, b.MustGetCipdPackageFromAsset("moltenvk"))
}
}

task.MaxAttempts = 1
Expand Down
1 change: 1 addition & 0 deletions infra/bots/recipe_modules/build/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
'infra',
'recipe_engine/context',
'recipe_engine/file',
'recipe_engine/path',
'recipe_engine/python',
'recipe_engine/step',
'run',
Expand Down
57 changes: 51 additions & 6 deletions infra/bots/recipe_modules/build/default.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,14 @@
from . import util


# See mapping of Xcode version to Xcode build version here:
# https://chromium.googlesource.com/chromium/tools/build/+/master/scripts/slave/recipe_modules/ios/api.py#37
# When updating XCODE_BUILD_VERSION, you will also need to update
# XCODE_CLANG_VERSION.
XCODE_BUILD_VERSION = '9c40b'
XCODE_CLANG_VERSION = '9.0.0'


def build_command_buffer(api, chrome_dir, skia_dir, out):
api.run(api.python, 'build command_buffer',
script=skia_dir.join('tools', 'build_command_buffer.py'),
Expand Down Expand Up @@ -63,11 +71,28 @@ def compile_fn(api, checkout_root, out_dir):
env = {}

if os == 'Mac':
# In the future we will use CIPD to install Xcode on Mac builders. If we are
# backfilling after a task that used XCode from CIPD, we need to select the
# default Xcode again.
api.step('select xcode', [
'sudo', 'xcode-select', '-switch', '/Applications/Xcode9.2.app'])
extra_cflags.append(
'-DDUMMY_xcode_build_version=%s' % XCODE_BUILD_VERSION)
mac_toolchain_cmd = api.vars.slave_dir.join(
'mac_toolchain', 'mac_toolchain')
xcode_app_path = api.vars.cache_dir.join('Xcode.app')
# Copied from
# https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#322
with api.step.nest('ensure xcode') as step_result:
step_result.presentation.step_text = (
'Ensuring Xcode version %s in %s' % (
XCODE_BUILD_VERSION, xcode_app_path))
install_xcode_cmd = [
mac_toolchain_cmd, 'install',
# "ios" is needed for simulator builds
# (Build-Mac-Clang-x64-Release-iOS).
'-kind', 'ios',
'-xcode-version', XCODE_BUILD_VERSION,
'-output-dir', xcode_app_path,
]
api.step('install xcode', install_xcode_cmd)
api.step('select xcode', [
'sudo', 'xcode-select', '-switch', xcode_app_path])

if compiler == 'Clang' and api.vars.is_linux:
cc = clang_linux + '/bin/clang'
Expand Down Expand Up @@ -270,7 +295,27 @@ def compile_fn(api, checkout_root, out_dir):


def copy_extra_build_products(api, src, dst):
if 'SwiftShader' in api.vars.extra_tokens:
extra_tokens = api.vars.extra_tokens
os = api.vars.builder_cfg.get('os', '')

if 'SwiftShader' in extra_tokens:
util.copy_whitelisted_build_products(api,
src.join('swiftshader_out'),
api.vars.swarming_out_dir.join('swiftshader_out'))

if os == 'Mac' and any('SAN' in t for t in extra_tokens):
# Hardcoding this path because it should only change when we upgrade to a
# new Xcode.
lib_dir = api.vars.cache_dir.join(
'Xcode.app', 'Contents', 'Developer', 'Toolchains',
'XcodeDefault.xctoolchain', 'usr', 'lib', 'clang', XCODE_CLANG_VERSION,
'lib', 'darwin')
dylibs = api.file.glob_paths('find xSAN dylibs', lib_dir,
'libclang_rt.*san_osx_dynamic.dylib',
test_data=[
'libclang_rt.asan_osx_dynamic.dylib',
'libclang_rt.tsan_osx_dynamic.dylib',
'libclang_rt.ubsan_osx_dynamic.dylib',
])
for f in dylibs:
api.file.copy('copy %s' % api.path.basename(f), f, dst)
Original file line number Diff line number Diff line change
@@ -1,12 +1,38 @@
[
{
"cmd": [],
"name": "ensure xcode",
"~followup_annotations": [
"@@@STEP_TEXT@Ensuring Xcode version 9c40b in [START_DIR]/cache/Xcode.app@@@"
]
},
{
"cmd": [
"[START_DIR]/mac_toolchain/mac_toolchain",
"install",
"-kind",
"ios",
"-xcode-version",
"9c40b",
"-output-dir",
"[START_DIR]/cache/Xcode.app"
],
"name": "ensure xcode.install xcode",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"sudo",
"xcode-select",
"-switch",
"/Applications/Xcode9.2.app"
"[START_DIR]/cache/Xcode.app"
],
"name": "select xcode"
"name": "ensure xcode.select xcode",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
Expand All @@ -27,7 +53,7 @@
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Mac-Clang-arm64-Debug-iOS/Debug",
"--args=cc=\"clang\" cxx=\"clang++\" extra_cflags=[\"-O1\"] skia_ios_identity=\".*GS9WA.*\" skia_ios_profile=\"Upstream Testing Provisioning Profile\" target_cpu=\"arm64\" target_os=\"ios\""
"--args=cc=\"clang\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_xcode_build_version=9c40b\", \"-O1\"] skia_ios_identity=\".*GS9WA.*\" skia_ios_profile=\"Upstream Testing Provisioning Profile\" target_cpu=\"arm64\" target_os=\"ios\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
[
{
"cmd": [],
"name": "ensure xcode",
"~followup_annotations": [
"@@@STEP_TEXT@Ensuring Xcode version 9c40b in [START_DIR]/cache/Xcode.app@@@"
]
},
{
"cmd": [
"[START_DIR]/mac_toolchain/mac_toolchain",
"install",
"-kind",
"ios",
"-xcode-version",
"9c40b",
"-output-dir",
"[START_DIR]/cache/Xcode.app"
],
"name": "ensure xcode.install xcode",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"sudo",
"xcode-select",
"-switch",
"[START_DIR]/cache/Xcode.app"
],
"name": "ensure xcode.select xcode",
"~followup_annotations": [
"@@@STEP_NEST_LEVEL@1@@@"
]
},
{
"cmd": [
"python",
"-u",
"[START_DIR]/cache/work/skia/bin/fetch-gn"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"infra_step": true,
"name": "fetch-gn"
},
{
"cmd": [
"[START_DIR]/cache/work/skia/bin/gn",
"gen",
"[START_DIR]/cache/work/skia/out/Build-Mac-Clang-x86_64-Debug-ASAN/Debug",
"--args=cc=\"clang\" cxx=\"clang++\" extra_cflags=[\"-DDUMMY_xcode_build_version=9c40b\", \"-O1\"] sanitize=\"ASAN\" skia_enable_spirv_validation=false target_cpu=\"x86_64\""
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "gn gen"
},
{
"cmd": [
"ninja",
"-k",
"0",
"-C",
"[START_DIR]/cache/work/skia/out/Build-Mac-Clang-x86_64-Debug-ASAN/Debug"
],
"cwd": "[START_DIR]/cache/work/skia",
"env": {
"CHROME_HEADLESS": "1",
"PATH": "<PATH>:RECIPE_PACKAGE_REPO[depot_tools]"
},
"name": "ninja"
},
{
"cmd": [
"python",
"-u",
"import errno\nimport glob\nimport os\nimport shutil\nimport sys\n\nsrc = sys.argv[1]\ndst = sys.argv[2]\nbuild_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']\n\ntry:\n os.makedirs(dst)\nexcept OSError as e:\n if e.errno != errno.EEXIST:\n raise\n\nfor pattern in build_products_whitelist:\n path = os.path.join(src, pattern)\n for f in glob.glob(path):\n dst_path = os.path.join(dst, os.path.relpath(f, src))\n if not os.path.isdir(os.path.dirname(dst_path)):\n os.makedirs(os.path.dirname(dst_path))\n print 'Copying build product %s to %s' % (f, dst_path)\n shutil.move(f, dst_path)\n",
"[START_DIR]/cache/work/skia/out/Build-Mac-Clang-x86_64-Debug-ASAN/Debug",
"[START_DIR]/[SWARM_OUT_DIR]/out/Debug"
],
"infra_step": true,
"name": "copy build products",
"~followup_annotations": [
"@@@STEP_LOG_LINE@python.inline@import errno@@@",
"@@@STEP_LOG_LINE@python.inline@import glob@@@",
"@@@STEP_LOG_LINE@python.inline@import os@@@",
"@@@STEP_LOG_LINE@python.inline@import shutil@@@",
"@@@STEP_LOG_LINE@python.inline@import sys@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@src = sys.argv[1]@@@",
"@@@STEP_LOG_LINE@python.inline@dst = sys.argv[2]@@@",
"@@@STEP_LOG_LINE@python.inline@build_products_whitelist = ['bookmaker', 'dm', 'dm.exe', 'dm.app', 'nanobench.app', 'get_images_from_skps', 'get_images_from_skps.exe', 'hello-opencl', 'hello-opencl.exe', 'nanobench', 'nanobench.exe', 'skpbench', 'skpbench.exe', '*.so', '*.dll', '*.dylib', 'skia_launcher', 'skiaserve', 'lib/*.so', 'run_testlab', 'skqp-universal-debug.apk', 'whitelist_devices.json']@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@try:@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(dst)@@@",
"@@@STEP_LOG_LINE@python.inline@except OSError as e:@@@",
"@@@STEP_LOG_LINE@python.inline@ if e.errno != errno.EEXIST:@@@",
"@@@STEP_LOG_LINE@python.inline@ raise@@@",
"@@@STEP_LOG_LINE@python.inline@@@@",
"@@@STEP_LOG_LINE@python.inline@for pattern in build_products_whitelist:@@@",
"@@@STEP_LOG_LINE@python.inline@ path = os.path.join(src, pattern)@@@",
"@@@STEP_LOG_LINE@python.inline@ for f in glob.glob(path):@@@",
"@@@STEP_LOG_LINE@python.inline@ dst_path = os.path.join(dst, os.path.relpath(f, src))@@@",
"@@@STEP_LOG_LINE@python.inline@ if not os.path.isdir(os.path.dirname(dst_path)):@@@",
"@@@STEP_LOG_LINE@python.inline@ os.makedirs(os.path.dirname(dst_path))@@@",
"@@@STEP_LOG_LINE@python.inline@ print 'Copying build product %s to %s' % (f, dst_path)@@@",
"@@@STEP_LOG_LINE@python.inline@ shutil.move(f, dst_path)@@@",
"@@@STEP_LOG_END@python.inline@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"glob",
"[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin",
"libclang_rt.*san_osx_dynamic.dylib"
],
"infra_step": true,
"name": "find xSAN dylibs",
"stdout": "/path/to/tmp/",
"~followup_annotations": [
"@@@STEP_LOG_LINE@glob@[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib@@@",
"@@@STEP_LOG_LINE@glob@[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib@@@",
"@@@STEP_LOG_LINE@glob@[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.ubsan_osx_dynamic.dylib@@@",
"@@@STEP_LOG_END@glob@@@"
]
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.asan_osx_dynamic.dylib",
"[START_DIR]/[SWARM_OUT_DIR]/out/Debug"
],
"infra_step": true,
"name": "copy libclang_rt.asan_osx_dynamic.dylib"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.tsan_osx_dynamic.dylib",
"[START_DIR]/[SWARM_OUT_DIR]/out/Debug"
],
"infra_step": true,
"name": "copy libclang_rt.tsan_osx_dynamic.dylib"
},
{
"cmd": [
"python",
"-u",
"RECIPE_MODULE[recipe_engine::file]/resources/fileutil.py",
"--json-output",
"/path/to/tmp/json",
"copy",
"[START_DIR]/cache/Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/clang/9.0.0/lib/darwin/libclang_rt.ubsan_osx_dynamic.dylib",
"[START_DIR]/[SWARM_OUT_DIR]/out/Debug"
],
"infra_step": true,
"name": "copy libclang_rt.ubsan_osx_dynamic.dylib"
},
{
"name": "$result",
"recipe_result": null,
"status_code": 0
}
]
Loading

0 comments on commit cdfa16d

Please sign in to comment.