tools: update gyp to 25ed9ac

Includes improved support for VS 2015[0] and makes it possible to build
with ninja again[1].

[0] https://codereview.chromium.org/1112753003
[1] https://codereview.chromium.org/1209553002

Fixes: #2065
PR-URL: #2074
Reviewed-By: Johan Bergström <bugs@bergstroem.nu>
Reviewed-By: Shigeki Ohtsu <ohtsu@iij.ad.jp>

tools/gyp/PRESUBMIT.py

@@ -129,7 +129,6 @@ def CheckChangeOnCommit(input_api, output_api):
  'gyp-win64',
  'gyp-linux',
  'gyp-mac',
- 'gyp-android'
 ]
 
 

tools/gyp/buildbot/buildbot_run.py

@@ -3,27 +3,17 @@
 # Use of this source code is governed by a BSD-style license that can be
 # found in the LICENSE file.
 
-
 """Argument-less script to select what to run on the buildbots."""
 
-
-import filecmp
 import os
 import shutil
 import subprocess
 import sys
 
 
-if sys.platform in ['win32', 'cygwin']:
- EXE_SUFFIX = '.exe'
-else:
- EXE_SUFFIX = ''
-
-
 BUILDBOT_DIR = os.path.dirname(os.path.abspath(__file__))
 TRUNK_DIR = os.path.dirname(BUILDBOT_DIR)
 ROOT_DIR = os.path.dirname(TRUNK_DIR)
-ANDROID_DIR = os.path.join(ROOT_DIR, 'android')
 CMAKE_DIR = os.path.join(ROOT_DIR, 'cmake')
 CMAKE_BIN_DIR = os.path.join(CMAKE_DIR, 'bin')
 OUT_DIR = os.path.join(TRUNK_DIR, 'out')
@@ -71,95 +61,6 @@ def PrepareCmake():
  CallSubProcess( ['make', 'cmake'], cwd=CMAKE_DIR)
 
 
-_ANDROID_SETUP = 'source build/envsetup.sh && lunch full-eng'
-
-
-def PrepareAndroidTree():
- """Prepare an Android tree to run 'android' format tests."""
- if os.environ['BUILDBOT_CLOBBER'] == '1':
- print '@@@BUILD_STEP Clobber Android checkout@@@'
- shutil.rmtree(ANDROID_DIR)
-
- # (Re)create the directory so that the following steps will succeed.
- if not os.path.isdir(ANDROID_DIR):
- os.mkdir(ANDROID_DIR)
-
- # We use a manifest from the gyp project listing pinned revisions of AOSP to
- # use, to ensure that we test against a stable target. This needs to be
- # updated to pick up new build system changes sometimes, so we must test if
- # it has changed.
- manifest_filename = 'aosp_manifest.xml'
- gyp_manifest = os.path.join(BUILDBOT_DIR, manifest_filename)
- android_manifest = os.path.join(ANDROID_DIR, '.repo', 'manifests',
- manifest_filename)
- manifest_is_current = (os.path.isfile(android_manifest) and
- filecmp.cmp(gyp_manifest, android_manifest))
- if not manifest_is_current:
- # It's safe to repeat these steps, so just do them again to make sure we are
- # in a good state.
- print '@@@BUILD_STEP Initialize Android checkout@@@'
- CallSubProcess(
- ['repo', 'init',
- '-u', 'https://android.googlesource.com/platform/manifest',
- '-b', 'master',
- '-g', 'all,-notdefault,-device,-darwin,-mips,-x86'],
- cwd=ANDROID_DIR)
- shutil.copy(gyp_manifest, android_manifest)
-
- print '@@@BUILD_STEP Sync Android@@@'
- CallSubProcess(['repo', 'sync', '-j4', '-m', manifest_filename],
- cwd=ANDROID_DIR)
-
- # If we already built the system image successfully and didn't sync to a new
- # version of the source, skip running the build again as it's expensive even
- # when there's nothing to do.
- system_img = os.path.join(ANDROID_DIR, 'out', 'target', 'product', 'generic',
- 'system.img')
- if manifest_is_current and os.path.isfile(system_img):
- return
-
- print '@@@BUILD_STEP Build Android@@@'
- CallSubProcess(
- ['/bin/bash',
- '-c', '%s && make -j4' % _ANDROID_SETUP],
- cwd=ANDROID_DIR)
-
-
-def StartAndroidEmulator():
- """Start an android emulator from the built android tree."""
- print '@@@BUILD_STEP Start Android emulator@@@'
-
- CallSubProcess(['/bin/bash', '-c',
- '%s && adb kill-server ' % _ANDROID_SETUP],
- cwd=ANDROID_DIR)
-
- # If taskset is available, use it to force adbd to run only on one core, as,
- # sadly, it improves its reliability (see crbug.com/268450).
- adbd_wrapper = ''
- with open(os.devnull, 'w') as devnull_fd:
- if subprocess.call(['which', 'taskset'], stdout=devnull_fd) == 0:
- adbd_wrapper = 'taskset -c 0'
- CallSubProcess(['/bin/bash', '-c',
- '%s && %s adb start-server ' % (_ANDROID_SETUP, adbd_wrapper)],
- cwd=ANDROID_DIR)
-
- subprocess.Popen(
- ['/bin/bash', '-c',
- '%s && emulator -no-window' % _ANDROID_SETUP],
- cwd=ANDROID_DIR)
- CallSubProcess(
- ['/bin/bash', '-c',
- '%s && adb wait-for-device' % _ANDROID_SETUP],
- cwd=ANDROID_DIR)
-
-
-def StopAndroidEmulator():
- """Stop all android emulators."""
- print '@@@BUILD_STEP Stop Android emulator@@@'
- # If this fails, it's because there is no emulator running.
- subprocess.call(['pkill', 'emulator.*'])
-
-
 def GypTestFormat(title, format=None, msvs_version=None, tests=[]):
  """Run the gyp tests for a given format, emitting annotator tags.
 
@@ -185,15 +86,7 @@ def GypTestFormat(title, format=None, msvs_version=None, tests=[]):
  '--format', format,
  '--path', CMAKE_BIN_DIR,
  '--chdir', 'gyp'] + tests)
- if format == 'android':
- # gyptest needs the environment setup from envsetup/lunch in order to build
- # using the 'android' backend, so this is done in a single shell.
- retcode = subprocess.call(
- ['/bin/bash',
- '-c', '%s && cd %s && %s' % (_ANDROID_SETUP, ROOT_DIR, command)],
- cwd=ANDROID_DIR, env=env)
- else:
- retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True)
+ retcode = subprocess.call(command, cwd=ROOT_DIR, env=env, shell=True)
  if retcode:
  # Emit failure tag, and keep going.
  print '@@@STEP_FAILURE@@@'
@@ -209,15 +102,7 @@ def GypBuild():
  print 'Done.'
 
  retcode = 0
- # The Android gyp bot runs on linux so this must be tested first.
- if os.environ['BUILDBOT_BUILDERNAME'] == 'gyp-android':
- PrepareAndroidTree()
- StartAndroidEmulator()
- try:
- retcode += GypTestFormat('android')
- finally:
- StopAndroidEmulator()
- elif sys.platform.startswith('linux'):
+ if sys.platform.startswith('linux'):
  retcode += GypTestFormat('ninja')
  retcode += GypTestFormat('make')
  PrepareCmake()

tools/gyp/buildbot/commit_queue/cq_config.json

@@ -3,7 +3,6 @@
  "launched": {
  "tryserver.nacl": {
  "gyp-presubmit": ["defaulttests"],
- "gyp-android": ["defaulttests"],
  "gyp-linux": ["defaulttests"],
  "gyp-mac": ["defaulttests"],
  "gyp-win32": ["defaulttests"],

tools/gyp/pylib/gyp/MSVSVersion.py

@@ -84,10 +84,11 @@ def SetupScript(self, target_arch):
  # vcvars32, which it can only find if VS??COMNTOOLS is set, which it
  # isn't always.
  if target_arch == 'x86':
- if self.short_name == '2013' and (
+ if self.short_name >= '2013' and self.short_name[-1] != 'e' and (
  os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or
  os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'):
- # VS2013 non-Express has a x64-x86 cross that we want to prefer.
+ # VS2013 and later, non-Express have a x64-x86 cross that we want
+ # to prefer.
  return [os.path.normpath(
  os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86']
  # Otherwise, the standard x86 compiler.

tools/gyp/pylib/gyp/__init__.py

@@ -49,7 +49,7 @@ def FindBuildFiles():
 
 def Load(build_files, format, default_variables={},
  includes=[], depth='.', params=None, check=False,
- circular_check=True):
+ circular_check=True, duplicate_basename_check=True):
  """
  Loads one or more specified build files.
  default_variables and includes will be copied before use.
@@ -126,6 +126,7 @@ def Load(build_files, format, default_variables={},
  # Process the input specific to this generator.
  result = gyp.input.Load(build_files, default_variables, includes[:],
  depth, generator_input_info, check, circular_check,
+ duplicate_basename_check,
  params['parallel'], params['root_targets'])
  return [generator] + result
 
@@ -324,6 +325,16 @@ def gyp_main(args):
  parser.add_option('--no-circular-check', dest='circular_check',
  action='store_false', default=True, regenerate=False,
  help="don't check for circular relationships between files")
+ # --no-duplicate-basename-check disables the check for duplicate basenames
+ # in a static_library/shared_library project. Visual C++ 2008 generator
+ # doesn't support this configuration. Libtool on Mac also generates warnings
+ # when duplicate basenames are passed into Make generator on Mac.
+ # TODO(yukawa): Remove this option when these legacy generators are
+ # deprecated.
+ parser.add_option('--no-duplicate-basename-check',
+ dest='duplicate_basename_check', action='store_false',
+ default=True, regenerate=False,
+ help="don't check for duplicate basenames")
  parser.add_option('--no-parallel', action='store_true', default=False,
  help='Disable multiprocessing')
  parser.add_option('-S', '--suffix', dest='suffix', default='',
@@ -499,7 +510,8 @@ def gyp_main(args):
  # Start with the default variables from the command line.
  [generator, flat_list, targets, data] = Load(
  build_files, format, cmdline_default_variables, includes, options.depth,
- params, options.check, options.circular_check)
+ params, options.check, options.circular_check,
+ options.duplicate_basename_check)
 
  # TODO(mark): Pass |data| for now because the generator needs a list of
  # build files that came in. In the future, maybe it should just accept

tools/gyp/pylib/gyp/generator/analyzer.py

@@ -183,7 +183,10 @@ class Target(object):
  added_to_compile_targets: used when determining if the target was added to the
  set of targets that needs to be built.
  in_roots: true if this target is a descendant of one of the root nodes.
- is_executable: true if the type of target is executable."""
+ is_executable: true if the type of target is executable.
+ is_static_library: true if the type of target is static_library.
+ is_or_has_linked_ancestor: true if the target does a link (eg executable), or
+ if there is a target in back_deps that does a link."""
  def __init__(self, name):
  self.deps = set()
  self.match_status = MATCH_STATUS_TBD
@@ -196,6 +199,8 @@ def __init__(self, name):
  self.added_to_compile_targets = False
  self.in_roots = False
  self.is_executable = False
+ self.is_static_library = False
+ self.is_or_has_linked_ancestor = False
 
 
 class Config(object):
@@ -266,8 +271,8 @@ def _GetOrCreateTargetByName(targets, target_name):
 def _DoesTargetTypeRequireBuild(target_dict):
  """Returns true if the target type is such that it needs to be built."""
  # If a 'none' target has rules or actions we assume it requires a build.
- return target_dict['type'] != 'none' or \
- target_dict.get('actions') or target_dict.get('rules')
+ return bool(target_dict['type'] != 'none' or
+  target_dict.get('actions') or target_dict.get('rules'))
 
 
 def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files,
@@ -309,7 +314,11 @@ def _GenerateTargets(data, target_list, target_dicts, toplevel_dir, files,
  target.visited = True
  target.requires_build = _DoesTargetTypeRequireBuild(
  target_dicts[target_name])
- target.is_executable = target_dicts[target_name]['type'] == 'executable'
+ target_type = target_dicts[target_name]['type']
+ target.is_executable = target_type == 'executable'
+ target.is_static_library = target_type == 'static_library'
+ target.is_or_has_linked_ancestor = (target_type == 'executable' or
+ target_type == 'shared_library')
 
  build_file = gyp.common.ParseQualifiedTarget(target_name)[0]
  if not build_file in build_file_in_files:
@@ -378,6 +387,7 @@ def _DoesTargetDependOn(target):
  for dep in target.deps:
  if _DoesTargetDependOn(dep):
  target.match_status = MATCH_STATUS_MATCHES_BY_DEPENDENCY
+ print '\t', target.name, 'matches by dep', dep.name
  return True
  target.match_status = MATCH_STATUS_DOESNT_MATCH
  return False
@@ -388,6 +398,7 @@ def _GetTargetsDependingOn(possible_targets):
  directly on indirectly) on the matched targets.
  possible_targets: targets to search from."""
  found = []
+ print 'Targets that matched by dependency:'
  for target in possible_targets:
  if _DoesTargetDependOn(target):
  found.append(target)
@@ -411,14 +422,27 @@ def _AddBuildTargets(target, roots, add_if_no_ancestor, result):
  _AddBuildTargets(back_dep_target, roots, False, result)
  target.added_to_compile_targets |= back_dep_target.added_to_compile_targets
  target.in_roots |= back_dep_target.in_roots
+ target.is_or_has_linked_ancestor |= (
+ back_dep_target.is_or_has_linked_ancestor)
 
  # Always add 'executable' targets. Even though they may be built by other
  # targets that depend upon them it makes detection of what is going to be
  # built easier.
+ # And always add static_libraries that have no dependencies on them from
+ # linkables. This is necessary as the other dependencies on them may be
+ # static libraries themselves, which are not compile time dependencies.
  if target.in_roots and \
  (target.is_executable or
  (not target.added_to_compile_targets and
- (add_if_no_ancestor or target.requires_build))):
+ (add_if_no_ancestor or target.requires_build)) or
+ (target.is_static_library and add_if_no_ancestor and
+ not target.is_or_has_linked_ancestor)):
+ print '\t\tadding to build targets', target.name, 'executable', \
+ target.is_executable, 'added_to_compile_targets', \
+ target.added_to_compile_targets, 'add_if_no_ancestor', \
+ add_if_no_ancestor, 'requires_build', target.requires_build, \
+ 'is_static_library', target.is_static_library, \
+ 'is_or_has_linked_ancestor', target.is_or_has_linked_ancestor
  result.add(target)
  target.added_to_compile_targets = True
 
@@ -429,6 +453,7 @@ def _GetBuildTargets(matching_targets, roots):
  roots: set of root targets in the build files to search from."""
  result = set()
  for target in matching_targets:
+ print '\tfinding build targets for match', target.name
  _AddBuildTargets(target, roots, True, result)
  return result
 
@@ -536,6 +561,10 @@ def GenerateOutput(target_list, target_dicts, data, params):
  data, target_list, target_dicts, toplevel_dir, frozenset(config.files),
  params['build_files'])
 
+ print 'roots:'
+ for root in roots:
+ print '\t', root.name
+
  unqualified_mapping = _GetUnqualifiedToTargetMapping(all_targets,
  config.targets)
  invalid_targets = None
@@ -544,10 +573,20 @@ def GenerateOutput(target_list, target_dicts, data, params):
 
  if matching_targets:
  search_targets = _LookupTargets(config.targets, unqualified_mapping)
+ print 'supplied targets'
+ for target in config.targets:
+ print '\t', target
+ print 'expanded supplied targets'
+ for target in search_targets:
+ print '\t', target.name
  matched_search_targets = _GetTargetsDependingOn(search_targets)
+ print 'raw matched search targets:'
+ for target in matched_search_targets:
+ print '\t', target.name
  # Reset the visited status for _GetBuildTargets.
  for target in all_targets.itervalues():
  target.visited = False
+ print 'Finding build targets'
  build_targets = _GetBuildTargets(matching_targets, roots)
  matched_search_targets = [gyp.common.ParseQualifiedTarget(target.name)[1]
  for target in matched_search_targets]