From 8f34b834b745fa06105a2f138318156b3c9114f9 Mon Sep 17 00:00:00 2001 From: Refael Ackermann Date: Wed, 9 Aug 2017 14:43:18 -0400 Subject: [PATCH] tools: update GYP to 324dd166 PR-URL: https://github.com/nodejs/node/pull/14718 Refs: https://chromium.googlesource.com/external/gyp/+/324dd166b7c0b39d513026fa52d6280ac6d56770 Refs: https://github.com/refack/GYP/commit/324dd166b7c0b39d513026fa52d6280ac6d56770 Reviewed-By: James M Snell --- tools/gyp/gyptest.py | 380 ++++++++---------- tools/gyp/pylib/gyp/MSVSVersion.py | 76 ++-- tools/gyp/pylib/gyp/easy_xml.py | 5 + .../gyp/generator/compile_commands_json.py | 115 ------ tools/gyp/pylib/gyp/generator/make.py | 10 +- tools/gyp/pylib/gyp/generator/msvs.py | 29 +- tools/gyp/pylib/gyp/generator/ninja.py | 9 +- tools/gyp/pylib/gyp/mac_tool.py | 14 +- tools/gyp/pylib/gyp/msvs_emulation.py | 22 +- tools/gyp/pylib/gyp/xcode_emulation.py | 2 - 10 files changed, 279 insertions(+), 383 deletions(-) delete mode 100644 tools/gyp/pylib/gyp/generator/compile_commands_json.py diff --git a/tools/gyp/gyptest.py b/tools/gyp/gyptest.py index 98957e16338c1d..9930e78c7b0f1c 100755 --- a/tools/gyp/gyptest.py +++ b/tools/gyp/gyptest.py @@ -1,134 +1,19 @@ #!/usr/bin/env python - # Copyright (c) 2012 Google Inc. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -__doc__ = """ -gyptest.py -- test runner for GYP tests. -""" +"""gyptest.py -- test runner for GYP tests.""" + +from __future__ import print_function +import argparse +import math import os -import optparse -import shlex +import platform import subprocess import sys - -class CommandRunner(object): - """ - Executor class for commands, including "commands" implemented by - Python functions. - """ - verbose = True - active = True - - def __init__(self, dictionary={}): - self.subst_dictionary(dictionary) - - def subst_dictionary(self, dictionary): - self._subst_dictionary = dictionary - - def subst(self, string, dictionary=None): - """ - Substitutes (via the format operator) the values in the specified - dictionary into the specified command. - - The command can be an (action, string) tuple. In all cases, we - perform substitution on strings and don't worry if something isn't - a string. (It's probably a Python function to be executed.) - """ - if dictionary is None: - dictionary = self._subst_dictionary - if dictionary: - try: - string = string % dictionary - except TypeError: - pass - return string - - def display(self, command, stdout=None, stderr=None): - if not self.verbose: - return - if type(command) == type(()): - func = command[0] - args = command[1:] - s = '%s(%s)' % (func.__name__, ', '.join(map(repr, args))) - if type(command) == type([]): - # TODO: quote arguments containing spaces - # TODO: handle meta characters? - s = ' '.join(command) - else: - s = self.subst(command) - if not s.endswith('\n'): - s += '\n' - sys.stdout.write(s) - sys.stdout.flush() - - def execute(self, command, stdout=None, stderr=None): - """ - Executes a single command. - """ - if not self.active: - return 0 - if type(command) == type(''): - command = self.subst(command) - cmdargs = shlex.split(command) - if cmdargs[0] == 'cd': - command = (os.chdir,) + tuple(cmdargs[1:]) - if type(command) == type(()): - func = command[0] - args = command[1:] - return func(*args) - else: - if stdout is sys.stdout: - # Same as passing sys.stdout, except python2.4 doesn't fail on it. - subout = None - else: - # Open pipe for anything else so Popen works on python2.4. - subout = subprocess.PIPE - if stderr is sys.stderr: - # Same as passing sys.stderr, except python2.4 doesn't fail on it. - suberr = None - elif stderr is None: - # Merge with stdout if stderr isn't specified. - suberr = subprocess.STDOUT - else: - # Open pipe for anything else so Popen works on python2.4. - suberr = subprocess.PIPE - p = subprocess.Popen(command, - shell=(sys.platform == 'win32'), - stdout=subout, - stderr=suberr) - p.wait() - if stdout is None: - self.stdout = p.stdout.read() - elif stdout is not sys.stdout: - stdout.write(p.stdout.read()) - if stderr not in (None, sys.stderr): - stderr.write(p.stderr.read()) - return p.returncode - - def run(self, command, display=None, stdout=None, stderr=None): - """ - Runs a single command, displaying it first. - """ - if display is None: - display = command - self.display(display) - return self.execute(command, stdout, stderr) - - -class Unbuffered(object): - def __init__(self, fp): - self.fp = fp - def write(self, arg): - self.fp.write(arg) - self.fp.flush() - def __getattr__(self, attr): - return getattr(self.fp, attr) - -sys.stdout = Unbuffered(sys.stdout) -sys.stderr = Unbuffered(sys.stderr) +import time def is_test_name(f): @@ -138,8 +23,6 @@ def is_test_name(f): def find_all_gyptest_files(directory): result = [] for root, dirs, files in os.walk(directory): - if '.svn' in dirs: - dirs.remove('.svn') result.extend([ os.path.join(root, f) for f in files if is_test_name(f) ]) result.sort() return result @@ -149,73 +32,68 @@ def main(argv=None): if argv is None: argv = sys.argv - usage = "gyptest.py [-ahlnq] [-f formats] [test ...]" - parser = optparse.OptionParser(usage=usage) - parser.add_option("-a", "--all", action="store_true", - help="run all tests") - parser.add_option("-C", "--chdir", action="store", default=None, - help="chdir to the specified directory") - parser.add_option("-f", "--format", action="store", default='', - help="run tests with the specified formats") - parser.add_option("-G", '--gyp_option', action="append", default=[], - help="Add -G options to the gyp command line") - parser.add_option("-l", "--list", action="store_true", - help="list available tests and exit") - parser.add_option("-n", "--no-exec", action="store_true", - help="no execute, just print the command line") - parser.add_option("--passed", action="store_true", - help="report passed tests") - parser.add_option("--path", action="append", default=[], - help="additional $PATH directory") - parser.add_option("-q", "--quiet", action="store_true", - help="quiet, don't print test command lines") - opts, args = parser.parse_args(argv[1:]) - - if opts.chdir: - os.chdir(opts.chdir) - - if opts.path: + parser = argparse.ArgumentParser() + parser.add_argument("-a", "--all", action="store_true", + help="run all tests") + parser.add_argument("-C", "--chdir", action="store", + help="change to directory") + parser.add_argument("-f", "--format", action="store", default='', + help="run tests with the specified formats") + parser.add_argument("-G", '--gyp_option', action="append", default=[], + help="Add -G options to the gyp command line") + parser.add_argument("-l", "--list", action="store_true", + help="list available tests and exit") + parser.add_argument("-n", "--no-exec", action="store_true", + help="no execute, just print the command line") + parser.add_argument("--path", action="append", default=[], + help="additional $PATH directory") + parser.add_argument("-q", "--quiet", action="store_true", + help="quiet, don't print anything unless there are failures") + parser.add_argument("-v", "--verbose", action="store_true", + help="print configuration info and test results.") + parser.add_argument('tests', nargs='*') + args = parser.parse_args(argv[1:]) + + if args.chdir: + os.chdir(args.chdir) + + if args.path: extra_path = [os.path.abspath(p) for p in opts.path] extra_path = os.pathsep.join(extra_path) os.environ['PATH'] = extra_path + os.pathsep + os.environ['PATH'] - if not args: - if not opts.all: + if not args.tests: + if not args.all: sys.stderr.write('Specify -a to get all tests.\n') return 1 - args = ['test'] + args.tests = ['test'] tests = [] - for arg in args: + for arg in args.tests: if os.path.isdir(arg): tests.extend(find_all_gyptest_files(os.path.normpath(arg))) else: if not is_test_name(os.path.basename(arg)): - print >>sys.stderr, arg, 'is not a valid gyp test name.' + print(arg, 'is not a valid gyp test name.', file=sys.stderr) sys.exit(1) tests.append(arg) - if opts.list: + if args.list: for test in tests: - print test + print(test) sys.exit(0) - CommandRunner.verbose = not opts.quiet - CommandRunner.active = not opts.no_exec - cr = CommandRunner() - os.environ['PYTHONPATH'] = os.path.abspath('test/lib') - if not opts.quiet: - sys.stdout.write('PYTHONPATH=%s\n' % os.environ['PYTHONPATH']) - passed = [] - failed = [] - no_result = [] + if args.verbose: + print_configuration_info() + + if args.gyp_option and not args.quiet: + print('Extra Gyp options: %s\n' % args.gyp_option) - if opts.format: - format_list = opts.format.split(',') + if args.format: + format_list = args.format.split(',') else: - # TODO: not duplicate this mapping from pylib/gyp/__init__.py format_list = { 'aix5': ['make'], 'freebsd7': ['make'], @@ -223,53 +101,143 @@ def main(argv=None): 'openbsd5': ['make'], 'cygwin': ['msvs'], 'win32': ['msvs', 'ninja'], + 'linux': ['make', 'ninja'], 'linux2': ['make', 'ninja'], 'linux3': ['make', 'ninja'], - 'darwin': ['make', 'ninja', 'xcode', 'xcode-ninja'], + + # TODO: Re-enable xcode-ninja. + # https://bugs.chromium.org/p/gyp/issues/detail?id=530 + # 'darwin': ['make', 'ninja', 'xcode', 'xcode-ninja'], + 'darwin': ['make', 'ninja', 'xcode'], }[sys.platform] - for format in format_list: - os.environ['TESTGYP_FORMAT'] = format - if not opts.quiet: - sys.stdout.write('TESTGYP_FORMAT=%s\n' % format) + gyp_options = [] + for option in args.gyp_option: + gyp_options += ['-G', option] - gyp_options = [] - for option in opts.gyp_option: - gyp_options += ['-G', option] - if gyp_options and not opts.quiet: - sys.stdout.write('Extra Gyp options: %s\n' % gyp_options) + runner = Runner(format_list, tests, gyp_options, args.verbose) + runner.run() - for test in tests: - status = cr.run([sys.executable, test] + gyp_options, - stdout=sys.stdout, - stderr=sys.stderr) - if status == 2: - no_result.append(test) - elif status: - failed.append(test) - else: - passed.append(test) - - if not opts.quiet: - def report(description, tests): - if tests: - if len(tests) == 1: - sys.stdout.write("\n%s the following test:\n" % description) - else: - fmt = "\n%s the following %d tests:\n" - sys.stdout.write(fmt % (description, len(tests))) - sys.stdout.write("\t" + "\n\t".join(tests) + "\n") - - if opts.passed: - report("Passed", passed) - report("Failed", failed) - report("No result from", no_result) - - if failed: + if not args.quiet: + runner.print_results() + + if runner.failures: return 1 else: return 0 +def print_configuration_info(): + print('Test configuration:') + if sys.platform == 'darwin': + sys.path.append(os.path.abspath('test/lib')) + import TestMac + print(' Mac %s %s' % (platform.mac_ver()[0], platform.mac_ver()[2])) + print(' Xcode %s' % TestMac.Xcode.Version()) + elif sys.platform == 'win32': + sys.path.append(os.path.abspath('pylib')) + import gyp.MSVSVersion + print(' Win %s %s\n' % platform.win32_ver()[0:2]) + print(' MSVS %s' % + gyp.MSVSVersion.SelectVisualStudioVersion().Description()) + elif sys.platform in ('linux', 'linux2'): + print(' Linux %s' % ' '.join(platform.linux_distribution())) + print(' Python %s' % platform.python_version()) + print(' PYTHONPATH=%s' % os.environ['PYTHONPATH']) + print() + + +class Runner(object): + def __init__(self, formats, tests, gyp_options, verbose): + self.formats = formats + self.tests = tests + self.verbose = verbose + self.gyp_options = gyp_options + self.failures = [] + self.num_tests = len(formats) * len(tests) + num_digits = len(str(self.num_tests)) + self.fmt_str = '[%%%dd/%%%dd] (%%s) %%s' % (num_digits, num_digits) + self.isatty = sys.stdout.isatty() and not self.verbose + self.env = os.environ.copy() + self.hpos = 0 + + def run(self): + run_start = time.time() + + i = 1 + for fmt in self.formats: + for test in self.tests: + self.run_test(test, fmt, i) + i += 1 + + if self.isatty: + self.erase_current_line() + + self.took = time.time() - run_start + + def run_test(self, test, fmt, i): + if self.isatty: + self.erase_current_line() + + msg = self.fmt_str % (i, self.num_tests, fmt, test) + self.print_(msg) + + start = time.time() + cmd = [sys.executable, test] + self.gyp_options + self.env['TESTGYP_FORMAT'] = fmt + proc = subprocess.Popen(cmd, stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, env=self.env) + proc.wait() + took = time.time() - start + + stdout = proc.stdout.read().decode('utf8') + if proc.returncode == 2: + res = 'skipped' + elif proc.returncode: + res = 'failed' + self.failures.append('(%s) %s' % (test, fmt)) + else: + res = 'passed' + res_msg = ' %s %.3fs' % (res, took) + self.print_(res_msg) + + if (stdout and + not stdout.endswith('PASSED\n') and + not (stdout.endswith('NO RESULT\n'))): + print() + for l in stdout.splitlines(): + print(' %s' % l) + elif not self.isatty: + print() + + def print_(self, msg): + print(msg, end='') + index = msg.rfind('\n') + if index == -1: + self.hpos += len(msg) + else: + self.hpos = len(msg) - index + sys.stdout.flush() + + def erase_current_line(self): + print('\b' * self.hpos + ' ' * self.hpos + '\b' * self.hpos, end='') + sys.stdout.flush() + self.hpos = 0 + + def print_results(self): + num_failures = len(self.failures) + if num_failures: + print() + if num_failures == 1: + print("Failed the following test:") + else: + print("Failed the following %d tests:" % num_failures) + print("\t" + "\n\t".join(sorted(self.failures))) + print() + print('Ran %d tests in %.3fs, %d failed.' % (self.num_tests, self.took, + num_failures)) + print() + + if __name__ == "__main__": sys.exit(main()) diff --git a/tools/gyp/pylib/gyp/MSVSVersion.py b/tools/gyp/pylib/gyp/MSVSVersion.py index c981e64b878122..44b958d5b3d22e 100644 --- a/tools/gyp/pylib/gyp/MSVSVersion.py +++ b/tools/gyp/pylib/gyp/MSVSVersion.py @@ -13,6 +13,10 @@ import glob +def JoinPath(*args): + return os.path.normpath(os.path.join(*args)) + + class VisualStudioVersion(object): """Information regarding a version of Visual Studio.""" @@ -71,45 +75,59 @@ def DefaultToolset(self): of a user override.""" return self.default_toolset + def _SetupScriptInternal(self, target_arch): """Returns a command (with arguments) to be used to set up the environment.""" + assert target_arch in ('x86', 'x64'), "target_arch not supported" # If WindowsSDKDir is set and SetEnv.Cmd exists then we are using the # depot_tools build tools and should run SetEnv.Cmd to set up the # environment. The check for WindowsSDKDir alone is not sufficient because # this is set by running vcvarsall.bat. - assert target_arch in ('x86', 'x64') - sdk_dir = os.environ.get('WindowsSDKDir') - if sdk_dir: - setup_path = os.path.normpath(os.path.join(sdk_dir, 'Bin/SetEnv.Cmd')) + sdk_dir = os.environ.get('WindowsSDKDir', '') + setup_path = JoinPath(sdk_dir, 'Bin', 'SetEnv.Cmd') if self.sdk_based and sdk_dir and os.path.exists(setup_path): return [setup_path, '/' + target_arch] - else: - # We don't use VC/vcvarsall.bat for x86 because vcvarsall calls - # vcvars32, which it can only find if VS??COMNTOOLS is set, which it - # isn't always. - if target_arch == 'x86': - if self.short_name >= '2013' and self.short_name[-1] != 'e' and ( - os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or - os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'): - # VS2013 and later, non-Express have a x64-x86 cross that we want - # to prefer. - return [os.path.normpath( - os.path.join(self.path, 'VC/vcvarsall.bat')), 'amd64_x86'] - # Otherwise, the standard x86 compiler. - return [os.path.normpath( - os.path.join(self.path, 'Common7/Tools/vsvars32.bat'))] + + is_host_arch_x64 = ( + os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or + os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64' + ) + + # For VS2017 (and newer) it's fairly easy + if self.short_name >= '2017': + script_path = JoinPath(self.path, + 'VC', 'Auxiliary', 'Build', 'vcvarsall.bat') + + # Always use a native executable, cross-compiling if necessary. + host_arch = 'amd64' if is_host_arch_x64 else 'x86' + msvc_target_arch = 'amd64' if target_arch == 'x64' else 'x86' + arg = host_arch + if host_arch != msvc_target_arch: + arg += '_' + msvc_target_arch + + return [script_path, arg] + + # We try to find the best version of the env setup batch. + vcvarsall = JoinPath(self.path, 'VC', 'vcvarsall.bat') + if target_arch == 'x86': + if self.short_name >= '2013' and self.short_name[-1] != 'e' and \ + is_host_arch_x64: + # VS2013 and later, non-Express have a x64-x86 cross that we want + # to prefer. + return [vcvarsall, 'amd64_x86'] else: - assert target_arch == 'x64' - arg = 'x86_amd64' - # Use the 64-on-64 compiler if we're not using an express - # edition and we're running on a 64bit OS. - if self.short_name[-1] != 'e' and ( - os.environ.get('PROCESSOR_ARCHITECTURE') == 'AMD64' or - os.environ.get('PROCESSOR_ARCHITEW6432') == 'AMD64'): - arg = 'amd64' - return [os.path.normpath( - os.path.join(self.path, 'VC/vcvarsall.bat')), arg] + # Otherwise, the standard x86 compiler. We don't use VC/vcvarsall.bat + # for x86 because vcvarsall calls vcvars32, which it can only find if + # VS??COMNTOOLS is set, which isn't guaranteed. + return [JoinPath(self.path, 'Common7', 'Tools', 'vsvars32.bat')] + elif target_arch == 'x64': + arg = 'x86_amd64' + # Use the 64-on-64 compiler if we're not using an express edition and + # we're running on a 64bit OS. + if self.short_name[-1] != 'e' and is_host_arch_x64: + arg = 'amd64' + return [vcvarsall, arg] def SetupScript(self, target_arch): script_data = self._SetupScriptInternal(target_arch) diff --git a/tools/gyp/pylib/gyp/easy_xml.py b/tools/gyp/pylib/gyp/easy_xml.py index bf949b6ac9b6e6..2522efb244d0a3 100644 --- a/tools/gyp/pylib/gyp/easy_xml.py +++ b/tools/gyp/pylib/gyp/easy_xml.py @@ -4,6 +4,7 @@ import re import os +import locale def XmlToString(content, encoding='utf-8', pretty=False): @@ -116,6 +117,10 @@ def WriteXmlIfChanged(content, path, encoding='utf-8', pretty=False, if win32 and os.linesep != '\r\n': xml_string = xml_string.replace('\n', '\r\n') + default_encoding = locale.getdefaultlocale()[1] + if default_encoding and default_encoding.upper() != encoding.upper(): + xml_string = xml_string.decode(default_encoding).encode(encoding) + # Get the old content try: f = open(path, 'r') diff --git a/tools/gyp/pylib/gyp/generator/compile_commands_json.py b/tools/gyp/pylib/gyp/generator/compile_commands_json.py deleted file mode 100644 index 575db63c4e1943..00000000000000 --- a/tools/gyp/pylib/gyp/generator/compile_commands_json.py +++ /dev/null @@ -1,115 +0,0 @@ -# Copyright (c) 2016 Ben Noordhuis . All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import gyp.common -import gyp.xcode_emulation -import json -import os - -generator_additional_non_configuration_keys = [] -generator_additional_path_sections = [] -generator_extra_sources_for_rules = [] -generator_filelist_paths = None -generator_supports_multiple_toolsets = True -generator_wants_sorted_dependencies = False - -# Lifted from make.py. The actual values don't matter much. -generator_default_variables = { - 'CONFIGURATION_NAME': '$(BUILDTYPE)', - 'EXECUTABLE_PREFIX': '', - 'EXECUTABLE_SUFFIX': '', - 'INTERMEDIATE_DIR': '$(obj).$(TOOLSET)/$(TARGET)/geni', - 'PRODUCT_DIR': '$(builddir)', - 'RULE_INPUT_DIRNAME': '%(INPUT_DIRNAME)s', - 'RULE_INPUT_EXT': '$(suffix $<)', - 'RULE_INPUT_NAME': '$(notdir $<)', - 'RULE_INPUT_PATH': '$(abspath $<)', - 'RULE_INPUT_ROOT': '%(INPUT_ROOT)s', - 'SHARED_INTERMEDIATE_DIR': '$(obj)/gen', - 'SHARED_LIB_PREFIX': 'lib', - 'STATIC_LIB_PREFIX': 'lib', - 'STATIC_LIB_SUFFIX': '.a', -} - - -def IsMac(params): - return 'mac' == gyp.common.GetFlavor(params) - - -def CalculateVariables(default_variables, params): - default_variables.setdefault('OS', gyp.common.GetFlavor(params)) - - -def AddCommandsForTarget(cwd, target, params, per_config_commands): - output_dir = params['generator_flags']['output_dir'] - for configuration_name, configuration in target['configurations'].iteritems(): - builddir_name = os.path.join(output_dir, configuration_name) - - if IsMac(params): - xcode_settings = gyp.xcode_emulation.XcodeSettings(target) - cflags = xcode_settings.GetCflags(configuration_name) - cflags_c = xcode_settings.GetCflagsC(configuration_name) - cflags_cc = xcode_settings.GetCflagsCC(configuration_name) - else: - cflags = configuration.get('cflags', []) - cflags_c = configuration.get('cflags_c', []) - cflags_cc = configuration.get('cflags_cc', []) - - cflags_c = cflags + cflags_c - cflags_cc = cflags + cflags_cc - - defines = configuration.get('defines', []) - defines = ['-D' + s for s in defines] - - # TODO(bnoordhuis) Handle generated source files. - sources = target.get('sources', []) - sources = [s for s in sources if s.endswith('.c') or s.endswith('.cc')] - - def resolve(filename): - return os.path.abspath(os.path.join(cwd, filename)) - - # TODO(bnoordhuis) Handle generated header files. - include_dirs = configuration.get('include_dirs', []) - include_dirs = [s for s in include_dirs if not s.startswith('$(obj)')] - includes = ['-I' + resolve(s) for s in include_dirs] - - defines = gyp.common.EncodePOSIXShellList(defines) - includes = gyp.common.EncodePOSIXShellList(includes) - cflags_c = gyp.common.EncodePOSIXShellList(cflags_c) - cflags_cc = gyp.common.EncodePOSIXShellList(cflags_cc) - - commands = per_config_commands.setdefault(configuration_name, []) - for source in sources: - file = resolve(source) - isc = source.endswith('.c') - cc = 'cc' if isc else 'c++' - cflags = cflags_c if isc else cflags_cc - command = ' '.join((cc, defines, includes, cflags, - '-c', gyp.common.EncodePOSIXShellArgument(file))) - commands.append(dict(command=command, directory=output_dir, file=file)) - - -def GenerateOutput(target_list, target_dicts, data, params): - per_config_commands = {} - for qualified_target, target in target_dicts.iteritems(): - build_file, target_name, toolset = ( - gyp.common.ParseQualifiedTarget(qualified_target)) - if IsMac(params): - settings = data[build_file] - gyp.xcode_emulation.MergeGlobalXcodeSettingsToSpec(settings, target) - cwd = os.path.dirname(build_file) - AddCommandsForTarget(cwd, target, params, per_config_commands) - - output_dir = params['generator_flags']['output_dir'] - for configuration_name, commands in per_config_commands.iteritems(): - filename = os.path.join(output_dir, - configuration_name, - 'compile_commands.json') - gyp.common.EnsureDirExists(filename) - fp = open(filename, 'w') - json.dump(commands, fp=fp, indent=0, check_circular=False) - - -def PerformBuild(data, configurations, params): - pass diff --git a/tools/gyp/pylib/gyp/generator/make.py b/tools/gyp/pylib/gyp/generator/make.py index f7f519b3e6b848..e80ebaed78521e 100644 --- a/tools/gyp/pylib/gyp/generator/make.py +++ b/tools/gyp/pylib/gyp/generator/make.py @@ -147,7 +147,7 @@ def CalculateGeneratorInputInfo(params): # special "figure out circular dependencies" flags around the entire # input list during linking. quiet_cmd_link = LINK($(TOOLSET)) $@ -cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) $(LIBS) -Wl,--end-group +cmd_link = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ -Wl,--start-group $(LD_INPUTS) -Wl,--end-group $(LIBS) # We support two kinds of shared objects (.so): # 1) shared_library, which is just bundling together many dependent libraries @@ -2074,10 +2074,10 @@ def CalculateMakefilePath(build_file, base_name): 'AR.target': GetEnvironFallback(('AR_target', 'AR'), '$(AR)'), 'CXX.target': GetEnvironFallback(('CXX_target', 'CXX'), '$(CXX)'), 'LINK.target': GetEnvironFallback(('LINK_target', 'LINK'), '$(LINK)'), - 'CC.host': GetEnvironFallback(('CC_host', 'CC'), 'gcc'), - 'AR.host': GetEnvironFallback(('AR_host', 'AR'), 'ar'), - 'CXX.host': GetEnvironFallback(('CXX_host', 'CXX'), 'g++'), - 'LINK.host': GetEnvironFallback(('LINK_host', 'LINK'), '$(CXX.host)'), + 'CC.host': GetEnvironFallback(('CC_host',), 'gcc'), + 'AR.host': GetEnvironFallback(('AR_host',), 'ar'), + 'CXX.host': GetEnvironFallback(('CXX_host',), 'g++'), + 'LINK.host': GetEnvironFallback(('LINK_host',), '$(CXX.host)'), }) build_file, _, _ = gyp.common.ParseQualifiedTarget(target_list[0]) diff --git a/tools/gyp/pylib/gyp/generator/msvs.py b/tools/gyp/pylib/gyp/generator/msvs.py index ab92979e5c1434..8fe9e5af23dd38 100644 --- a/tools/gyp/pylib/gyp/generator/msvs.py +++ b/tools/gyp/pylib/gyp/generator/msvs.py @@ -306,9 +306,19 @@ def _ConfigWindowsTargetPlatformVersion(config_data, version): continue version = MSVSVersion._RegistryGetValue(key % ver, 'ProductVersion') or '' # Find a matching entry in sdk_dir\include. - names = sorted([x for x in os.listdir(r'%s\include' % sdk_dir) + expected_sdk_dir=r'%s\include' % sdk_dir + names = sorted([x for x in (os.listdir(expected_sdk_dir) + if os.path.isdir(expected_sdk_dir) + else [] + ) if x.startswith(version)], reverse=True) - return names[0] + if names: + return names[0] + else: + print >> sys.stdout, ( + 'Warning: No include files found for ' + 'detected Windows SDK version %s' % (version) + ) def _BuildCommandLineForRuleRaw(spec, cmd, cygwin_shell, has_input_path, @@ -1717,14 +1727,17 @@ def _GetCopies(spec): src_bare = src[:-1] base_dir = posixpath.split(src_bare)[0] outer_dir = posixpath.split(src_bare)[1] - cmd = 'cd "%s" && xcopy /e /f /y "%s" "%s\\%s\\"' % ( - _FixPath(base_dir), outer_dir, _FixPath(dst), outer_dir) + fixed_dst = _FixPath(dst) + full_dst = '"%s\\%s\\"' % (fixed_dst, outer_dir) + cmd = 'mkdir %s 2>nul & cd "%s" && xcopy /e /f /y "%s" %s' % ( + full_dst, _FixPath(base_dir), outer_dir, full_dst) copies.append(([src], ['dummy_copies', dst], cmd, - 'Copying %s to %s' % (src, dst))) + 'Copying %s to %s' % (src, fixed_dst))) else: + fix_dst = _FixPath(cpy['destination']) cmd = 'mkdir "%s" 2>nul & set ERRORLEVEL=0 & copy /Y "%s" "%s"' % ( - _FixPath(cpy['destination']), _FixPath(src), _FixPath(dst)) - copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, dst))) + fix_dst, _FixPath(src), _FixPath(dst)) + copies.append(([src], [dst], cmd, 'Copying %s to %s' % (src, fix_dst))) return copies @@ -2718,7 +2731,7 @@ def _GetMSBuildGlobalProperties(spec, version, guid, gyp_file_name): properties[0].append(['WindowsTargetPlatformVersion', str(msvs_windows_sdk_version)]) elif version.compatible_sdks: - raise GypError('%s requires any SDK of %o version, but non were found' % + raise GypError('%s requires any SDK of %s version, but none were found' % (version.description, version.compatible_sdks)) if platform_name == 'ARM': diff --git a/tools/gyp/pylib/gyp/generator/ninja.py b/tools/gyp/pylib/gyp/generator/ninja.py index 163281b3e3f12b..0555a4a90d3885 100644 --- a/tools/gyp/pylib/gyp/generator/ninja.py +++ b/tools/gyp/pylib/gyp/generator/ninja.py @@ -1931,10 +1931,6 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, ld = os.path.join(build_to_root, value) if key == 'LD.host': ld_host = os.path.join(build_to_root, value) - if key == 'LDXX': - ldxx = os.path.join(build_to_root, value) - if key == 'LDXX.host': - ldxx_host = os.path.join(build_to_root, value) if key == 'NM': nm = os.path.join(build_to_root, value) if key == 'NM.host': @@ -2028,7 +2024,6 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, CommandWithWrapper('CXX.host', wrappers, cxx_host)) if flavor == 'win': master_ninja.variable('ld_host', ld_host) - master_ninja.variable('ldxx_host', ldxx_host) else: master_ninja.variable('ld_host', CommandWithWrapper( 'LINK', wrappers, ld_host)) @@ -2153,13 +2148,13 @@ def GenerateOutputForConfig(target_list, target_dicts, data, params, restat=True, command=mtime_preserving_solink_base % {'suffix': '@$link_file_list'}, rspfile='$link_file_list', - rspfile_content='-Wl,--start-group $in $solibs $libs -Wl,--end-group', + rspfile_content='-Wl,--start-group $in -Wl,--end-group $solibs $libs', pool='link_pool') master_ninja.rule( 'link', description='LINK $out', command=('$ld $ldflags -o $out ' - '-Wl,--start-group $in $solibs $libs -Wl,--end-group'), + '-Wl,--start-group $in -Wl,--end-group $solibs $libs'), pool='link_pool') elif flavor == 'win': master_ninja.rule( diff --git a/tools/gyp/pylib/gyp/mac_tool.py b/tools/gyp/pylib/gyp/mac_tool.py index b0363cc3934f12..0ad7e7a1b66b8a 100755 --- a/tools/gyp/pylib/gyp/mac_tool.py +++ b/tools/gyp/pylib/gyp/mac_tool.py @@ -105,17 +105,21 @@ def _CopyXIBFile(self, source, dest): ibtool_section_re = re.compile(r'/\*.*\*/') ibtool_re = re.compile(r'.*note:.*is clipping its content') - ibtoolout = subprocess.Popen(args, stdout=subprocess.PIPE) + try: + stdout = subprocess.check_output(args) + except subprocess.CalledProcessError as e: + print(e.output) + raise current_section_header = None - for line in ibtoolout.stdout: + for line in stdout.splitlines(): if ibtool_section_re.match(line): current_section_header = line elif not ibtool_re.match(line): if current_section_header: - sys.stdout.write(current_section_header) + print(current_section_header) current_section_header = None - sys.stdout.write(line) - return ibtoolout.returncode + print(line) + return 0 def _ConvertToBinary(self, dest): subprocess.check_call([ diff --git a/tools/gyp/pylib/gyp/msvs_emulation.py b/tools/gyp/pylib/gyp/msvs_emulation.py index 14daaec4c7fabc..6d5b5bd234e04b 100644 --- a/tools/gyp/pylib/gyp/msvs_emulation.py +++ b/tools/gyp/pylib/gyp/msvs_emulation.py @@ -30,6 +30,10 @@ def QuoteForRspFile(arg): # works more or less because most programs (including the compiler, etc.) # use that function to handle command line arguments. + # Use a heuristic to try to find args that are paths, and normalize them + if arg.find('/') > 0 or arg.count('/') > 1: + arg = os.path.normpath(arg) + # For a literal quote, CommandLineToArgvW requires 2n+1 backslashes # preceding it, and results in n backslashes + the quote. So we substitute # in 2* what we match, +1 more, plus the quote. @@ -269,8 +273,8 @@ def ConvertVSMacros(self, s, base_to_build=None, config=None): def AdjustLibraries(self, libraries): """Strip -l from library if it's specified with that.""" libs = [lib[2:] if lib.startswith('-l') else lib for lib in libraries] - return [lib + '.lib' if not lib.endswith('.lib') \ - and not lib.endswith('.obj') else lib for lib in libs] + return [lib + '.lib' if not lib.lower().endswith('.lib') else lib + for lib in libs] def _GetAndMunge(self, field, path, default, prefix, append, map): """Retrieve a value from |field| at |path| or return |default|. If @@ -307,7 +311,10 @@ def _TargetConfig(self, config): # There's two levels of architecture/platform specification in VS. The # first level is globally for the configuration (this is what we consider # "the" config at the gyp level, which will be something like 'Debug' or - # 'Release_x64'), and a second target-specific configuration, which is an + # 'Release'), VS2015 and later only use this level + if self.vs_version.short_name >= 2015: + return config + # and a second target-specific configuration, which is an # override for the global one. |config| is remapped here to take into # account the local target-specific overrides to the global configuration. arch = self.GetArch(config) @@ -469,8 +476,10 @@ def GetCflags(self, config): prefix='/arch:') cflags.extend(['/FI' + f for f in self._Setting( ('VCCLCompilerTool', 'ForcedIncludeFiles'), config, default=[])]) - if self.vs_version.short_name in ('2013', '2013e', '2015'): - # New flag required in 2013 to maintain previous PDB behavior. + if self.vs_version.project_version >= 12.0: + # New flag introduced in VS2013 (project version 12.0) Forces writes to + # the program database (PDB) to be serialized through MSPDBSRV.EXE. + # https://msdn.microsoft.com/en-us/library/dn502518.aspx cflags.append('/FS') # ninja handles parallelism by itself, don't have the compiler do it too. cflags = filter(lambda x: not x.startswith('/MP'), cflags) @@ -530,7 +539,8 @@ def GetDefFile(self, gyp_to_build_path): """Returns the .def file from sources, if any. Otherwise returns None.""" spec = self.spec if spec['type'] in ('shared_library', 'loadable_module', 'executable'): - def_files = [s for s in spec.get('sources', []) if s.endswith('.def')] + def_files = [s for s in spec.get('sources', []) + if s.lower().endswith('.def')] if len(def_files) == 1: return gyp_to_build_path(def_files[0]) elif len(def_files) > 1: diff --git a/tools/gyp/pylib/gyp/xcode_emulation.py b/tools/gyp/pylib/gyp/xcode_emulation.py index 667c53695a12d8..dba8e7699e520f 100644 --- a/tools/gyp/pylib/gyp/xcode_emulation.py +++ b/tools/gyp/pylib/gyp/xcode_emulation.py @@ -1662,8 +1662,6 @@ def _GetXcodeEnv(xcode_settings, built_products_dir, srcroot, configuration, sdk_root = xcode_settings._SdkRoot(configuration) if not sdk_root: sdk_root = xcode_settings._XcodeSdkPath('') - if sdk_root is None: - sdk_root = '' env['SDKROOT'] = sdk_root if not additional_settings: