tools: update gyp to v0.5.0

PR-URL: #32698
Reviewed-By: Christian Clauss <cclauss@me.com>
Reviewed-By: Jiawen Geng <technicalcute@gmail.com>
Reviewed-By: Michaël Zasso <targos@protonmail.com>

Author: ryzokuken

tools/gyp/.flake8

@@ -1,4 +1,4 @@
 [flake8]
-max-complexity = 10
+max-complexity = 101
 max-line-length = 88
-extend-ignore = E203,C901,E501
+extend-ignore = E203  # whitespace before ':' to agree with psf/black

tools/gyp/.github/workflows/Python_tests.yml

@@ -12,7 +12,7 @@ jobs:
       max-parallel: 15
       matrix:
         os: [macos-latest, ubuntu-latest] # , windows-latest]
-        python-version: [2.7, 3.6, 3.7, 3.8] # 3.5,
+        python-version: [2.7, 3.6, 3.7, 3.8, 3.9.0-rc.1] # 3.5,
     steps:
       - uses: actions/checkout@v2
       - name: Set up Python ${{ matrix.python-version }}

tools/gyp/CHANGELOG.md

@@ -6,6 +6,18 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 
 ## [Unreleased]
 
+## [0.5.0] - 2020-09-30
+
+### Added
+- Extended compile_commands_json generator to consider more file extensions than
+  just `c` and `cc`. `cpp` and `cxx` are now supported.
+- Source files with duplicate basenames are now supported.
+
+### Removed
+- The `--no-duplicate-basename-check` option was removed.
+- The `msvs_enable_marmasm` configuration option was removed in favor of
+  auto-inclusion of the "marmasm" sections for Windows on ARM.
+
 ## [0.4.0] - 2020-07-14
 
 ### Added
@@ -34,7 +46,8 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/).
 This is the first release of this project, based on https://chromium.googlesource.com/external/gyp
 with changes made over the years in Node.js and node-gyp.
 
-[Unreleased]: https://github.com/nodejs/gyp-next/compare/v0.4.0...HEAD
+[Unreleased]: https://github.com/nodejs/gyp-next/compare/v0.5.0...HEAD
+[0.5.0]: https://github.com/nodejs/gyp-next/compare/v0.4.0...v0.5.0
 [0.4.0]: https://github.com/nodejs/gyp-next/compare/v0.3.0...v0.4.0
 [0.3.0]: https://github.com/nodejs/gyp-next/compare/v0.2.1...v0.3.0
 [0.2.1]: https://github.com/nodejs/gyp-next/compare/v0.2.0...v0.2.1

tools/gyp/README.md

@@ -2,3 +2,6 @@ GYP can Generate Your Projects.
 ===================================
 
 Documents are available at [gyp.gsrc.io](https://gyp.gsrc.io), or you can check out ```md-pages``` branch to read those documents offline.
+
+__gyp-next__ is [released](https://github.com/nodejs/gyp-next/releases) to the [__Python Packaging Index__](https://pypi.org/project/gyp-next) (PyPI) and can be installed with the command:
+* `python3 -m pip install gyp-next`

tools/gyp/pylib/gyp/MSVSSettings_test.py

@@ -678,7 +678,8 @@ def testConvertToMSBuildSettings_warnings(self):
                 "MSBuild, index value (21) not in expected range [0, 3)",
                 "Warning: while converting VCCLCompilerTool/UsePrecompiledHeader to "
                 "MSBuild, index value (13) not in expected range [0, 3)",
-                "Warning: while converting VCCLCompilerTool/GeneratePreprocessedFile to "
+                "Warning: while converting "
+                "VCCLCompilerTool/GeneratePreprocessedFile to "
                 "MSBuild, value must be one of [0, 1, 2]; got 14",
                 "Warning: while converting VCLinkerTool/Driver to "
                 "MSBuild, index value (10) not in expected range [0, 4)",
@@ -1348,7 +1349,8 @@ def testConvertToMSBuildSettings_actual(self):
                 "EmbedManifest": "false",
                 "GenerateCatalogFiles": "true",
                 "InputResourceManifests": "asfsfdafs",
-                "ManifestResourceFile": "$(IntDir)\\$(TargetFileName).embed.manifest.resfdsf",
+                "ManifestResourceFile":
+                    "$(IntDir)\\$(TargetFileName).embed.manifest.resfdsf",
                 "OutputManifestFile": "$(TargetPath).manifestdfs",
                 "RegistrarScriptFile": "sdfsfd",
                 "ReplacementsFile": "sdffsd",
@@ -1532,7 +1534,8 @@ def testConvertToMSBuildSettings_actual(self):
                 "LinkIncremental": "",
             },
             "ManifestResourceCompile": {
-                "ResourceOutputFileName": "$(IntDir)$(TargetFileName).embed.manifest.resfdsf"
+                "ResourceOutputFileName":
+                    "$(IntDir)$(TargetFileName).embed.manifest.resfdsf"
             },
         }
         self.maxDiff = 9999  # on failure display a long diff

tools/gyp/pylib/gyp/__init__.py

@@ -68,7 +68,6 @@ def Load(
     params=None,
     check=False,
     circular_check=True,
-    duplicate_basename_check=True,
 ):
     """
   Loads one or more specified build files.
@@ -156,7 +155,6 @@ def Load(
         generator_input_info,
         check,
         circular_check,
-        duplicate_basename_check,
         params["parallel"],
         params["root_targets"],
     )
@@ -431,20 +429,6 @@ def gyp_main(args):
         regenerate=False,
         help="don't check for circular relationships between files",
     )
-    # --no-duplicate-basename-check disables the check for duplicate basenames
-    # in a static_library/shared_library project. Visual C++ 2008 generator
-    # doesn't support this configuration. Libtool on Mac also generates warnings
-    # when duplicate basenames are passed into Make generator on Mac.
-    # TODO(yukawa): Remove this option when these legacy generators are
-    # deprecated.
-    parser.add_argument(
-        "--no-duplicate-basename-check",
-        dest="duplicate_basename_check",
-        action="store_false",
-        default=True,
-        regenerate=False,
-        help="don't check for duplicate basenames",
-    )
     parser.add_argument(
         "--no-parallel",
         action="store_true",
@@ -651,7 +635,6 @@ def gyp_main(args):
             params,
             options.check,
             options.circular_check,
-            options.duplicate_basename_check,
         )
 
         # TODO(mark): Pass |data| for now because the generator needs a list of

tools/gyp/pylib/gyp/common.py

@@ -352,10 +352,14 @@ class Writer(object):
         """Wrapper around file which only covers the target if it differs."""
 
         def __init__(self):
-            # On Cygwin remove the "dir" argument because `C:` prefixed paths are treated as relative,
-            # consequently ending up with current dir "/cygdrive/c/..." being prefixed to those, which was
-            # obviously a non-existent path, for example: "/cygdrive/c/<some folder>/C:\<my win style abs path>".
-            # See https://docs.python.org/2/library/tempfile.html#tempfile.mkstemp for more details
+            # On Cygwin remove the "dir" argument
+            # `C:` prefixed paths are treated as relative,
+            # consequently ending up with current dir "/cygdrive/c/..."
+            # being prefixed to those, which was
+            # obviously a non-existent path,
+            # for example: "/cygdrive/c/<some folder>/C:\<my win style abs path>".
+            # For more details see:
+            # https://docs.python.org/2/library/tempfile.html#tempfile.mkstemp
             base_temp_dir = "" if IsCygwin() else os.path.dirname(filename)
             # Pick temporary file.
             tmp_fd, self.tmp_path = tempfile.mkstemp(
@@ -391,13 +395,15 @@ def close(self):
                     # one.
                     os.unlink(self.tmp_path)
                 else:
-                    # The new file is different from the old one, or there is no old one.
+                    # The new file is different from the old one,
+                    # or there is no old one.
                     # Rename the new file to the permanent name.
                     #
                     # tempfile.mkstemp uses an overly restrictive mode, resulting in a
                     # file that can only be read by the owner, regardless of the umask.
-                    # There's no reason to not respect the umask here, which means that
-                    # an extra hoop is required to fetch it and reset the new file's mode.
+                    # There's no reason to not respect the umask here,
+                    # which means that an extra hoop is required
+                    # to fetch it and reset the new file's mode.
                     #
                     # No way to get the umask without setting a new one?  Set a safe one
                     # and then set it back to the old value.
@@ -406,8 +412,8 @@ def close(self):
                     os.chmod(self.tmp_path, 0o666 & ~umask)
                     if sys.platform == "win32" and os.path.exists(filename):
                         # NOTE: on windows (but not cygwin) rename will not replace an
-                        # existing file, so it must be preceded with a remove. Sadly there
-                        # is no way to make the switch atomic.
+                        # existing file, so it must be preceded with a remove.
+                        # Sadly there is no way to make the switch atomic.
                         os.remove(filename)
                     os.rename(self.tmp_path, filename)
             except Exception:

tools/gyp/pylib/gyp/generator/android.py

@@ -557,7 +557,7 @@ def WriteSources(self, spec, configs, extra_sources):
     These are source files necessary to build the current target.
     We need to handle shared_intermediate directory source files as
     a special case by copying them to the intermediate directory and
-    treating them as a genereated sources. Otherwise the Android build
+    treating them as a generated sources. Otherwise the Android build
     rules won't pick them up.
 
     Args:

tools/gyp/pylib/gyp/generator/compile_commands_json.py

@@ -61,8 +61,8 @@ def AddCommandsForTarget(cwd, target, params, per_config_commands):
         defines = ["-D" + s for s in defines]
 
         # TODO(bnoordhuis) Handle generated source files.
-        sources = target.get("sources", [])
-        sources = [s for s in sources if s.endswith(".c") or s.endswith(".cc")]
+        extensions = (".c", ".cc", ".cpp", ".cxx")
+        sources = [s for s in target.get("sources", []) if s.endswith(extensions)]
 
         def resolve(filename):
             return os.path.abspath(os.path.join(cwd, filename))

tools/gyp/pylib/gyp/generator/make.py

@@ -30,7 +30,6 @@
 import gyp.common
 import gyp.xcode_emulation
 from gyp.common import GetEnvironFallback
-from gyp.common import GypError
 
 import hashlib
 
@@ -177,7 +176,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
 cmd_solink_module = $(LINK.$(TOOLSET)) -o $@ -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -Wl,-soname=$(@F) -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
-"""
+"""  # noqa: E501
 
 LINK_COMMANDS_MAC = """\
 quiet_cmd_alink = LIBTOOL-STATIC $@
@@ -191,7 +190,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
 cmd_solink_module = $(LINK.$(TOOLSET)) -bundle $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS)
-"""
+"""  # noqa: E501
 
 LINK_COMMANDS_ANDROID = """\
 quiet_cmd_alink = AR($(TOOLSET)) $@
@@ -218,7 +217,7 @@ def CalculateGeneratorInputInfo(params):
 cmd_solink_module = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -Wl,-soname=$(@F) -o $@ -Wl,--start-group $(filter-out FORCE_DO_CMD, $^) -Wl,--end-group $(LIBS)
 quiet_cmd_solink_module_host = SOLINK_MODULE($(TOOLSET)) $@
 cmd_solink_module_host = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -Wl,-soname=$(@F) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS)
-"""
+"""  # noqa: E501
 
 
 LINK_COMMANDS_AIX = """\
@@ -236,7 +235,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
 cmd_solink_module = $(LINK.$(TOOLSET)) -shared $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS)
-"""
+"""  # noqa: E501
 
 
 LINK_COMMANDS_OS390 = """\
@@ -254,8 +253,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_solink_module = SOLINK_MODULE($(TOOLSET)) $@
 cmd_solink_module = $(LINK.$(TOOLSET)) $(GYP_LDFLAGS) $(LDFLAGS.$(TOOLSET)) -o $@ $(filter-out FORCE_DO_CMD, $^) $(LIBS) -Wl,DLL
-
-"""
+"""  # noqa: E501
 
 
 # Header of toplevel Makefile.
@@ -404,7 +402,7 @@ def CalculateGeneratorInputInfo(params):
 cmd_copy = ln -f "$<" "$@" 2>/dev/null || (rm -rf "$@" && cp %(copy_archive_args)s "$<" "$@")
 
 %(link_commands)s
-"""
+"""  # noqa: E501
     r"""
 # Define an escape_quotes function to escape single quotes.
 # This allows us to handle quotes properly as long as we always use
@@ -503,7 +501,7 @@ def CalculateGeneratorInputInfo(params):
 .PHONY: FORCE_DO_CMD
 FORCE_DO_CMD:
 
-"""
+"""  # noqa: E501
 )
 
 SHARED_HEADER_MAC_COMMANDS = """
@@ -534,7 +532,7 @@ def CalculateGeneratorInputInfo(params):
 
 quiet_cmd_infoplist = INFOPLIST $@
 cmd_infoplist = $(CC.$(TOOLSET)) -E -P -Wno-trigraphs -x c $(INFOPLIST_DEFINES) "$<" -o "$@"
-"""
+"""  # noqa: E501
 
 
 def WriteRootHeaderSuffixRules(writer):
@@ -672,43 +670,6 @@ def SourceifyAndQuoteSpaces(path):
     return QuoteSpaces(Sourceify(path))
 
 
-# TODO: Avoid code duplication with _ValidateSourcesForMSVSProject in msvs.py.
-def _ValidateSourcesForOSX(spec, all_sources):
-    """Makes sure if duplicate basenames are not specified in the source list.
-
-  Arguments:
-    spec: The target dictionary containing the properties of the target.
-  """
-    if spec.get("type", None) != "static_library":
-        return
-
-    basenames = {}
-    for source in all_sources:
-        name, ext = os.path.splitext(source)
-        is_compiled_file = ext in [".c", ".cc", ".cpp", ".cxx", ".m", ".mm", ".s", ".S"]
-        if not is_compiled_file:
-            continue
-        basename = os.path.basename(name)  # Don't include extension.
-        basenames.setdefault(basename, []).append(source)
-
-    error = ""
-    for basename, files in basenames.items():
-        if len(files) > 1:
-            error += "  %s: %s\n" % (basename, " ".join(files))
-
-    if error:
-        print(
-            (
-                "static library %s has several files with the same basename:\n"
-                % spec["target_name"]
-            )
-            + error
-            + "libtool on OS X will generate"
-            + " warnings for them."
-        )
-        raise GypError("Duplicate basenames in sources section, see list above")
-
-
 # Map from qualified target to path to output.
 target_outputs = {}
 # Map from qualified target to any linkable output.  A subset
@@ -867,10 +828,6 @@ def Write(
         # Sources.
         all_sources = spec.get("sources", []) + extra_sources
         if all_sources:
-            if self.flavor == "mac":
-                # libtool on OS X generates warnings for duplicate basenames in the same
-                # target.
-                _ValidateSourcesForOSX(spec, all_sources)
             self.WriteSources(
                 configs,
                 deps,
@@ -1342,7 +1299,10 @@ def WriteSources(
             )
 
             if self.flavor == "mac":
-                cflags = self.xcode_settings.GetCflags(configname, arch=config.get('xcode_configuration_platform'))
+                cflags = self.xcode_settings.GetCflags(
+                    configname,
+                    arch=config.get('xcode_configuration_platform')
+                )
                 cflags_c = self.xcode_settings.GetCflagsC(configname)
                 cflags_cc = self.xcode_settings.GetCflagsCC(configname)
                 cflags_objc = self.xcode_settings.GetCflagsObjC(configname)
@@ -1659,8 +1619,8 @@ def WriteTarget(
                     ldflags = config.get("ldflags", [])
                     # Compute an rpath for this output if needed.
                     if any(dep.endswith(".so") or ".so." in dep for dep in deps):
-                        # We want to get the literal string "$ORIGIN" into the link command,
-                        # so we need lots of escaping.
+                        # We want to get the literal string "$ORIGIN"
+                        # into the link command, so we need lots of escaping.
                         ldflags.append(r"-Wl,-rpath=\$$ORIGIN/lib.%s/" % self.toolset)
                         ldflags.append(
                             r"-Wl,-rpath-link=\$(builddir)/lib.%s/" % self.toolset