From fc96b8518b222638781806041ab5e0814fe882f8 Mon Sep 17 00:00:00 2001 From: Ian Anderson Date: Fri, 18 Jul 2025 10:45:35 -0700 Subject: [PATCH] [utils] expand @-filelist files with .resp filename extensions Xcode puts clang command line arguments in a single line in .resp files. Split the argument printing routine into its own function print_arg. When a clang response file is encountered, split each line into arguments and loop them through print_arg. -c has an argument for clang --- utils/dev-scripts/split-cmdline | 624 ++++++++++++++++---------------- 1 file changed, 321 insertions(+), 303 deletions(-) diff --git a/utils/dev-scripts/split-cmdline b/utils/dev-scripts/split-cmdline index 70e81124bd95a..dcb54a1a8f19f 100755 --- a/utils/dev-scripts/split-cmdline +++ b/utils/dev-scripts/split-cmdline @@ -48,314 +48,332 @@ def main(): is_arg_param = False # Handle escaped spaces args = shlex.split(line) + cmd_name = args[0] for arg in args: - if arg == "": - continue - if not first: - # Print option arguments in the same line - print(" " if is_arg_param else " \\\n ", end="") - first = False + first, is_arg_param = print_arg(arg, first, is_arg_param, cmd_name) - # Expand @ option files - m = re.match(r"^@(\S+(\.txt|FileList))$", arg) - if m: - cmd_file = m.group(1) - if os.path.isfile(cmd_file): - with open(cmd_file) as f: - for ln in f.readlines(): - for name in ln.rstrip().split(";"): - if name != "": - print(name + " \\") - first = True - continue + # Print 2 new lines after each command line + print("\n") - if " " in arg or ";" in arg: - print('"' + arg + '"', end="") - else: - print(arg, end="") - # A hard-coded list of options which expect a parameter - is_arg_param = arg in [ - # This list is generated by scraping --help menus from running: - # - # $ ( \ - # swift --help; \ - # swift build --help; \ - # swift run --help; \ - # swift repl --help; \ - # swift test --help; \ - # swift package --help; \ - # swiftc --help; \ - # xcrun swift-frontend --help; \ - # clang --help; \ - # ) | rg -o -r '$3 $2' -- ' ((-[\w-]+),)? (-[\w-]+) <' \ - # | sed 's/ *$//' | tr ' ' '\n' \ - # | sort | uniq \ - # | grep -v -e '-c$' \ - # | sed -E 's/(.*)/"\1",/' - "--analyzer-output", - "--build-system", - "--cache-path", - "--config-path", - "--configuration", - "--default-registry-url", - "--experimental-traits", - "--explicit-target-dependency-import-check", - "--filter", - "--jobs", - "--manifest-cache", - "--netrc-file", - "--num-workers", - "--package-path", - "--pkg-config-path", - "--product", - "--resolver-fingerprint-checking", - "--resolver-signing-entity-checking", - "--sanitize", - "--scratch-path", - "--sdk", - "--security-path", - "--skip", - "--specifier", - "--swift-sdk", - "--swift-sdks-path", - "--target", - "--toolchain", - "--triple", - "--xunit-output", - "-B", - "-D", - "-F", - "-Fsystem", - "-G", - "-I", - "-Isystem", - "-L", - "-MF", - "-MJ", - "-MQ", - "-MT", - "-T", - "-U", - "-Xanalyzer", - "-Xarch_device", - "-Xarch_host", - "-Xassembler", - "-Xcc", - "-Xclang", - "-Xcuda-fatbinary", - "-Xcuda-ptxas", - "-Xcxx", - "-Xlinker", - "-Xopenmp-target", - "-Xpreprocessor", - "-Xswiftc", - "-access-notes-path", - "-allowable-client", - "-api-diff-data-dir", - "-api-diff-data-file", - "-arcmt-migrate-report-output", - "-assert-config", - "-autolink-library", - "-b", - "-backup-module-interface-path", - "-blocklist-file", - "-build-id", - "-cache-replay-prefix-map", - "-cas-path", - "-cas-plugin-option", - "-cas-plugin-path", - "-cc1-args", - "-clang-build-session-file", - "-clang-scanner-module-cache-path", - "-clang-target", - "-compare-to-baseline-path", - "-const-gather-protocols-file", - "-coverage-prefix-map", - "-cxx-isystem", - "-darwin-target-variant", - "-darwin-target-variant-triple", - "-debug-info-format", - "-debug-prefix-map", - "-define-availability", - "-dependency-dot", - "-dependency-file", - "-dependency-scan-cache-path", - "-diagnostic-documentation-path", - "-diagnostic-style", - "-digester-breakage-allowlist-path", - "-digester-mode", - "-dsym-dir", - "-dump-api-path", - "-dump-migration-states-dir", - "-dump-scope-maps", - "-dumpdir", - "-e", - "-embed-tbd-for-module", - "-emit-abi-descriptor-path", - "-emit-api-descriptor-path", - "-emit-clang-header-path", - "-emit-const-values-path", - "-emit-dependencies-path", - "-emit-digester-baseline-path", - "-emit-fixits-path", - "-emit-loaded-module-trace-path", - "-emit-migrated-file-path", - "-emit-module-dependencies-path", - "-emit-module-doc-path", - "-emit-module-interface-path", - "-emit-module-path", - "-emit-module-semantic-info-path", - "-emit-module-serialize-diagnostics-path", - "-emit-module-source-info-path", - "-emit-module-summary-path", - "-emit-objc-header-path", - "-emit-reference-dependencies-path", - "-emit-remap-file-path", - "-emit-tbd-path", - "-enable-experimental-feature", - "-enable-upcoming-feature", - "-explain-module-dependency", - "-explain-module-dependency-detailed", - "-explicit-swift-module-map-file", - "-export-as", - "-external-plugin-path", - "-fdepscan-share-identifier", - "-file-compilation-dir", - "-file-prefix-map", - "-filelist", - "-fmodules-user-build-path", - "-framework", - "-group-info-path", - "-iapinotes-modules", - "-iapinotes-path", - "-idirafter", - "-iframework", - "-iframeworkwithsysroot", - "-imacros", - "-in-process-plugin-server-path", - "-include", - "-include-pch", - "-index-file-path", - "-index-store-path", - "-index-unit-output-path", - "-index-unit-output-path-filelist", - "-iprefix", - "-iquote", - "-isysroot", - "-isystem", - "-isystem-after", - "-ivfsoverlay", - "-ivfsstatcache", - "-iwithprefix", - "-iwithprefixbefore", - "-iwithsysroot", - "-j", - "-l", - "-libc", - "-load-plugin-executable", - "-load-plugin-library", - "-load-resolved-plugin", - "-locale", - "-localization-path", - "-lto-library", - "-meabi", - "-mllvm", - "-mmlir", - "-module-abi-name", - "-module-alias", - "-module-cache-path", - "-module-can-import", - "-module-can-import-version", - "-module-dependency-dir", - "-module-link-name", - "-module-name", - "-mthread-model", - "-num-threads", - "-o", - "-output-file-map", - "-output-filelist", - "-package-name", - "-placeholder-dependency-module-map-file", - "-plugin-path", - "-prebuilt-module-cache-path", - "-primary-file", - "-primary-filelist", - "-project-name", - "-require-explicit-availability-target", - "-runtime-compatibility-version", - "-s", - "-save-optimization-record-passes", - "-save-optimization-record-path", - "-scanner-prefix-map", - "-scanner-prefix-map-sdk", - "-scanner-prefix-map-toolchain", - "-sdk", - "-serialize-breaking-changes-path", - "-serialize-diagnostics", - "-serialize-diagnostics-path", - "-serialized-path-obfuscate", - "-supplementary-output-file-map", - "-swift-isa-ptrauth-mode", - "-swift-module-cross-import", - "-swift-ptrauth-mode", - "-swift-version", - "-sysroot", - "-target", - "-target-cpu", - "-target-min-inlining-version", - "-target-variant", - "-tbd-compatibility-version", - "-tbd-current-version", - "-tbd-install_name", - "-tools-directory", - "-user-module-version", - "-verify-additional-file", - "-verify-additional-prefix", - "-verify-generic-signatures", - "-vfsoverlay", - "-visualc-tools-root", - "-visualc-tools-version", - "-windows-sdk-root", - "-windows-sdk-version", - "-working-directory", - "-x", - "-z", - ] + [ - # This list contains options that do not appear in --help menus: - "-Xfrontend", - "-Xllvm", - "-add_ast_path", - "-arch", - "-emit-ir", - "-emit-sil", - "-ferror-limit", - "-fileno", - "-fmodule-feature", - "-import-objc-header", - "-internal-externc-isystem", - "-macosx_version_min", - "-pic-level", - "-resource-dir", - "-rpath", - "-stack-protector", - "-stack-protector-buffer-size", - "-syslibroot", - "-target-abi", - "-target-feature", - "-target-linker-version", - "-target-sdk-version", - "-target-sdk-name", - "-triple", - ] - # Heuristic: options ending in -path expect an argument - if arg != "-finclude-tree-preserve-pch-path" and arg.startswith("-") and arg.endswith("-path"): - is_arg_param = True - if arg == "-c" and args[0] == "swift": - is_arg_param = True +def print_arg(arg, first, is_arg_param, cmd_name): + if arg == "": + return first, is_arg_param + if not first: + # Print option arguments in the same line + print(" " if is_arg_param else " \\\n ", end="") - # Print 2 new lines after each command line - print("\n") + # Expand @ option files + m = re.match(r"^@(\S+(\.resp))$", arg) + if m: + cmd_file = m.group(1) + cmd_first = False + if os.path.isfile(cmd_file): + cmd_first = True + with open(cmd_file) as f: + for ln in f.readlines(): + cmd_args = shlex.split(ln) + for cmd_arg in cmd_args: + cmd_first, is_arg_param = print_arg(cmd_arg, cmd_first, is_arg_param, cmd_name) + else: + print(arg, end="") + return cmd_first, is_arg_param + + m = re.match(r"^@(\S+(\.txt|FileList))$", arg) + if m: + cmd_file = m.group(1) + if os.path.isfile(cmd_file): + with open(cmd_file) as f: + for ln in f.readlines(): + for name in ln.rstrip().split(";"): + print(name + " \\") + return True, is_arg_param + + if " " in arg or ";" in arg: + print('"' + arg + '"', end="") + else: + print(arg, end="") + + # A hard-coded list of options which expect a parameter + is_next_arg_param = arg in [ + # This list is generated by scraping --help menus from running: + # + # $ ( \ + # swift --help; \ + # swift build --help; \ + # swift run --help; \ + # swift repl --help; \ + # swift test --help; \ + # swift package --help; \ + # swiftc --help; \ + # xcrun swift-frontend --help; \ + # clang --help; \ + # ) | rg -o -r '$3 $2' -- ' ((-[\w-]+),)? (-[\w-]+) <' \ + # | sed 's/ *$//' | tr ' ' '\n' \ + # | sort | uniq \ + # | grep -v -e '-c$' \ + # | sed -E 's/(.*)/"\1",/' + "--analyzer-output", + "--build-system", + "--cache-path", + "--config-path", + "--configuration", + "--default-registry-url", + "--experimental-traits", + "--explicit-target-dependency-import-check", + "--filter", + "--jobs", + "--manifest-cache", + "--netrc-file", + "--num-workers", + "--package-path", + "--pkg-config-path", + "--product", + "--resolver-fingerprint-checking", + "--resolver-signing-entity-checking", + "--sanitize", + "--scratch-path", + "--sdk", + "--security-path", + "--skip", + "--specifier", + "--swift-sdk", + "--swift-sdks-path", + "--target", + "--toolchain", + "--triple", + "--xunit-output", + "-B", + "-D", + "-F", + "-Fsystem", + "-G", + "-I", + "-Isystem", + "-L", + "-MF", + "-MJ", + "-MQ", + "-MT", + "-T", + "-U", + "-Xanalyzer", + "-Xarch_device", + "-Xarch_host", + "-Xassembler", + "-Xcc", + "-Xclang", + "-Xcuda-fatbinary", + "-Xcuda-ptxas", + "-Xcxx", + "-Xlinker", + "-Xopenmp-target", + "-Xpreprocessor", + "-Xswiftc", + "-access-notes-path", + "-allowable-client", + "-api-diff-data-dir", + "-api-diff-data-file", + "-arcmt-migrate-report-output", + "-assert-config", + "-autolink-library", + "-b", + "-backup-module-interface-path", + "-blocklist-file", + "-build-id", + "-cache-replay-prefix-map", + "-cas-path", + "-cas-plugin-option", + "-cas-plugin-path", + "-cc1-args", + "-clang-build-session-file", + "-clang-scanner-module-cache-path", + "-clang-target", + "-compare-to-baseline-path", + "-const-gather-protocols-file", + "-coverage-prefix-map", + "-cxx-isystem", + "-darwin-target-variant", + "-darwin-target-variant-triple", + "-debug-info-format", + "-debug-prefix-map", + "-define-availability", + "-dependency-dot", + "-dependency-file", + "-dependency-scan-cache-path", + "-diagnostic-documentation-path", + "-diagnostic-style", + "-digester-breakage-allowlist-path", + "-digester-mode", + "-dsym-dir", + "-dump-api-path", + "-dump-migration-states-dir", + "-dump-scope-maps", + "-dumpdir", + "-e", + "-embed-tbd-for-module", + "-emit-abi-descriptor-path", + "-emit-api-descriptor-path", + "-emit-clang-header-path", + "-emit-const-values-path", + "-emit-dependencies-path", + "-emit-digester-baseline-path", + "-emit-fixits-path", + "-emit-loaded-module-trace-path", + "-emit-migrated-file-path", + "-emit-module-dependencies-path", + "-emit-module-doc-path", + "-emit-module-interface-path", + "-emit-module-path", + "-emit-module-semantic-info-path", + "-emit-module-serialize-diagnostics-path", + "-emit-module-source-info-path", + "-emit-module-summary-path", + "-emit-objc-header-path", + "-emit-reference-dependencies-path", + "-emit-remap-file-path", + "-emit-tbd-path", + "-enable-experimental-feature", + "-enable-upcoming-feature", + "-explain-module-dependency", + "-explain-module-dependency-detailed", + "-explicit-swift-module-map-file", + "-export-as", + "-external-plugin-path", + "-fdepscan-share-identifier", + "-file-compilation-dir", + "-file-prefix-map", + "-filelist", + "-fmodules-user-build-path", + "-framework", + "-group-info-path", + "-iapinotes-modules", + "-iapinotes-path", + "-idirafter", + "-iframework", + "-iframeworkwithsysroot", + "-imacros", + "-in-process-plugin-server-path", + "-include", + "-include-pch", + "-index-file-path", + "-index-store-path", + "-index-unit-output-path", + "-index-unit-output-path-filelist", + "-iprefix", + "-iquote", + "-isysroot", + "-isystem", + "-isystem-after", + "-ivfsoverlay", + "-ivfsstatcache", + "-iwithprefix", + "-iwithprefixbefore", + "-iwithsysroot", + "-j", + "-l", + "-libc", + "-load-plugin-executable", + "-load-plugin-library", + "-load-resolved-plugin", + "-locale", + "-localization-path", + "-lto-library", + "-meabi", + "-mllvm", + "-mmlir", + "-module-abi-name", + "-module-alias", + "-module-cache-path", + "-module-can-import", + "-module-can-import-version", + "-module-dependency-dir", + "-module-link-name", + "-module-name", + "-mthread-model", + "-num-threads", + "-o", + "-output-file-map", + "-output-filelist", + "-package-name", + "-placeholder-dependency-module-map-file", + "-plugin-path", + "-prebuilt-module-cache-path", + "-primary-file", + "-primary-filelist", + "-project-name", + "-require-explicit-availability-target", + "-runtime-compatibility-version", + "-s", + "-save-optimization-record-passes", + "-save-optimization-record-path", + "-scanner-prefix-map", + "-scanner-prefix-map-sdk", + "-scanner-prefix-map-toolchain", + "-sdk", + "-serialize-breaking-changes-path", + "-serialize-diagnostics", + "-serialize-diagnostics-path", + "-serialized-path-obfuscate", + "-supplementary-output-file-map", + "-swift-isa-ptrauth-mode", + "-swift-module-cross-import", + "-swift-ptrauth-mode", + "-swift-version", + "-sysroot", + "-target", + "-target-cpu", + "-target-min-inlining-version", + "-target-variant", + "-tbd-compatibility-version", + "-tbd-current-version", + "-tbd-install_name", + "-tools-directory", + "-user-module-version", + "-verify-additional-file", + "-verify-additional-prefix", + "-verify-generic-signatures", + "-vfsoverlay", + "-visualc-tools-root", + "-visualc-tools-version", + "-windows-sdk-root", + "-windows-sdk-version", + "-working-directory", + "-x", + "-z", + ] + [ + # This list contains options that do not appear in --help menus: + "-Xfrontend", + "-Xllvm", + "-add_ast_path", + "-arch", + "-emit-ir", + "-emit-sil", + "-ferror-limit", + "-fileno", + "-fmodule-feature", + "-import-objc-header", + "-internal-externc-isystem", + "-macosx_version_min", + "-pic-level", + "-resource-dir", + "-rpath", + "-stack-protector", + "-stack-protector-buffer-size", + "-syslibroot", + "-target-abi", + "-target-feature", + "-target-linker-version", + "-target-sdk-version", + "-target-sdk-name", + "-triple", + ] + # Heuristic: options ending in -path expect an argument + if arg != "-finclude-tree-preserve-pch-path" and arg.startswith("-") and arg.endswith("-path"): + is_next_arg_param = True + if arg == "-c" and (cmd_name == "swift" or os.path.basename(cmd_name) == "clang"): + is_next_arg_param = True + return False, is_next_arg_param if __name__ == "__main__":