[5.3] Support cross compile Xcode toolchain for Apple Silicon

utils/build-presets.ini

@@ -1214,13 +1214,21 @@ compiler-vendor=apple
 
 dash-dash
 
+# Cross compile for Apple Silicon
+cross-compile-hosts=macosx-arm64
+
 lldb-no-debugserver
 lldb-use-system-debugserver
 lldb-build-type=Release
 verbose-build
 build-ninja
 build-swift-stdlib-unittest-extra
 
+# When producing a package, don't copy the Swift Resource/ directory.
+# This is mainly a space optimization.
+extra-cmake-options=
+   -DLLDB_FRAMEWORK_COPY_SWIFT_RESOURCES=0
+
 install-swift
 install-lldb
 install-llbuild

utils/build-script

@@ -919,8 +919,8 @@ class BuildScriptInvocation(object):
             for product_class in impl_product_classes:
                 self._execute_install_action(host_target, product_class)
 
-        # Lipo...
-        self._execute_merged_host_lipo_action()
+        # Core Lipo...
+        self._execute_merged_host_lipo_core_action()
 
         # Non-build-script-impl products...
         # Note: currently only supports building for the host.
@@ -960,6 +960,9 @@ class BuildScriptInvocation(object):
         for host_target in all_hosts:
             self._execute_package_action(host_target)
 
+        # Lipo...
+        self._execute_merged_host_lipo_action()
+
     def _execute_build_action(self, host_target, product_class):
         action_name = "{}-{}-build".format(host_target.name,
                                            product_class.product_name())
@@ -986,6 +989,9 @@ class BuildScriptInvocation(object):
     def _execute_merged_host_lipo_action(self):
         self._execute_action("merged-hosts-lipo")
 
+    def _execute_merged_host_lipo_core_action(self):
+        self._execute_action("merged-hosts-lipo-core")
+
     def _execute_action(self, action_name):
         shell.call_without_sleeping(
             [BUILD_SCRIPT_IMPL_PATH] + self.impl_args +

utils/build-script-impl

@@ -446,6 +446,7 @@ function set_build_options_for_host() {
     swift_cmake_options=()
     cmark_cmake_options=()
     lldb_cmake_options=()
+    llbuild_cmake_options=()
     SWIFT_HOST_VARIANT=
     SWIFT_HOST_VARIANT_SDK=
     SWIFT_HOST_VARIANT_ARCH=
@@ -656,6 +657,7 @@ function set_build_options_for_host() {
                 -DCMAKE_CXX_FLAGS="$(cmark_c_flags ${host})"
                 -DCMAKE_OSX_SYSROOT:PATH="${cmake_os_sysroot}"
                 -DCMAKE_OSX_DEPLOYMENT_TARGET="${cmake_osx_deployment_target}"
+                -DCMAKE_OSX_ARCHITECTURES="${architecture}"
             )
             llvm_cmake_options=(
                 -DCMAKE_OSX_DEPLOYMENT_TARGET:STRING="${cmake_osx_deployment_target}"
@@ -665,6 +667,7 @@ function set_build_options_for_host() {
                 -DCOMPILER_RT_ENABLE_TVOS:BOOL=FALSE
                 -DSANITIZER_MIN_OSX_VERSION="${cmake_osx_deployment_target}"
                 -DLLVM_ENABLE_MODULES:BOOL="$(true_false ${LLVM_ENABLE_MODULES})"
+                -DCMAKE_OSX_ARCHITECTURES="${architecture}"
             )
             if [[ $(is_llvm_lto_enabled) == "TRUE" ]]; then
                 llvm_cmake_options+=(
@@ -693,6 +696,13 @@ function set_build_options_for_host() {
                 # in the compiler checks CMake performs
                 -DCMAKE_OSX_ARCHITECTURES="${architecture}"
             )
+
+            lldb_cmake_options+=(
+                -DCMAKE_OSX_ARCHITECTURES="${architecture}"
+            )
+            llbuild_cmake_options+=(
+                -DCMAKE_OSX_ARCHITECTURES="${architecture}"
+            )
             ;;
     esac
 
@@ -1992,6 +2002,7 @@ for host in "${ALL_HOSTS[@]}"; do
             llbuild)
                 cmake_options=(
                     "${cmake_options[@]}"
+                    "${llbuild_cmake_options[@]}"
 
                     -DCMAKE_BUILD_TYPE:STRING="${LLBUILD_BUILD_TYPE}"
                     -DCMAKE_INSTALL_PREFIX:PATH="$(get_host_install_prefix ${host})"
@@ -2433,13 +2444,15 @@ for host in "${ALL_HOSTS[@]}"; do
                 executable_target=
                 results_targets=
                 if ! [[ "${SKIP_TEST_SWIFT}" ]]; then
-                    executable_target=SwiftUnitTests
-                    results_targets=("${SWIFT_TEST_TARGETS[@]}")
-                    if [[ "${STRESS_TEST_SOURCEKIT}" ]]; then
-                        results_targets=(
-                            "${results_targets[@]}"
-                            stress-SourceKit
-                        )
+                    if ! [[ $(is_cross_tools_host ${host}) ]] ; then
+                       executable_target=SwiftUnitTests
+                       results_targets=("${SWIFT_TEST_TARGETS[@]}")
+                       if [[ "${STRESS_TEST_SOURCEKIT}" ]]; then
+                          results_targets=(
+                               "${results_targets[@]}"
+                               stress-SourceKit
+                          )
+                       fi
                     fi
                 fi
                 if ! [[ "${SKIP_TEST_BENCHMARKS}" ]]; then
@@ -2456,6 +2469,10 @@ for host in "${ALL_HOSTS[@]}"; do
                 if [[ "${SKIP_TEST_LLDB}" ]]; then
                     continue
                 fi
+                if [[ $(is_cross_tools_host ${host}) ]]; then
+                    echo "--- Can't execute tests for ${host}, skipping... ---"
+                    continue
+                fi
                 llvm_build_dir=$(build_directory ${host} llvm)
                 lldb_build_dir=$(build_directory ${host} lldb)
                 results_dir="${lldb_build_dir}/test-results"
@@ -2758,8 +2775,12 @@ for host in "${ALL_HOSTS[@]}"; do
                 fi
                 if [[ "${LLVM_INSTALL_COMPONENTS}" == "all" ]]; then
                     INSTALL_TARGETS=install
-                else
-                    INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/;/ install-/g')
+                elif [[ -n "${LLVM_INSTALL_COMPONENTS}" ]] ; then
+                    if [[ $(is_cross_tools_host ${host}) && "${LLVM_INSTALL_COMPONENTS}" == *"compiler-rt"* ]]; then
+                        INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/compiler-rt;//g' |sed -E 's/;/ install-/g')
+                    else
+                        INSTALL_TARGETS=install-$(echo ${LLVM_INSTALL_COMPONENTS} | sed -E 's/;/ install-/g')
+                    fi
                 fi
                 ;;
             libcxx)
@@ -2986,11 +3007,7 @@ function build_and_test_installable_package() {
         local host_install_destdir="$(get_host_install_destdir ${host})"
         local host_install_prefix="$(get_host_install_prefix ${host})"
 
-        if [[ $(has_cross_compile_hosts) ]]; then
-            package_for_host="${INSTALLABLE_PACKAGE}-${host}"
-        else
-            package_for_host="${INSTALLABLE_PACKAGE}"
-        fi
+        package_for_host="${INSTALLABLE_PACKAGE}"
 
         echo "--- Creating installable package ---"
         echo "-- Package file: ${package_for_host} --"
@@ -3060,7 +3077,7 @@ function build_and_test_installable_package() {
             PKG_TESTS_SANDBOX_PARENT="$(build_directory swift_package_sandbox_${host} none)"
             PKG_TESTS_TEMPS="${PKG_TESTS_SANDBOX_PARENT}"/"tests"
 
-            if [[ "${host}" == "macosx-"* ]] ; then
+            if [[ "${host}" == "macosx-"* ]] || [[ "${host}" == "merged-hosts" ]]; then
                 PKG_TESTS_SANDBOX="${PKG_TESTS_SANDBOX_PARENT}"/"${TOOLCHAIN_PREFIX}"
             else # Linux
                 PKG_TESTS_SANDBOX="${PKG_TESTS_SANDBOX_PARENT}"
@@ -3101,7 +3118,7 @@ if [[ ${#LIPO_SRC_DIRS[@]} -gt 0 ]]; then
     # This is from multiple hosts; Which host should we say it is?
     # Let's call it 'merged-hosts' so that we can identify it.
 
-    if [[ $(should_execute_action "${mergedHost}-lipo") ]]; then
+    if [[ $(should_execute_action "${mergedHost}-lipo") || $(should_execute_action "${mergedHost}-lipo-core") ]]; then
         # Allow passing lipo with --host-lipo
         if [[ -z "${HOST_LIPO}" ]] ; then
             LIPO_PATH=$(xcrun_find_tool lipo)
@@ -3110,8 +3127,10 @@ if [[ ${#LIPO_SRC_DIRS[@]} -gt 0 ]]; then
         fi
         call "${SWIFT_SOURCE_DIR}"/utils/recursive-lipo --lipo=${LIPO_PATH} --copy-subdirs="$(get_host_install_prefix ${host})lib/swift $(get_host_install_prefix ${host})lib/swift_static" --destination="$(get_host_install_destdir ${mergedHost})" ${LIPO_SRC_DIRS[@]}
 
-        # Build and test the lipo-ed package.
-        build_and_test_installable_package ${mergedHost}
+        if [[ $(should_execute_action "${mergedHost}-lipo") ]]; then
+            # Build and test the lipo-ed package.
+            build_and_test_installable_package ${mergedHost}
+        fi
     fi
 fi
 # END

utils/swift_build_support/swift_build_support/products/benchmarks.py

@@ -14,6 +14,7 @@
 import platform
 
 from . import product
+from . import swiftpm
 from .. import shell
 from .. import targets
 
@@ -59,7 +60,9 @@ def install(self, host_target):
 
 
 def run_build_script_helper(host_target, product, args):
-    toolchain_path = args.install_destdir
+    toolchain_path = swiftpm.SwiftPM.get_install_destdir(args,
+                                                         host_target,
+                                                         product.build_dir)
     if platform.system() == 'Darwin':
         # The prefix is an absolute path, so concatenate without os.path.
         toolchain_path += \

utils/swift_build_support/swift_build_support/products/indexstoredb.py

@@ -13,6 +13,7 @@
 import os
 
 from . import product
+from . import swiftpm
 from .. import shell
 from .. import targets
 
@@ -51,7 +52,12 @@ def run_build_script_helper(action, host_target, product, args,
     script_path = os.path.join(
         product.source_dir, 'Utilities', 'build-script-helper.py')
 
-    toolchain_path = targets.toolchain_path(args.install_destdir,
+    install_destdir = args.install_destdir
+    if swiftpm.SwiftPM.has_cross_compile_hosts(args):
+        install_destdir = swiftpm.SwiftPM.get_install_destdir(args,
+                                                              host_target,
+                                                              product.build_dir)
+    toolchain_path = targets.toolchain_path(install_destdir,
                                             args.install_prefix)
     is_release = product.is_release()
     configuration = 'release' if is_release else 'debug'

utils/swift_build_support/swift_build_support/products/product.py

@@ -11,6 +11,7 @@
 # ----------------------------------------------------------------------------
 
 import abc
+import os
 
 from .. import cmake
 from .. import targets
@@ -136,13 +137,17 @@ def is_release(self):
         """
         return is_release_variant(self.args.build_variant)
 
-    def install_toolchain_path(self):
+    def install_toolchain_path(self, host_target):
         """toolchain_path() -> string
 
         Returns the path to the toolchain that is being created as part of this
         build.
         """
-        return targets.toolchain_path(self.args.install_destdir,
+        install_destdir = self.args.install_destdir
+        if self.args.cross_compile_hosts:
+            build_root = os.path.dirname(self.build_dir)
+            install_destdir = '%s/intermediate-install/%s' % (build_root, host_target)
+        return targets.toolchain_path(install_destdir,
                                       self.args.install_prefix)
 
 

utils/swift_build_support/swift_build_support/products/skstresstester.py

@@ -17,6 +17,7 @@
 from build_swift.build_swift.constants import MULTIROOT_DATA_FILE_PATH
 
 from . import product
+from . import swiftpm
 from .. import shell
 
 
@@ -40,7 +41,7 @@ def is_swiftpm_unified_build_product(cls):
     def package_name(self):
         return 'SourceKitStressTester'
 
-    def run_build_script_helper(self, action, additional_params=[]):
+    def run_build_script_helper(self, action, host_target, additional_params=[]):
         script_path = os.path.join(
             self.source_dir, 'build-script-helper.py')
 
@@ -50,7 +51,7 @@ def run_build_script_helper(self, action, additional_params=[]):
             script_path,
             action,
             '--package-dir', self.package_name(),
-            '--toolchain', self.install_toolchain_path(),
+            '--toolchain', self.install_toolchain_path(host_target),
             '--config', configuration,
             '--build-dir', self.build_dir,
             '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
@@ -74,19 +75,22 @@ def build(self, host_target):
                                "than Darwin".format(
                                    product=self.package_name()))
 
-        self.run_build_script_helper('build')
+        self.run_build_script_helper('build', host_target)
 
     def should_test(self, host_target):
         return self.args.test_skstresstester
 
     def test(self, host_target):
-        self.run_build_script_helper('test')
+        self.run_build_script_helper('test', host_target)
 
     def should_install(self, host_target):
         return self.args.install_skstresstester
 
     def install(self, host_target):
-        install_prefix = self.args.install_destdir + self.args.install_prefix
-        self.run_build_script_helper('install', [
+        install_destdir = swiftpm.SwiftPM.get_install_destdir(self.args,
+                                                              host_target,
+                                                              self.build_dir)
+        install_prefix = install_destdir + self.args.install_prefix
+        self.run_build_script_helper('install', host_target, [
             '--prefix', install_prefix
         ])

utils/swift_build_support/swift_build_support/products/swiftpm.py

@@ -31,7 +31,7 @@ def should_build(self, host_target):
     def run_bootstrap_script(self, action, host_target, additional_params=[]):
         script_path = os.path.join(
             self.source_dir, 'Utilities', 'bootstrap')
-        toolchain_path = self.install_toolchain_path()
+        toolchain_path = self.install_toolchain_path(host_target)
         swiftc = os.path.join(toolchain_path, "bin", "swiftc")
 
         # FIXME: We require llbuild build directory in order to build. Is
@@ -53,6 +53,13 @@ def run_bootstrap_script(self, action, host_target, additional_params=[]):
             "--build-dir", self.build_dir,
             "--llbuild-build-dir", llbuild_build_dir
         ]
+
+        # Pass Cross compile host info
+        if self.has_cross_compile_hosts(self.args):
+            helper_cmd += ['--cross-compile-hosts']
+            for cross_compile_host in self.args.cross_compile_hosts:
+                helper_cmd += [cross_compile_host]
+
         helper_cmd.extend(additional_params)
 
         shell.call(helper_cmd)
@@ -69,8 +76,24 @@ def test(self, host_target):
     def should_install(self, host_target):
         return self.args.install_swiftpm
 
+    @classmethod
+    def has_cross_compile_hosts(self, args):
+        return args.cross_compile_hosts
+
+    @classmethod
+    def get_install_destdir(self, args, host_target, build_dir):
+        install_destdir = args.install_destdir
+        if self.has_cross_compile_hosts(args):
+            build_root = os.path.dirname(build_dir)
+            install_destdir = '%s/intermediate-install/%s' % (build_root, host_target)
+        return install_destdir
+
     def install(self, host_target):
-        install_prefix = self.args.install_destdir + self.args.install_prefix
+        install_destdir = self.get_install_destdir(self.args,
+                                                   host_target,
+                                                   self.build_dir)
+        install_prefix = install_destdir + self.args.install_prefix
+
         self.run_bootstrap_script('install', host_target, [
             '--prefix', install_prefix
         ])

utils/swift_build_support/swift_build_support/products/swiftsyntax.py

@@ -45,7 +45,7 @@ def run_swiftsyntax_build_script(self, target, additional_params=[]):
             script_path,
             '--build-dir', self.build_dir,
             '--multiroot-data-file', MULTIROOT_DATA_FILE_PATH,
-            '--toolchain', self.install_toolchain_path(),
+            '--toolchain', self.install_toolchain_path(target),
             '--filecheck-exec', os.path.join(llvm_build_dir, 'bin',
                                              'FileCheck'),
         ]