[CMake] Support Macros in Linux

For compiling codes required for macro support, we now need swiftc
compiler in the build machine.

Unlike Darwin OSes, where swiftCore runtime is guaranteed to be present
in /usr/lib, Linux doesn't have ABI stability and the stdlib of the
build machine is not at the specific location. So the built compiler
cannot relies on the shared object in the toolchain.

Author: rintaro

CMakeLists.txt

@@ -794,6 +794,7 @@ include(SwiftConfigureSDK)
 include(SwiftComponents)
 include(SwiftList)
 include(AddPureSwift)
+include(SwiftStripBuilderRpath)
 
 # Configure swift include, install, build components.
 swift_configure_components()

SwiftCompilerSources/CMakeLists.txt

@@ -235,7 +235,7 @@ else()
   #
   # step 1: generate a dummy source file, which just includes all headers
   # defined in include/swift/module.modulemap
-  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+  file(WRITE "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp"
        "
 #include \"Basic/BridgedSwiftObject.h\"
 #include \"Basic/BasicBridging.h\"
@@ -251,6 +251,12 @@ else()
 
 #include \"Parse/RegexParserBridging.h\"
 ")
+  add_custom_command(
+    OUTPUT "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+    COMMAND ${CMAKE_COMMAND} -E copy_if_different
+      "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp.tmp"
+      "${CMAKE_CURRENT_BINARY_DIR}/HeaderDependencies.cpp"
+  )
 
   # step 2: build a library containing that source file. This library depends on all the included header files.
   #         The swift modules can now depend on that target.

cmake/modules/AddPureSwift.cmake

@@ -65,6 +65,27 @@ function(_add_host_swift_compile_options name)
   _add_host_variant_swift_sanitizer_flags(${name})
 endfunction()
 
+function(_set_pure_swift_link_flags name relpath_to_lib_dir)
+  if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+    # Don't add builder's stdlib RPATH automatically, because we want to *de-prioritize* it.
+    target_compile_options(${name} PRIVATE
+      $<$<COMPILE_LANGUAGE:Swift>:-no-toolchain-stdlib-rpath>
+    )
+
+    get_filename_component(swift_bin_dir ${CMAKE_Swift_COMPILER} DIRECTORY)
+    get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY)
+    set(host_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
+
+    set_property(TARGET ${name}
+      APPEND PROPERTY INSTALL_RPATH
+        # At runtime, use swiftCore in the current toolchain.
+        "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}"
+        # But before building the stdlib with the tool, use the builder libs. This should be removed in install time.
+        "${host_lib_dir}")
+
+  endif()
+endfunction()
+
 # Add a new "pure" Swift host library.
 #
 # "Pure" Swift host libraries can only contain Swift code, and will be built
@@ -266,6 +287,7 @@ function(add_pure_swift_host_tool name)
   # Create the library.
   add_executable(${name} ${APSHT_SOURCES})
   _add_host_swift_compile_options(${name})
+  _set_pure_swift_link_flags(${name} "../lib")
 
   if(SWIFT_HOST_VARIANT_SDK IN_LIST SWIFT_DARWIN_PLATFORMS)
     set_property(TARGET ${name}

cmake/modules/AddSwift.cmake

@@ -548,10 +548,14 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
       target_link_libraries(${target} PRIVATE "swiftCore")
 
       target_link_directories(${target} PRIVATE ${host_lib_dir})
+
+      # At runtime, use swiftCore in the current toolchain.
+      # FIXME: This assumes the ABI hasn't changed since the builder.
+      set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
+
       if(ASRLF_BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS")
-        set(swift_runtime_rpath "${host_lib_dir}")
-      else()
-        set(swift_runtime_rpath "$ORIGIN/${relpath_to_lib_dir}/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
+        # But before building the stdlib with the tool, use the builder libs. This should be removed in install time.
+        list(APPEND swift_runtime_rpath "${host_lib_dir}")
       endif()
 
     elseif(ASRLF_BOOTSTRAPPING_MODE STREQUAL "BOOTSTRAPPING")
@@ -576,9 +580,6 @@ function(_add_swift_runtime_link_flags target relpath_to_lib_dir bootstrapping)
   endif()
 
   if(SWIFT_SWIFT_PARSER)
-    # Make sure we can find the early SwiftSyntax libraries.
-    target_link_directories(${target} PRIVATE "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
-
     # For the "end step" of bootstrapping configurations, we need to be
     # able to fall back to the SDK directory for libswiftCore et al.
     if (BOOTSTRAPPING_MODE MATCHES "BOOTSTRAPPING.*")
@@ -967,7 +968,14 @@ function(add_swift_host_tool executable)
     swift_install_in_component(TARGETS ${executable}
                                RUNTIME
                                  DESTINATION bin
-                                 COMPONENT ${ASHT_SWIFT_COMPONENT})
+                                 COMPONENT ${ASHT_SWIFT_COMPONENT}
+    )
+
+    swift_install_strip_builder_rpath(
+      TARGETS ${executable}
+      DESTINATION bin
+      COMPONENT ${ASHT_SWIFT_COMPONENT}
+    )
 
     swift_is_installing_component(${ASHT_SWIFT_COMPONENT} is_installing)
   endif()

cmake/modules/AddSwiftUnittests.cmake

@@ -5,10 +5,32 @@ add_custom_target(SwiftUnitTests)
 
 set_target_properties(SwiftUnitTests PROPERTIES FOLDER "Tests")
 
+# Add a new Swift unit test executable.
+#
+# Usage:
+#   add_swift_unittest(name
+#     [IS_TARGET_TEST]
+#     source1 [source2 source3 ...])
+#
+# name
+#   Name of the test (e.g., SwiftASTTest).
+#
+# IS_TARGET_TEST
+#   Indicates this is a test for target libraries. Not host library.
+#   This avoids linking with toolchains stdlib.
+#
+# source1 ...
+#   Sources to add into this executable.
 function(add_swift_unittest test_dirname)
+  cmake_parse_arguments(ASU
+                        "IS_TARGET_TEST"
+                        ""
+                        ""
+                        ${ARGN})
+
   # *NOTE* Even though "add_unittest" does not have llvm in its name, it is a
   # function defined by AddLLVM.cmake.
-  add_unittest(SwiftUnitTests ${test_dirname} ${ARGN})
+  add_unittest(SwiftUnitTests ${test_dirname} ${ASU_UNPARSED_ARGUMENTS})
 
   set_target_properties(${test_dirname} PROPERTIES LINKER_LANGUAGE CXX)
 
@@ -89,7 +111,8 @@ function(add_swift_unittest test_dirname)
     endif()
   endif()
 
-  if (SWIFT_SWIFT_PARSER)
+  if (SWIFT_SWIFT_PARSER AND NOT ASU_IS_TARGET_TEST)
+    # Link to stdlib the compiler uses.
     _add_swift_runtime_link_flags(${test_dirname} "../../lib" "")
     set_property(TARGET ${test_dirname} PROPERTY BUILD_WITH_INSTALL_RPATH OFF)
   endif()

cmake/modules/SwiftImplicitImport.cmake

@@ -6,7 +6,7 @@ function(swift_supports_implicit_module module_name out_var)
       "${CMAKE_Swift_COMPILER}"
       -Xfrontend -disable-implicit-${module_name}-module-import
       -Xfrontend -parse-stdlib
-      -c - -o /dev/null
+      -parse -
     INPUT_FILE
       "${CMAKE_BINARY_DIR}/tmp/empty-check-${module_name}.swift"
     OUTPUT_QUIET ERROR_QUIET

cmake/modules/SwiftStripBuilderRpath.cmake

@@ -0,0 +1,71 @@
+set(SWIFT_SET_RPATH_SCRIPT_FILE "${CMAKE_CURRENT_LIST_FILE}")
+
+function(swift_get_set_rpath_script_file out_var)
+  set(${out_var} "${SWIFT_SET_RPATH_SCRIPT_FILE}" PARENT_SCOPE)
+endfunction()
+
+# Actual RPATH_SET operation to the file.
+function(_swift_set_rpath_impl file new_rpath)
+  # NOTE: RPATH_SET is not documented, and works only for ELF and XCOFF. 
+  file(RPATH_SET FILE "${file}" NEW_RPATH "${new_rpath}")
+endfunction()
+
+# For 'install(SCRIPT <script> CODE swift_set_rpath(...)'.
+function(_swift_file_set_rpath_installed file new_rpath)
+  set(DESTDIR $ENV{DESTDIR})
+  if(NOT IS_ABSOLUTE "${file}")
+    string(PREPEND file "${CMAKE_INSTALL_PREFIX}/")
+  endif()
+  string(PREPEND file "${DESTDIR}")
+  _swift_set_rpath_impl("${file}" "${new_rpath}")
+endfunction()
+
+# Add 'install' script that set runtime path to the specified file.
+function(swift_install_file_set_rpath file install_rpath component)
+  if(NOT(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX"))
+      return()
+  endif()
+  if((NOT SWIFT_SWIFT_PARSER) AND NOT(BOOTSTRAPPING_MODE STREQUAL "HOSTTOOLS"))
+      return()
+  endif()
+
+  swift_get_set_rpath_script_file(script)
+
+  swift_install_in_component(
+    SCRIPT "${script}" 
+    CODE "_swift_file_set_rpath_installed(\"${file}\" \"${install_rpath}\")"
+    COMPONENT ${component}
+  )
+endfunction()
+
+# Add 'install' script that set runtime path to the target but only startinig with '$ORIGIN'
+function(swift_install_strip_builder_rpath)
+  cmake_parse_arguments(RPATH
+                        ""
+                        "DESTINATION;COMPONENT"
+                        "TARGETS"
+                        ${ARGN})
+
+  foreach(target ${RPATH_TARGETS})
+    # Filter out RPATHs *not* starting with $ORIGIN
+    set(stripped_rpaths)
+    get_target_property(install_rpaths ${target} INSTALL_RPATH)
+    foreach(path ${install_rpaths})
+        if (path MATCHES "^\\$ORIGIN")
+        list(APPEND stripped_rpaths "${path}")
+        endif()
+    endforeach()
+
+    list(JOIN stripped_rpaths ":" install_rpath_str)
+    swift_install_file_set_rpath(
+      "${RPATH_DESTINATION}/$<TARGET_FILE_NAME:${target}>"
+      "${install_rpath_str}"
+      ${RPATH_COMPONENT}
+    )
+  endforeach()
+endfunction()
+
+# For 'cmake -P <scirpt>'.
+if (SWIFT_SET_RPATH_FILE AND SWIFT_SET_RPATH_NEW_RPATH)
+  _swift_set_rpath_impl("${SWIFT_SET_RPATH_FILE}" "${SWIFT_SET_RPATH_NEW_RPATH}")
+endif()

lib/CMakeLists.txt

@@ -33,7 +33,7 @@ if (SWIFT_SWIFT_PARSER)
   list(TRANSFORM SWIFT_SYNTAX_MODULES PREPEND "SwiftSyntax::"
        OUTPUT_VARIABLE SWIFT_SYNTAX_TARGETS)
 
-  set(SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR
+  set(SWIFT_SYNTAX_LIBRARIES_BUILD_DIR
       "${SWIFT_PATH_TO_EARLYSWIFTSYNTAX_BUILD_DIR}/lib/swift/host")
   set(SWIFT_HOST_LIBRARIES_DEST_DIR
       "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}/swift/host")
@@ -53,17 +53,46 @@ if (SWIFT_SWIFT_PARSER)
   # Copy over all of the shared libraries from earlyswiftsyntax so they can
   # be found via RPATH.
   foreach (sharedlib ${SWIFT_SYNTAX_SHARED_LIBRARIES})
+    set(add_origin_rpath)
+    if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+      get_filename_component(swift_bin_dir ${CMAKE_Swift_COMPILER} DIRECTORY)
+      get_filename_component(swift_dir ${swift_bin_dir} DIRECTORY)
+      set(host_lib_dir "${swift_dir}/lib/swift/${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}")
+
+      # At runtime, use swiftCore in the current toolchain.
+      
+      swift_get_set_rpath_script_file(setrpath_command)
+      set(add_origin_rpath COMMAND ${CMAKE_COMMAND}
+        "-DSWIFT_SET_RPATH_FILE=${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
+        "-DSWIFT_SET_RPATH_NEW_RPATH='$$ORIGIN:$$ORIGIN/../${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}:${host_lib_dir}'"
+        -P "${setrpath_command}"
+      )
+    endif()
+
     add_custom_command(
       OUTPUT "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
-      DEPENDS "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib}"
-      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${sharedlib} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}
+      DEPENDS "${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${sharedlib}"
+      COMMAND ${CMAKE_COMMAND} -E copy_if_different ${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${sharedlib} ${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}
+      ${add_origin_rpath}
     )
 
     add_custom_target(copy_swiftSyntaxLibrary_${sharedlib}
       DEPENDS "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
       COMMENT "Copying ${sharedlib}"
     )
 
+    swift_install_in_component(
+      FILES "${SWIFT_HOST_LIBRARIES_DEST_DIR}/${sharedlib}"
+      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host"
+      COMPONENT compiler
+    )
+
+    swift_install_file_set_rpath(
+      "lib${LLVM_LIBDIR_SUFFIX}/swift/host/${sharedlib}"
+      "$ORIGIN:$ORIGIN/../${SWIFT_SDK_${SWIFT_HOST_VARIANT_SDK}_LIB_SUBDIR}"
+      compiler
+    )
+
     add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxLibrary_${sharedlib})
   endforeach()
 
@@ -76,7 +105,7 @@ if (SWIFT_SWIFT_PARSER)
   foreach(module_dir ${SWIFT_SYNTAX_MODULE_DIRS})
     # Find all of the source module files.
     file(GLOB module_files
-         "${SWIFT_SYNTAX_LIBRARIES_SOURCE_DIR}/${module_dir}/*.swiftinterface")
+      "${SWIFT_SYNTAX_LIBRARIES_BUILD_DIR}/${module_dir}/*.swiftinterface")
 
     # Determine the destination module files.
     set(dest_module_files)
@@ -98,6 +127,12 @@ if (SWIFT_SWIFT_PARSER)
       COMMENT "Copying ${module_dir}"
     )
 
+    swift_install_in_component(
+      FILES ${dest_module_files}
+      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/${module_dir}"
+      COMPONENT compiler
+    )
+
     add_dependencies(swiftSyntaxLibraries copy_swiftSyntaxModule_${module_dir})
   endforeach()
 

lib/Macros/CMakeLists.txt

@@ -31,6 +31,19 @@ function(add_swift_macro_library name)
   # Add the library.
   add_pure_swift_host_library(${name} SHARED ${ASML_SOURCES})
 
+  # Add rpath to 'lib/{platform}'
+  file(RELATIVE_PATH relpath_to_lib
+    "${SWIFT_HOST_PLUGINS_DEST_DIR}"
+    "${CMAKE_LIBRARY_OUTPUT_DIRECTORY}"
+  )
+  _set_pure_swift_link_flags(${name} "${relpath_to_lib}")
+
+  # Add rpath to 'lib/host'
+  if(SWIFT_HOST_VARIANT_SDK STREQUAL "LINUX")
+    set_property(TARGET ${name}
+      APPEND PROPERTY INSTALL_RPATH "$ORIGIN/..")
+  endif()
+
   # If we don't have the Swift swift parser, bail out, because the above
   # add_pure_swift_host_library did nothing.
   if (NOT SWIFT_SWIFT_PARSER)
@@ -44,14 +57,22 @@ function(add_swift_macro_library name)
       LIBRARY_OUTPUT_DIRECTORY "${SWIFT_HOST_PLUGINS_DEST_DIR}"
   )
 
+  set(destination_dir "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins")
+
   swift_install_in_component(TARGETS ${name}
     LIBRARY
-      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins"
+      DESTINATION "${destination_dir}"
       COMPONENT compiler
     ARCHIVE
-      DESTINATION "lib${LLVM_LIBDIR_SUFFIX}/swift/host/plugins"
+      DESTINATION "${destination_dir}"
       COMPONENT compiler)
 
+  swift_install_strip_builder_rpath(
+    TARGETS ${name}
+    DESTINATION "${destination_dir}"
+    COMPONENT compiler
+  )
+
   # Export this macro plugin target.
   set_property(GLOBAL APPEND PROPERTY SWIFT_MACRO_PLUGINS ${name})
 endfunction()

lib/SwiftDemangle/CMakeLists.txt

@@ -17,3 +17,10 @@ swift_install_in_component(TARGETS swiftDemangle
   ARCHIVE
     DESTINATION "lib${LLVM_LIBDIR_SUFFIX}"
     COMPONENT compiler)
+
+
+swift_install_strip_builder_rpath(
+  TARGETS swiftDemangle
+  DESTINATION "lib${LLVM_LIBDIR_SUFFIX}"
+  COMPONENT compiler
+)