Support building on macOS to develop this project (#8)

* Add simple GitHub Actions config for macOS
* Add brew install to the macOS GH action
* Add update-checkout call to macOS GH action
* Export sourcedir shell variable in macOS GH action
* [WASM] Use prebuilt llvm-ar only on Linux
* [WASM] Added new Object format WASM to distinguish from ELF
* [WASM] Separate SwiftRT from ELF
* [WASM] Add preprocessor condition for wasm32 to track latest upstream
* Add wasi-sdk and icu install steps to the script
* [WASM] Set LIBC_INCLUDE_DIRECTORY without CACHE attribute to force new value
* [WASM] Adding share path to point correct directory
* [WASM] Add Linux CI job
* [WASM] Checkout branch after checking out Swift repos
* [WASM] Use release build to reduce artifact size
* [WASM] Use SWIFT_PRIMARY_VARIANT_SDK instead of HOST_SDK
In this case, SWIFT_PRIMARY_VARIANT_ARCH is used but host SDK was
not PRIMARY_SDK.
* [WASM] Remove ICU_STATICLIB on Linux
* [WASM] Output build log verbose
* [WASM] Set ICU lib directory instead of archive
* [WASM] Sort build options
* Revert "[WASM] Use SWIFT_PRIMARY_VARIANT_SDK instead of HOST_SDK"

Author: kateinoigakukun

.github/workflows/main.yml

@@ -0,0 +1,111 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+    - swiftwasm
+
+jobs:
+  linux_build:
+    timeout-minutes: 0
+    runs-on: ubuntu-18.04
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Run a multi-line script
+      run: |
+        sudo apt update
+        sudo apt install \
+          git ninja-build clang python \
+          uuid-dev libicu-dev icu-devtools libbsd-dev \
+          libedit-dev libxml2-dev libsqlite3-dev swig \
+          libpython-dev libncurses5-dev pkg-config \
+          libblocksruntime-dev libcurl4-openssl-dev \
+          systemtap-sdt-dev tzdata rsync
+
+        ./utils/update-checkout --clone --scheme wasm
+        git checkout $GITHUB_SHA
+        export sourcedir=$PWD/..
+        cd $sourcedir
+
+        wget -O install_cmake.sh "https://github.com/Kitware/CMake/releases/download/v3.15.3/cmake-3.15.3-Linux-x86_64.sh"
+        chmod +x install_cmake.sh
+        sudo mkdir -p /opt/cmake
+        sudo ./install_cmake.sh --skip-license --prefix=/opt/cmake
+        sudo ln -sf /opt/cmake/bin/* /usr/local/bin
+        cmake --version
+
+        wget -O wasi-sdk.tar.gz https://github.com/swiftwasm/wasi-sdk/releases/download/20190421.6/wasi-sdk-3.19gefb17cb478f9.m-linux.tar.gz
+        tar xfz wasi-sdk.tar.gz
+        mv wasi-sdk-3.19gefb17cb478f9+m/opt/wasi-sdk ./wasi-sdk
+
+        wget -O icu.tar.xz "https://github.com/swiftwasm/icu4c-wasi/releases/download/20190421.3/icu4c-wasi.tar.xz"
+        tar xf icu.tar.xz
+
+        cd swift
+        utils/build-script --release --wasm --verbose \
+          --skip-build-benchmarks \
+          --extra-cmake-options=" \
+            -DSWIFT_SDKS='WASM;LINUX' \
+            -DSWIFT_BUILD_SOURCEKIT=FALSE \
+            -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE \
+          " \
+          --build-stdlib-deployment-targets "wasm-wasm32" \
+          --build-swift-static-stdlib \
+          --install-destdir="$sourcedir/install" \
+          --install-prefix="/opt/swiftwasm-sdk" \
+          --install-swift \
+          --installable-package="$sourcedir/swiftwasm.tar.gz" \
+          --llvm-targets-to-build "X86;WebAssembly" \
+          --stdlib-deployment-targets "wasm-wasm32" \
+          --wasm-icu-data "todo-icu-data" \
+          --wasm-icu-i18n "$sourcedir/icu_out/lib" \
+          --wasm-icu-i18n-include "$sourcedir/icu_out/include" \
+          --wasm-icu-uc "$sourcedir/icu_out/lib" \
+          --wasm-icu-uc-include "$sourcedir/icu_out/include" \
+          --wasm-wasi-sdk "$sourcedir/wasi-sdk"
+
+  macos_build:
+    timeout-minutes: 0
+    runs-on: macOS-10.14
+
+    steps:
+    - uses: actions/checkout@v1
+    - name: Run a multi-line script
+      run: |
+        brew install cmake ninja
+        ./utils/update-checkout --clone --scheme wasm
+        git checkout $GITHUB_SHA
+        export sourcedir=$PWD/..
+        cd $sourcedir
+        wget -O wasi-sdk.tar.gz https://github.com/swiftwasm/wasi-sdk/releases/download/20190421.6/wasi-sdk-3.19gefb17cb478f9.m-linux.tar.gz
+        tar xfz wasi-sdk.tar.gz
+        mv wasi-sdk-3.19gefb17cb478f9+m/opt/wasi-sdk ./wasi-sdk
+        # Link sysroot/usr/include to sysroot/include because Darwin sysroot doesn't 
+        # find header files in sysroot/include but sysroot/usr/include
+        mkdir wasi-sdk/share/sysroot/usr/
+        ln -s ../include wasi-sdk/share/sysroot/usr/include
+        wget -O icu.tar.xz "https://github.com/swiftwasm/icu4c-wasi/releases/download/20190421.3/icu4c-wasi.tar.xz"
+        tar xf icu.tar.xz
+        cd swift
+        ./utils/build-script --release --wasm --verbose \
+          --skip-build-benchmarks \
+          --extra-cmake-options=" \
+            -DSWIFT_PRIMARY_VARIANT_SDK:STRING=WASM \
+            -DSWIFT_PRIMARY_VARIANT_ARCH:STRING=wasm32 \
+            -DSWIFT_OSX_x86_64_ICU_STATICLIB=TRUE \
+            -DSWIFT_BUILD_SOURCEKIT=FALSE \
+            -DSWIFT_ENABLE_SOURCEKIT_TESTS=FALSE \
+          " \
+          --build-stdlib-deployment-targets "wasm-wasm32" \
+          --build-swift-dynamic-sdk-overlay false \
+          --build-swift-static-sdk-overlay false \
+          --build-swift-static-stdlib \
+          --llvm-targets-to-build "X86;WebAssembly" \
+          --stdlib-deployment-targets "wasm-wasm32" \
+          --wasm-icu-data "todo-icu-data" \
+          --wasm-icu-i18n "$sourcedir/icu_out/lib" \
+          --wasm-icu-i18n-include "$sourcedir/icu_out/include" \
+          --wasm-icu-uc "$sourcedir/icu_out/lib" \
+          --wasm-icu-uc-include "$sourcedir/icu_out/include" \
+          --wasm-wasi-sdk "$sourcedir/wasi-sdk"

CMakeLists.txt

@@ -36,9 +36,6 @@ ENABLE_LANGUAGE(C)
 include(SwiftUtils)
 include(CheckSymbolExists)
 
-# WebAssembly: hack: use llvm-ar for creating static libraries; Ubuntu's GNU ar doesn't work with wasm-ld
-set(CMAKE_AR "${SWIFT_WASM_WASI_SDK_PATH}/bin/llvm-ar")
-
 #
 # User-configurable options that control the inclusion and default build
 # behavior for components which may not strictly be necessary (tools, examples,
@@ -794,6 +791,11 @@ if(swift_build_wasm AND NOT "${SWIFT_HOST_VARIANT_SDK}" STREQUAL "WASM")
   #  message(FATAL_ERROR "A Darwin or Linux host is required to build the Swift runtime for Android")
   #endif()
 
+  if ("${CMAKE_HOST_SYSTEM_NAME}" STREQUAL "Linux")
+    # WebAssembly: hack: use llvm-ar for creating static libraries; Ubuntu's GNU ar doesn't work with wasm-ld
+    set(CMAKE_AR "${SWIFT_WASM_WASI_SDK_PATH}/bin/llvm-ar")
+  endif()
+
   if("${SWIFT_SDK_WASM_ARCHITECTURES}" STREQUAL "")
     set(SWIFT_SDK_WASM_ARCHITECTURES wasm32)
   endif()

cmake/modules/AddSwift.cmake

@@ -1076,7 +1076,8 @@ function(_add_swift_library_single target name)
               ${INCORPORATED_OBJECT_LIBRARIES_EXPRESSIONS}
               ${SWIFTLIB_SINGLE_XCODE_WORKAROUND_SOURCES})
   if(("${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "ELF" OR
-      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "COFF") AND
+      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "COFF" OR
+      "${SWIFT_SDK_${SWIFTLIB_SINGLE_SDK}_OBJECT_FORMAT}" STREQUAL "WASM") AND
      SWIFTLIB_SINGLE_TARGET_LIBRARY)
     if("${libkind}" STREQUAL "SHARED" AND NOT SWIFTLIB_SINGLE_NOSWIFTRT)
       # TODO(compnerd) switch to the generator expression when cmake is upgraded

cmake/modules/SwiftConfigureSDK.cmake

@@ -217,6 +217,8 @@ macro(configure_sdk_unix name architectures)
   set(SWIFT_SDK_${prefix}_ARCHITECTURES "${architectures}")
   if("${prefix}" STREQUAL "CYGWIN")
     set(SWIFT_SDK_${prefix}_OBJECT_FORMAT "COFF")
+  elseif("${prefix}" STREQUAL "WASM")
+    set(SWIFT_SDK_${prefix}_OBJECT_FORMAT "WASM")
   else()
     set(SWIFT_SDK_${prefix}_OBJECT_FORMAT "ELF")
   endif()
@@ -340,8 +342,8 @@ macro(configure_sdk_unix name architectures)
         set(SWIFT_SDK_WASM_ARCH_wasm32_PATH "${SWIFT_WASM_WASI_SDK_PATH}/share/sysroot")
         # fixme: Wasi is wasm32-unknown-wasi-musl. This LLVM doesn't have it yet.
         set(SWIFT_SDK_WASM_ARCH_wasm32_TRIPLE "wasm32-unknown-unknown-wasm")
-        set(SWIFT_SDK_WASM_ARCH_wasm32_LIBC_INCLUDE_DIRECTORY "${SWIFT_WASM_WASI_SDK_PATH}/share/sysroot/include" CACHE STRING "Path to C library headers")
-        set(SWIFT_SDK_WASM_ARCH_wasm32_LIBC_ARCHITECTURE_INCLUDE_DIRECTORY "${SWIFT_WASM_WASI_SDK_PATH}/sysroot/include" CACHE STRING "Path to C library architecture headers")
+        set(SWIFT_SDK_WASM_ARCH_wasm32_LIBC_INCLUDE_DIRECTORY "${SWIFT_WASM_WASI_SDK_PATH}/share/sysroot/include")
+        set(SWIFT_SDK_WASM_ARCH_wasm32_LIBC_ARCHITECTURE_INCLUDE_DIRECTORY "${SWIFT_WASM_WASI_SDK_PATH}/share/sysroot/include")
       else()
         message(FATAL_ERROR "unknown Unix OS: ${prefix}")
       endif()

lib/Driver/CMakeLists.txt

@@ -33,7 +33,8 @@ target_link_libraries(swiftDriver PRIVATE
 if(SWIFT_BUILD_STATIC_STDLIB)
   set(static_stdlib_lnk_file_list)
   foreach(sdk ${SWIFT_CONFIGURED_SDKS})
-    if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF")
+    if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF" OR
+       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
       string(TOLOWER "${sdk}" lowercase_sdk)
       if(SWIFT_${SWIFT_HOST_VARIANT_SDK}_${SWIFT_HOST_VARIANT_ARCH}_ICU_STATICLIB)
         set(ICU_STATICLIB "TRUE")

stdlib/public/core/StringStorage.swift

@@ -186,7 +186,7 @@ extension __StringStorage {
     let count = try initializer(buffer)
 
     let countAndFlags = CountAndFlags(mortalCount: count, isASCII: false)
-    #if arch(i386) || arch(arm)
+    #if arch(i386) || arch(arm) || arch(wasm32)
     storage._count = countAndFlags.count
     storage._flags = countAndFlags.flags
     #else

stdlib/public/runtime/CMakeLists.txt

@@ -80,6 +80,7 @@ endif(LLVM_ENABLE_ASSERTIONS)
 set(LLVM_OPTIONAL_SOURCES
     SwiftRT-COFF.cpp
     SwiftRT-ELF.cpp
+    SwiftRT-WASM.cpp
     ${swift_runtime_sources}
     ${swift_runtime_objc_sources}
     ${swift_runtime_leaks_sources})
@@ -179,11 +180,14 @@ add_swift_target_library(swiftRuntime OBJECT_LIBRARY
 
 set(ELFISH_SDKS)
 set(COFF_SDKS)
+set(WASM_SDKS)
 foreach(sdk ${SWIFT_CONFIGURED_SDKS})
   if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF")
     list(APPEND ELFISH_SDKS ${sdk})
   elseif("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF")
     list(APPEND COFF_SDKS ${sdk})
+  elseif("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
+    list(APPEND WASM_SDKS ${sdk})
   endif()
 endforeach()
 
@@ -196,6 +200,16 @@ add_swift_target_library(swiftImageRegistrationObjectELF
                   TARGET_SDKS ${ELFISH_SDKS}
                   SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
                   INSTALL_IN_COMPONENT none)
+
+add_swift_target_library(swiftImageRegistrationObjectWASM
+                  OBJECT_LIBRARY IS_STDLIB IS_STDLIB_CORE
+                  SwiftRT-WASM.cpp
+                  C_COMPILE_FLAGS ${SWIFT_RUNTIME_CORE_CXX_FLAGS}
+                  LINK_FLAGS ${SWIFT_RUNTIME_CORE_LINK_FLAGS}
+                  TARGET_SDKS ${WASM_SDKS}
+                  SWIFT_COMPILE_FLAGS ${SWIFT_STANDARD_LIBRARY_SWIFT_FLAGS}
+                  INSTALL_IN_COMPONENT none)
+
 # FIXME(compnerd) this should be compiled twice, once for static and once for
 # shared.  The static version should be used for building the standard library.
 add_swift_target_library(swiftImageRegistrationObjectCOFF
@@ -213,7 +227,8 @@ foreach(sdk ${SWIFT_CONFIGURED_SDKS})
     set(arch_suffix "${SWIFT_SDK_${sdk}_LIB_SUBDIR}-${arch}")
 
     if("${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "ELF" OR
-       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF")
+       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "COFF" OR
+       "${SWIFT_SDK_${sdk}_OBJECT_FORMAT}" STREQUAL "WASM")
       # TODO(compnerd) switch to the generator expression when cmake is upgraded
       # to a version which supports it.
       # set(swiftrtObject "$<TARGET_OBJECTS:swiftImageRegistrationObject${SWIFT_SDK_${sdk}_OBJECT_FORMAT}-${arch_suffix}>")

stdlib/public/runtime/SwiftRT-ELF.cpp

@@ -46,16 +46,9 @@ static swift::MetadataSections sections{};
 
 __attribute__((__constructor__))
 static void swift_image_constructor() {
-#ifndef __wasm__
 #define SWIFT_SECTION_RANGE(name)                                              \
   { reinterpret_cast<uintptr_t>(&__start_##name),                              \
     static_cast<uintptr_t>(&__stop_##name - &__start_##name) }
-#else
-// WebAssembly hack: ok this should really go in its own file
-#define SWIFT_SECTION_RANGE(name)                                              \
-  { reinterpret_cast<uintptr_t>(&__start_##name) + sizeof(void*),                              \
-    static_cast<uintptr_t>(&__stop_##name - &__start_##name - sizeof(void*)) }
-#endif
 
   sections = {
       swift::CurrentSectionMetadataVersion,

stdlib/public/runtime/SwiftRT-WASM.cpp

@@ -0,0 +1,11 @@
+//===--- SwiftRT-WASM.cpp --------------------------------------------------===//
+//
+// This source file is part of the Swift.org open source project
+//
+// Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+// Licensed under Apache License v2.0 with Runtime Library Exception
+//
+// See https://swift.org/LICENSE.txt for license information
+// See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+//
+//===----------------------------------------------------------------------===//