feat: update metadata generator to work with llvm 13.0.1 and CLANG_ENABLE_MODULES

Author: edusperoni

.gitignore

@@ -43,3 +43,5 @@ package-lock.json
 *.pyc
 v8
 .npmrc
+
+llvm/

README.md

@@ -5,13 +5,20 @@ To start diving into the v8 iOS runtime make sure you have XCode and [Homebrew](
 # Install CMake
 brew install cmake
 
+# To avoid errors, you might need to link cmake to: /usr/local/bin/cmake
+# xcode doesn't read your profile during the build step, which causes it to ignore the PATH
+sudo ln -s /usr/local/bin/cmake $(which cmake)
+
 # Clone repo
 git clone https://github.com/NativeScript/ns-v8ios-runtime.git
 
 # Initialize and clone the submodules
 cd ns-v8ios-runtime
 git submodule update --init
 
+# Ensure that you have the required llvm binaries for building the metadata generator
+./download_llvm.sh
+
 # Open the runtime in XCode
 open v8ios.xcodeproj
 ```

build_metadata_generator.sh

@@ -1,13 +1,19 @@
 #!/bin/bash
 set -e
 
+./download_llvm.sh
+
+# try to build in the amount of threads available
+# change this to 1 if you want single threaded builds
+NUMJOBS=$(nproc)
+
 function build {
     rm -rf build
     mkdir build
     pushd "build"
     cmake -DCMAKE_BUILD_TYPE=Release -DMETADATA_BINARY_ARCH=$1 -DCMAKE_OSX_ARCHITECTURES=$1 ../
     make clean
-    make
+    make -j$NUMJOBS
     cp ../build-step-metadata-generator.py bin
     popd
     mkdir "dist/$1"

download_llvm.sh

@@ -0,0 +1,15 @@
+#!/bin/bash
+set -e
+
+function download_llvm() {
+    echo 'Downloading llvm...'
+    mkdir -p /tmp/llvm-dl/
+    curl -L https://github.com/NativeScript/ios-llvm/releases/download/v13.0.1/llvm-13.0.1.tgz -o /tmp/llvm-dl/llvm-13.0.1.tgz
+    mkdir -p ./llvm/
+    echo 'extracting llvm...'
+    tar -xzf /tmp/llvm-dl/llvm-13.0.1.tgz -C ./llvm/
+}
+
+if [ ! -d "./llvm/13.0.1" ]; then
+    download_llvm
+fi

llvm

@@ -1,5 +1,5 @@
 project(MetadataGenerator)
-cmake_minimum_required(VERSION 2.8)
+cmake_minimum_required(VERSION 3.20)
 
 #set(CMAKE_VERBOSE_MAKEFILE ON)
 
@@ -12,16 +12,20 @@ if (NOT LIBXML2_FOUND)
     message(FATAL_ERROR "libXML2 not found")
 endif ()
 
-get_filename_component(LLVM_ROOT "../../llvm/8.0.0" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
+get_filename_component(LLVM_ROOT "../../llvm/13.0.1" REALPATH BASE_DIR "${CMAKE_BINARY_DIR}")
 set(CMAKE_OSX_DEPLOYMENT_TARGET 10.12)
 
 set(LLVM_SYSTEM_LIBS "-lz -lcurses -lm -lxml2")
 set(LLVM_PREPROCESSOR_FLAGS "-I${LLVM_ROOT}/include -D__STDC_CONSTANT_MACROS -D__STDC_FORMAT_MACROS -D__STDC_LIMIT_MACROS")
-set(LLVM_LIBS "-lLLVMLTO -lLLVMPasses -lLLVMObjCARCOpts -lLLVMMIRParser -lLLVMSymbolize -lLLVMDebugInfoPDB -lLLVMDebugInfoDWARF -lLLVMCoverage -lLLVMMCA -lLLVMTableGen -lLLVMDlltoolDriver -lLLVMXRay -lLLVMOrcJIT -lLLVMXCoreDisassembler -lLLVMXCoreCodeGen -lLLVMXCoreDesc -lLLVMXCoreInfo -lLLVMXCoreAsmPrinter -lLLVMWebAssemblyDisassembler -lLLVMWebAssemblyCodeGen -lLLVMWebAssemblyDesc -lLLVMWebAssemblyAsmPrinter -lLLVMWebAssemblyAsmParser -lLLVMWebAssemblyInfo -lLLVMSystemZDisassembler -lLLVMSystemZCodeGen -lLLVMSystemZAsmParser -lLLVMSystemZDesc -lLLVMSystemZInfo -lLLVMSystemZAsmPrinter -lLLVMSparcDisassembler -lLLVMSparcCodeGen -lLLVMSparcAsmParser -lLLVMSparcDesc -lLLVMSparcInfo -lLLVMSparcAsmPrinter -lLLVMPowerPCDisassembler -lLLVMPowerPCCodeGen -lLLVMPowerPCAsmParser -lLLVMPowerPCDesc -lLLVMPowerPCInfo -lLLVMPowerPCAsmPrinter -lLLVMNVPTXCodeGen -lLLVMNVPTXDesc -lLLVMNVPTXInfo -lLLVMNVPTXAsmPrinter -lLLVMMSP430Disassembler -lLLVMMSP430CodeGen -lLLVMMSP430AsmParser -lLLVMMSP430Desc -lLLVMMSP430Info -lLLVMMSP430AsmPrinter -lLLVMMipsDisassembler -lLLVMMipsCodeGen -lLLVMMipsAsmParser -lLLVMMipsDesc -lLLVMMipsInfo -lLLVMMipsAsmPrinter -lLLVMLanaiDisassembler -lLLVMLanaiCodeGen -lLLVMLanaiAsmParser -lLLVMLanaiDesc -lLLVMLanaiAsmPrinter -lLLVMLanaiInfo -lLLVMHexagonDisassembler -lLLVMHexagonCodeGen -lLLVMHexagonAsmParser -lLLVMHexagonDesc -lLLVMHexagonInfo -lLLVMBPFDisassembler -lLLVMBPFCodeGen -lLLVMBPFAsmParser -lLLVMBPFDesc -lLLVMBPFInfo -lLLVMBPFAsmPrinter -lLLVMARMDisassembler -lLLVMARMCodeGen -lLLVMARMAsmParser -lLLVMARMDesc -lLLVMARMInfo -lLLVMARMAsmPrinter -lLLVMARMUtils -lLLVMAMDGPUDisassembler -lLLVMAMDGPUCodeGen -lLLVMAMDGPUAsmParser -lLLVMAMDGPUDesc -lLLVMAMDGPUInfo -lLLVMAMDGPUAsmPrinter -lLLVMAMDGPUUtils -lLLVMAArch64Disassembler -lLLVMAArch64CodeGen -lLLVMAArch64AsmParser -lLLVMAArch64Desc -lLLVMAArch64Info -lLLVMAArch64AsmPrinter -lLLVMAArch64Utils -lLLVMObjectYAML -lLLVMLibDriver -lLLVMOption -lLLVMOptRemarks -lLLVMWindowsManifest -lLLVMTextAPI -lLLVMFuzzMutate -lLLVMX86Disassembler -lLLVMX86AsmParser -lLLVMX86CodeGen -lLLVMGlobalISel -lLLVMSelectionDAG -lLLVMAsmPrinter -lLLVMX86Desc -lLLVMMCDisassembler -lLLVMX86Info -lLLVMX86AsmPrinter -lLLVMX86Utils -lLLVMMCJIT -lLLVMLineEditor -lLLVMInterpreter -lLLVMExecutionEngine -lLLVMRuntimeDyld -lLLVMCodeGen -lLLVMTarget -lLLVMCoroutines -lLLVMipo -lLLVMInstrumentation -lLLVMVectorize -lLLVMScalarOpts -lLLVMLinker -lLLVMIRReader -lLLVMAsmParser -lLLVMInstCombine -lLLVMBitWriter -lLLVMAggressiveInstCombine -lLLVMTransformUtils -lLLVMAnalysis -lLLVMProfileData -lLLVMObject -lLLVMMCParser -lLLVMMC -lLLVMDebugInfoCodeView -lLLVMDebugInfoMSF -lLLVMBitReader -lLLVMCore -lLLVMBinaryFormat -lLLVMSupport -lLLVMDemangle")
+# these are all available libs currently, we probably don't need all
+#set(LLVM_LIBS "-lLLVMCFGuard  -lLLVMMipsInfo  -lclangFrontend  -lclangToolingRefactoring  -lLLVMGlobalISel  -lLLVMFileCheck  -lLLVMAnalysis  -lLLVMAArch64AsmParser  -lLLVMBPFAsmParser  -lclangToolingASTDiff  -lLLVMAArch64Utils  -lLLVMMCA  -lclangHandleLLVM  -lLLVMMIRParser  -lLLVMHexagonAsmParser  -lLLVMLanaiDesc  -lLLVMSparcDisassembler  -llldELF  -lLLVMMipsAsmParser  -lLLVMDebugInfoDWARF  -lLLVMAVRDisassembler  -lLLVMWebAssemblyInfo  -lLLVMHexagonInfo  -lLLVMARMCodeGen  -lLLVMPowerPCCodeGen  -lLLVMXRay  -lLLVMMSP430CodeGen  -lLLVMVectorize  -lLLVMHexagonCodeGen  -lclangCrossTU  -lLLVMSupport  -lLLVMDWP  -lLLVMWindowsManifest  -lclangToolingInclusions  -lLLVMNVPTXDesc  -lc++  -lLLVMXCoreCodeGen  -lLLVMSystemZInfo  -lclangFormat  -lLLVMWebAssemblyUtils  -lLLVMCFIVerify  -lLLVMXCoreInfo  -lLLVMAVRDesc  -lLLVMARMDisassembler  -lclangAPINotes  -lLLVMSparcDesc  -llldCOFF  -lLLVMAggressiveInstCombine  -lLLVMAMDGPUDesc  -lLLVMMSP430AsmParser  -lclangIndexSerialization  -lLLVMExtensions  -lLLVMInstCombine  -lLLVMDWARFLinker  -lclangHandleCXX  -lLLVMRISCVDisassembler  -lclangDependencyScanning  -lLLVMRISCVAsmParser  -lLLVMFuzzMutate  -lLLVMExegesisMips  -lclangDynamicASTMatchers  -lclangTransformer  -lclangCodeGen  -lLLVMScalarOpts  -lLLVMTextAPI  -lclangSerialization  -lLLVMPasses  -lLLVMBPFInfo  -lLLVMLinker  -lLLVMObject  -lLLVMMCJIT  -lPollyISL  -lLLVMExegesisPowerPC  -lLLVMCore  -lLLVMLanaiCodeGen  -lLLVMipo  -lLLVMAArch64Info  -lLLVMMC  -lLLVMObjCARCOpts  -lclangRewrite  -lLLVMCodeGen  -lclangAST  -lLLVMTarget  -lLLVMX86AsmParser  -lLLVMFrontendOpenACC  -llldYAML  -lLLVMDebugInfoPDB  -lLLVMMCDisassembler  -lclangEdit  -lLLVMX86CodeGen  -lLLVMX86Info  -lLLVMBPFCodeGen  -lLLVMDebugInfoGSYM  -lLLVMBPFDisassembler  -lclangRewriteFrontend  -lclangLex  -lLLVMRISCVInfo  -lLLVMJITLink  -lLLVMBitReader  -lLLVMHexagonDisassembler  -lLLVMExecutionEngine  -lLLVMMSP430Desc  -lLLVMPowerPCDesc  -llldCommon  -lLLVMWebAssemblyAsmParser  -lclangBasic  -lLLVMSystemZAsmParser  -lLLVMLTO  -lLLVMSystemZCodeGen  -lLLVMRemarks  -lLLVMOption  -lLLVMARMDesc  -lLLVMAMDGPUUtils  -lclangAnalysis  -lLLVMPowerPCAsmParser  -lLLVMOrcTargetProcess  -lLLVMInterfaceStub  -lclangStaticAnalyzerCore  -lclangASTMatchers  -lLLVMIRReader  -llldWasm  -lLLVMFrontendOpenMP  -llldDriver  -lLLVMAArch64Desc  -lclangDirectoryWatcher  -lLLVMAVRCodeGen  -lLLVMWebAssemblyDisassembler  -lLLVMLanaiAsmParser  -lLLVMRISCVCodeGen  -lLLVMBPFDesc  -lLLVMMSP430Disassembler  -lclangDriver  -lLLVMTransformUtils  -lLLVMLibDriver  -lLLVMCoverage  -lLLVMARMInfo  -lLLVMExegesisAArch64  -lclangParse  -lLLVMInterpreter  -lLLVMPowerPCInfo  -lLLVMRuntimeDyld  -lLLVMMSP430Info  -lLLVMProfileData  -lLLVMSymbolize  -lLLVMSelectionDAG  -lLLVMRISCVDesc  -lLLVMExegesisX86  -lLLVMX86Desc  -lLLVMSparcCodeGen  -lLLVMBitWriter  -lclangStaticAnalyzerCheckers  -lLLVMObjectYAML  -lclangTooling  -lLLVMBinaryFormat  -lLLVMMCParser  -lLLVMWebAssemblyDesc  -lLLVMHexagonDesc  -lLLVMAMDGPUCodeGen  -lLLVMX86Disassembler  -lclangToolingCore  -lLLVMDemangle  -lLLVMARMAsmParser  -lLLVMLanaiDisassembler  -lLLVMLineEditor  -lLLVMMipsCodeGen  -lclangTesting  -lLLVMLanaiInfo  -lLLVMOrcJIT  -lclangInterpreter  -llldReaderWriter  -lLLVMAsmPrinter  -lLLVMExegesis  -lLLVMAArch64Disassembler  -lLLVMTableGenGlobalISel  -lclangToolingSyntax  -lLLVMMipsDesc  -lPolly  -lLLVMMCACustomBehaviourAMDGPU  -llldMinGW  -lLLVMXCoreDisassembler  -lLLVMTableGen  -lLLVMAArch64CodeGen  -llldMachO  -lLLVMCoroutines  -llldMachO2  -lLLVMAVRAsmParser  -lLLVMPowerPCDisassembler  -lLLVMDebugInfoCodeView  -lLLVMNVPTXCodeGen  -lclangFrontendTool  -lLLVMAMDGPUInfo  -llldCore  -lLLVMWebAssemblyCodeGen  -lLLVMSystemZDisassembler  -lLLVMSparcInfo  -lLLVMAMDGPUDisassembler  -lLLVMAVRInfo  -lclangARCMigrate  -lLLVMSparcAsmParser  -lLLVMXCoreDesc  -lLLVMAsmParser  -lLLVMDebugInfoMSF  -lLLVMOrcShared  -lLLVMSystemZDesc  -lLLVMNVPTXInfo  -lLLVMBitstreamReader  -lclangStaticAnalyzerFrontend  -lclangIndex  -lLLVMInstrumentation  -lclangSema  -lLLVMDlltoolDriver  -lLLVMARMUtils  -lLLVMMipsDisassembler  -lLLVMAMDGPUAsmParser")
+# these come from the older llvm plus some added ones at the end
+set(LLVM_LIBS "-lLLVMLTO -lLLVMPasses -lLLVMObjCARCOpts -lLLVMMIRParser -lLLVMSymbolize -lLLVMDebugInfoPDB -lLLVMDebugInfoDWARF -lLLVMCoverage -lLLVMMCA -lLLVMTableGen -lLLVMDlltoolDriver -lLLVMXRay -lLLVMOrcJIT -lLLVMXCoreDisassembler -lLLVMXCoreCodeGen -lLLVMXCoreDesc -lLLVMXCoreInfo -lLLVMWebAssemblyDisassembler -lLLVMWebAssemblyCodeGen -lLLVMWebAssemblyDesc -lLLVMWebAssemblyAsmParser -lLLVMWebAssemblyInfo -lLLVMSystemZDisassembler -lLLVMSystemZCodeGen -lLLVMSystemZAsmParser -lLLVMSystemZDesc -lLLVMSystemZInfo -lLLVMSparcDisassembler -lLLVMSparcCodeGen -lLLVMSparcAsmParser -lLLVMSparcDesc -lLLVMSparcInfo -lLLVMPowerPCDisassembler -lLLVMPowerPCCodeGen -lLLVMPowerPCAsmParser -lLLVMPowerPCDesc -lLLVMPowerPCInfo -lLLVMNVPTXCodeGen -lLLVMNVPTXDesc -lLLVMNVPTXInfo -lLLVMMSP430Disassembler -lLLVMMSP430CodeGen -lLLVMMSP430AsmParser -lLLVMMSP430Desc -lLLVMMSP430Info -lLLVMMipsDisassembler -lLLVMMipsCodeGen -lLLVMMipsAsmParser -lLLVMMipsDesc -lLLVMMipsInfo -lLLVMLanaiDisassembler -lLLVMLanaiCodeGen -lLLVMLanaiAsmParser -lLLVMLanaiDesc -lLLVMLanaiInfo -lLLVMHexagonDisassembler -lLLVMHexagonCodeGen -lLLVMHexagonAsmParser -lLLVMHexagonDesc -lLLVMHexagonInfo -lLLVMBPFDisassembler -lLLVMBPFCodeGen -lLLVMBPFAsmParser -lLLVMBPFDesc -lLLVMBPFInfo -lLLVMARMDisassembler -lLLVMARMCodeGen -lLLVMARMAsmParser -lLLVMARMDesc -lLLVMARMInfo -lLLVMARMUtils -lLLVMAMDGPUDisassembler -lLLVMAMDGPUCodeGen -lLLVMAMDGPUAsmParser -lLLVMAMDGPUDesc -lLLVMAMDGPUInfo -lLLVMAMDGPUUtils -lLLVMAArch64Disassembler -lLLVMAArch64CodeGen -lLLVMAArch64AsmParser -lLLVMAArch64Desc -lLLVMAArch64Info -lLLVMAArch64Utils -lLLVMObjectYAML -lLLVMLibDriver -lLLVMOption -lLLVMWindowsManifest -lLLVMTextAPI -lLLVMFuzzMutate -lLLVMX86Disassembler -lLLVMX86AsmParser -lLLVMX86CodeGen -lLLVMGlobalISel -lLLVMSelectionDAG -lLLVMAsmPrinter -lLLVMX86Desc -lLLVMMCDisassembler -lLLVMX86Info -lLLVMMCJIT -lLLVMLineEditor -lLLVMInterpreter -lLLVMExecutionEngine -lLLVMRuntimeDyld -lLLVMCodeGen -lLLVMTarget -lLLVMCoroutines -lLLVMipo -lLLVMInstrumentation -lLLVMVectorize -lLLVMScalarOpts -lLLVMLinker -lLLVMIRReader -lLLVMAsmParser -lLLVMInstCombine -lLLVMBitWriter -lLLVMAggressiveInstCombine -lLLVMTransformUtils -lLLVMAnalysis -lLLVMProfileData -lLLVMObject -lLLVMMCParser -lLLVMMC -lLLVMDebugInfoCodeView -lLLVMDebugInfoMSF -lLLVMBitReader -lLLVMCore -lLLVMBinaryFormat -lLLVMSupport -lLLVMDemangle -lLLVMRemarks -lLLVMFrontendOpenMP -lLLVMBitstreamReader")
+
 set(LLVM_LIBDIR "${LLVM_ROOT}/lib_${METADATA_BINARY_ARCH}")
 set(LLVM_LINKER_FLAGS "-L${LLVM_LIBDIR} -Wl,-search_paths_first -Wl,-headerpad_max_install_names")
 
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target ${METADATA_BINARY_ARCH}-apple-darwin -std=c++11 -fno-rtti ${LLVM_PREPROCESSOR_FLAGS}")
+set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -target ${METADATA_BINARY_ARCH}-apple-darwin -std=c++17 -fno-rtti ${LLVM_PREPROCESSOR_FLAGS}")
 set(LLVM_LINKER_FLAGS "${LLVM_LINKER_FLAGS} ${LLVM_SYSTEM_LIBS} ${LLVM_LIBS}")
 
 set(CMAKE_ARCHIVE_OUTPUT_DIRECTORY ${CMAKE_BINARY_DIR}/bin/lib)

llvm/.gitkeep

@@ -35,33 +35,40 @@ def map_and_list(func, iterable):
 
 
 # process environment variables
-effective_platofrm_name = env("EFFECTIVE_PLATFORM_NAME")
+effective_platform_name = env("EFFECTIVE_PLATFORM_NAME")
 docset_platform = "iOS"
 default_deployment_target_flag_name = "-mios-simulator-version-min"
 default_deployment_target_clang_env_name = "IPHONEOS_DEPLOYMENT_TARGET"
-if effective_platofrm_name is "-macosx":
+if effective_platform_name is "-macosx":
     docset_platform = "OSX"
     default_deployment_target_flag_name = "-mmacosx-version-min"
     default_deployment_target_clang_env_name = "MACOSX_DEPLOYMENT_TARGET"
-elif effective_platofrm_name is "-watchos":
+elif effective_platform_name is "-watchos":
     docset_platform = "watchOS"
     default_deployment_target_flag_name = "-mwatchos-version-min"
     default_deployment_target_clang_env_name = "WATCHOS_DEPLOYMENT_TARGET"
-elif effective_platofrm_name is "-watchsimulator":
+elif effective_platform_name is "-watchsimulator":
     docset_platform = "watchOS"
     default_deployment_target_flag_name = "-mwatchos-simulator-version-min"
     default_deployment_target_clang_env_name = "WATCHOS_DEPLOYMENT_TARGET"
-elif effective_platofrm_name is "-appletvos":
+elif effective_platform_name is "-appletvos":
     docset_platform = "tvOS"
     default_deployment_target_flag_name = "-mappletvos-version-min"
     default_deployment_target_clang_env_name = "APPLETVOS_DEPLOYMENT_TARGET"
-elif effective_platofrm_name is "-appletvsimulator":
+elif effective_platform_name is "-appletvsimulator":
     docset_platform = "tvOS"
     default_deployment_target_flag_name = "-mappletvsimulator-version-min"
     default_deployment_target_clang_env_name = "APPLETVOS_DEPLOYMENT_TARGET"
-elif effective_platofrm_name is "-iphoneos":
+elif effective_platform_name is "-iphoneos":
     default_deployment_target_flag_name = "-miphoneos-version-min"
 
+sdk_version = env("SDK_VERSION") or "13.0"
+llvm_target_triple_suffix = env_or_empty("LLVM_TARGET_TRIPLE_SUFFIX")
+llvm_target_triple_os_version = "ios{}".format(sdk_version)
+# env("LLVM_TARGET_TRIPLE_OS_VERSION") is the deployment target, so doesn't have all APIs
+# usually it's ios9.0 for NativeScript projects
+llvm_target_triple_vendor = env("LLVM_TARGET_TRIPLE_VENDOR") or "apple"
+
 conf_build_dir = env("CONFIGURATION_BUILD_DIR")
 sdk_root = env("SDKROOT")
 src_root = env("SRCROOT")
@@ -136,7 +143,7 @@ def generate_metadata(arch):
     if env_or_empty("IS_UIKITFORMAC").capitalize() is "YES":
       generator_call.extend(["-arch", arch])
     else:
-      generator_call.extend(["-target", "{}-apple-ios13.0-macabi".format(arch)])
+      generator_call.extend(["-target", "{}-{}-{}{}".format(arch, llvm_target_triple_vendor, llvm_target_triple_os_version, llvm_target_triple_suffix)])
 
     generator_call.extend(header_search_paths_parsed)  # HEADER_SEARCH_PATHS
     generator_call.extend(framework_search_paths_parsed)  # FRAMEWORK_SEARCH_PATHS