Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Reorganize CoreCLR native build to reduce CMake reconfigures when the build system is untouched #49906

Merged
merged 21 commits into from
Apr 5, 2021
Merged

Reorganize CoreCLR native build to reduce CMake reconfigures when the build system is untouched #49906

Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
21 commits
Select commit Hold shift + click to select a range
72fe54e
Remove all references to the subset args in the CMakeLists.txt files
jkoritzinsky Mar 16, 2021
21bdc25
Merge commit '4197e651c8a' into HEAD
jkoritzinsky Mar 19, 2021
1d4dfe1
Fixups after merging with upstream clr.iltools work.
jkoritzinsky Mar 19, 2021
35d617f
Make iltools a minimal subset that doesn't build all dependencies.
jkoritzinsky Mar 19, 2021
a6ebd8c
Add missing target to mono build expected in coreclr build.
jkoritzinsky Mar 19, 2021
e42ed30
Fix paltests arg in CI.
jkoritzinsky Mar 19, 2021
6f4166c
Fix DacTableGen missing in linker tests.
jkoritzinsky Mar 19, 2021
ecfa8c5
Fix typo
jkoritzinsky Mar 22, 2021
8d6e369
Explicitly pass config down to install command line. Fixes VS build.
jkoritzinsky Mar 22, 2021
8d198d1
Specify a component for the static lib installations from libraries s…
jkoritzinsky Mar 22, 2021
4b86b76
Move over the test of the CoreCLR targets to use install_clr to ensur…
jkoritzinsky Mar 22, 2021
7acef32
Fix destinations.
jkoritzinsky Mar 22, 2021
b470eb7
Merge branch 'main' of github.com:dotnet/runtime into single-configur…
jkoritzinsky Mar 23, 2021
f4a66a6
Bump minimum CMake version to the recommended version as per @jkotas'…
jkoritzinsky Mar 23, 2021
002dd85
Create the target names from the component names and reduce duplication.
jkoritzinsky Mar 26, 2021
d092098
Fix add_subdirectory ordering to match ordering in main.
jkoritzinsky Mar 26, 2021
cb629ec
Bring back previous processing because batch is annoying and I don't …
jkoritzinsky Mar 27, 2021
3cfb2a7
Fix bash script issue.
jkoritzinsky Mar 27, 2021
81a1f4f
Rename all_jits target to alljits
jkoritzinsky Mar 27, 2021
18bc65f
Merge branch 'main' into single-configure-coreclr-cmake
jkoritzinsky Apr 5, 2021
bc9c13f
Remove duplicate function definitions from bad merge.
jkoritzinsky Apr 5, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 11 additions & 6 deletions eng/Subsets.props
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,7 @@
</PropertyGroup>

<PropertyGroup>
<DefaultCoreClrSubsets>clr.runtime+clr.jit+clr.alljits+linuxdac+clr.corelib+clr.tools+clr.nativecorelib+clr.iltools+clr.packages</DefaultCoreClrSubsets>
<DefaultCoreClrSubsets>clr.native+linuxdac+clr.corelib+clr.tools+clr.nativecorelib+clr.packages</DefaultCoreClrSubsets>

<DefaultMonoSubsets Condition="'$(MonoEnableLLVM)' == 'true' and '$(MonoLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
<DefaultMonoSubsets Condition="'$(MonoAOTEnableLLVM)' == 'true' and '$(MonoAOTLLVMDir)' == ''">mono.llvm+</DefaultMonoSubsets>
Expand Down Expand Up @@ -81,8 +81,9 @@
<SubsetName Include="Clr.DacTools" Description="Managed tools that support building a runtime that supports debugging (such as DacTableGen)." />
<SubsetName Include="Clr.ILTools" Description="The CoreCLR IL tools." />
<SubsetName Include="Clr.Runtime" Description="The CoreCLR .NET runtime." />
<SubsetName Include="Clr.PalTests" Description="The CoreCLR PAL tests." />
<SubsetName Include="Clr.PalTestList" Description="Generate the list of the CoreCLR PAL tests. When using the command line, use Clr.PalTests instead." />
<SubsetName Include="Clr.Native" Description="All CoreCLR native non-test components, including the runtime, jits, and other native tools." />
<SubsetName Include="Clr.PalTests" OnDemand="true" Description="The CoreCLR PAL tests." />
<SubsetName Include="Clr.PalTestList" OnDemand="true" Description="Generate the list of the CoreCLR PAL tests. When using the command line, use Clr.PalTests instead." />
<SubsetName Include="Clr.Jit" Description="The JIT for the CoreCLR .NET runtime." />
<SubsetName Include="Clr.AllJits" Description="All of the cross-targeting JIT compilers for the CoreCLR .NET runtime." />
<SubsetName Include="Clr.CoreLib" Description="The managed System.Private.CoreLib library for CoreCLR." />
Expand Down Expand Up @@ -146,14 +147,18 @@
<ProjectToBuild Include="$(CoreClrProjectRoot)System.Private.CoreLib\System.Private.CoreLib.csproj" Category="clr" />
</ItemGroup>

<ItemGroup Condition="'$(TargetOS)' == 'windows' and ($(_subset.Contains('+clr.dactools+')) or $(_subset.Contains('+clr.runtime+')))">
<ItemGroup Condition="'$(TargetOS)' == 'windows' and ($(_subset.Contains('+clr.dactools+')) or $(_subset.Contains('+clr.runtime+')) or $(_subset.Contains('+clr.native+')))">
<ProjectToBuild Include="$(CoreClrProjectRoot)ToolBox\SOS\DacTableGen\DacTableGen.csproj;
$(CoreClrProjectRoot)ToolBox\SOS\DIALib\DIALib.ilproj"
Category="clr"/>
</ItemGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.runtime+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrRuntimeSubset=true;ClrJitSubset=true</ClrRuntimeBuildSubsets>
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrRuntimeSubset=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.native+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrFullNativeBuild=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.jit+'))">
Expand All @@ -165,7 +170,7 @@
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.alljits+'))">
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrAllJitsSubset=true;ClrJitSubset=true</ClrRuntimeBuildSubsets>
<ClrRuntimeBuildSubsets>$(ClrRuntimeBuildSubsets);ClrAllJitsSubset=true</ClrRuntimeBuildSubsets>
</PropertyGroup>

<PropertyGroup Condition="$(_subset.Contains('+clr.iltools+'))">
Expand Down
30 changes: 20 additions & 10 deletions eng/native/build-commons.sh
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -72,11 +72,12 @@ build_native()
platformArch="$2"
cmakeDir="$3"
intermediatesDir="$4"
cmakeArgs="$5"
message="$6"
target="$5"
cmakeArgs="$6"
message="$7"

# All set to commence the build
echo "Commencing build of \"$message\" for $__TargetOS.$__BuildArch.$__BuildType in $intermediatesDir"
echo "Commencing build of \"$target\" target in \"$message\" for $__TargetOS.$__BuildArch.$__BuildType in $intermediatesDir"

if [[ "$targetOS" == OSX || "$targetOS" == MacCatalyst ]]; then
if [[ "$platformArch" == x64 ]]; then
Expand Down Expand Up @@ -188,20 +189,29 @@ EOF
pushd "$intermediatesDir"

buildTool="$SCAN_BUILD_COMMAND -o $__BinDir/scan-build-log $buildTool"
echo "Executing $buildTool install -j $__NumProc"
"$buildTool" install -j "$__NumProc"
echo "Executing $buildTool $target -j $__NumProc"
"$buildTool" $target -j "$__NumProc"
exit_code="$?"

popd
else
cmake_command=cmake
if [[ "$build_arch" == "wasm" ]]; then
cmake_command="emcmake $cmake_command"
fi
cmake_command="emcmake cmake"
echo "Executing $cmake_command --build \"$intermediatesDir\" --target $target -- -j $__NumProc"
$cmake_command --build "$intermediatesDir" --target $target -- -j "$__NumProc"
exit_code="$?"
else
# For non-wasm Unix scenarios, we may have to use an old version of CMake that doesn't support
# multiple targets. Instead, directly invoke the build tool to build multiple targets in one invocation.
pushd "$intermediatesDir"

echo "Executing $cmake_command --build \"$intermediatesDir\" --target install -- -j $__NumProc"
$cmake_command --build "$intermediatesDir" --target install -- -j "$__NumProc"
exit_code="$?"
echo "Executing $buildTool $target -j $__NumProc"
"$buildTool" $target -j "$__NumProc"
exit_code="$?"

popd
fi
fi

CFLAGS="${SAVED_CFLAGS}"
Expand Down
166 changes: 95 additions & 71 deletions eng/native/functions.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -206,6 +206,27 @@ function(compile_asm)
set(${COMPILE_ASM_OUTPUT_OBJECTS} ${ASSEMBLED_OBJECTS} PARENT_SCOPE)
endfunction()

# add_component(componentName [targetName] [EXCLUDE_FROM_ALL])
function(add_component componentName)
if (${ARGC} GREATER 2 OR ${ARGC} EQUAL 2)
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

lol there's no >= operator?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not in CMake 3.6, which we still need to support for Linux source build.

set(componentTargetName "${ARGV1}")
else()
set(componentTargetName "${componentName}")
endif()
if (${ARGC} EQUAL 3 AND "${ARG2}" STREQUAL "EXCLUDE_FROM_ALL")
set(exclude_from_all_flag "EXCLUDE_FROM_ALL")
endif()
get_property(definedComponents GLOBAL PROPERTY CLR_CMAKE_COMPONENTS)
list (FIND definedComponents "${componentName}" componentIndex)
if (${componentIndex} EQUAL -1)
list (APPEND definedComponents "${componentName}")
add_custom_target("${componentTargetName}"
COMMAND "${CMAKE_COMMAND}" "-DCMAKE_INSTALL_COMPONENT=${componentName}" "-DBUILD_TYPE=$<CONFIG>" -P "${CMAKE_BINARY_DIR}/cmake_install.cmake"
${exclude_from_all_flag})
set_property(GLOBAL PROPERTY CLR_CMAKE_COMPONENTS ${definedComponents})
endif()
endfunction()

function(generate_exports_file)
set(INPUT_LIST ${ARGN})
list(GET INPUT_LIST -1 outputFilename)
Expand Down Expand Up @@ -248,12 +269,29 @@ function(generate_exports_file_prefix inputFilename outputFilename prefix)
PROPERTIES GENERATED TRUE)
endfunction()

function (get_symbol_file_name targetName outputSymbolFilename)
if (CLR_CMAKE_HOST_UNIX)
if (CLR_CMAKE_TARGET_OSX OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS)
set(strip_destination_file $<TARGET_FILE:${targetName}>.dwarf)
else ()
set(strip_destination_file $<TARGET_FILE:${targetName}>.dbg)
endif ()

set(${outputSymbolFilename} ${strip_destination_file} PARENT_SCOPE)
else(CLR_CMAKE_HOST_UNIX)
# We can't use the $<TARGET_PDB_FILE> generator expression here since
# the generator expression isn't supported on resource DLLs.
set(${outputSymbolFilename} $<TARGET_FILE_DIR:${targetName}>/$<TARGET_FILE_PREFIX:${targetName}>$<TARGET_FILE_BASE_NAME:${targetName}>.pdb PARENT_SCOPE)
endif(CLR_CMAKE_HOST_UNIX)
endfunction()

function(strip_symbols targetName outputFilename)
get_symbol_file_name(${targetName} strip_destination_file)
set(${outputFilename} ${strip_destination_file} PARENT_SCOPE)
if (CLR_CMAKE_HOST_UNIX)
set(strip_source_file $<TARGET_FILE:${targetName}>)

if (CLR_CMAKE_TARGET_OSX OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS)
set(strip_destination_file ${strip_source_file}.dwarf)

# Ensure that dsymutil and strip are present
find_program(DSYMUTIL dsymutil)
Expand Down Expand Up @@ -282,7 +320,6 @@ function(strip_symbols targetName outputFilename)
COMMENT "Stripping symbols from ${strip_source_file} into file ${strip_destination_file}"
)
else (CLR_CMAKE_TARGET_OSX OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS)
set(strip_destination_file ${strip_source_file}.dbg)

add_custom_command(
TARGET ${targetName}
Expand All @@ -294,26 +331,13 @@ function(strip_symbols targetName outputFilename)
COMMENT "Stripping symbols from ${strip_source_file} into file ${strip_destination_file}"
)
endif (CLR_CMAKE_TARGET_OSX OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS)

set(${outputFilename} ${strip_destination_file} PARENT_SCOPE)
else(CLR_CMAKE_HOST_UNIX)
get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(is_multi_config)
# We can't use the $<TARGET_PDB_FILE> generator expression here since
# the generator expression isn't supported on resource DLLs.
set(${outputFilename} ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${targetName}.pdb PARENT_SCOPE)
else()
# We can't use the $<TARGET_PDB_FILE> generator expression here since
# the generator expression isn't supported on resource DLLs.
set(${outputFilename} ${CMAKE_CURRENT_BINARY_DIR}/${targetName}.pdb PARENT_SCOPE)
endif()
endif(CLR_CMAKE_HOST_UNIX)
endfunction()

function(install_with_stripped_symbols targetName kind destination)
if(NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS)
strip_symbols(${targetName} symbol_file)
install_symbols(${symbol_file} ${destination})
install_symbol_file(${symbol_file} ${destination} ${ARGN})
endif()

if ((CLR_CMAKE_TARGET_OSX OR CLR_CMAKE_TARGET_MACCATALYST OR CLR_CMAKE_TARGET_IOS OR CLR_CMAKE_TARGET_TVOS) AND ("${kind}" STREQUAL "TARGETS"))
Expand All @@ -328,59 +352,75 @@ function(install_with_stripped_symbols targetName kind destination)
else()
message(FATAL_ERROR "The `kind` argument has to be either TARGETS or PROGRAMS, ${kind} was provided instead")
endif()
install(${kind} ${install_source} DESTINATION ${destination})
install(${kind} ${install_source} DESTINATION ${destination} ${ARGN})
endfunction()

function(install_symbols symbol_file destination_path)
function(install_symbol_file symbol_file destination_path)
if(CLR_CMAKE_TARGET_WIN32)
install(FILES ${symbol_file} DESTINATION ${destination_path}/PDB)
install(FILES ${symbol_file} DESTINATION ${destination_path}/PDB ${ARGN})
else()
install(FILES ${symbol_file} DESTINATION ${destination_path})
install(FILES ${symbol_file} DESTINATION ${destination_path} ${ARGN})
endif()
endfunction()

# install_clr(TARGETS TARGETS targetName [targetName2 ...] [ADDITIONAL_DESTINATIONS destination])
# install_clr(TARGETS targetName [targetName2 ...] [DESTINATIONS destination [destination2 ...]] [COMPONENT componentName])
function(install_clr)
set(multiValueArgs TARGETS ADDITIONAL_DESTINATIONS)
cmake_parse_arguments(INSTALL_CLR "" "" "${multiValueArgs}" ${ARGV})
set(multiValueArgs TARGETS DESTINATIONS)
set(singleValueArgs COMPONENT)
set(options "")
cmake_parse_arguments(INSTALL_CLR "${options}" "${singleValueArgs}" "${multiValueArgs}" ${ARGV})

if ("${INSTALL_CLR_TARGETS}" STREQUAL "")
message(FATAL_ERROR "At least one target must be passed to install_clr(TARGETS )")
endif()

set(destinations ".")
if ("${INSTALL_CLR_DESTINATIONS}" STREQUAL "")
message(FATAL_ERROR "At least one destination must be passed to install_clr.")
endif()

set(destinations "")

if (NOT "${INSTALL_CLR_DESTINATIONS}" STREQUAL "")
list(APPEND destinations ${INSTALL_CLR_DESTINATIONS})
endif()

if (NOT "${INSTALL_CLR_ADDITIONAL_DESTINATIONS}" STREQUAL "")
list(APPEND destinations ${INSTALL_CLR_ADDITIONAL_DESTINATIONS})
if ("${INSTALL_CLR_COMPONENT}" STREQUAL "")
set(INSTALL_CLR_COMPONENT ${CMAKE_INSTALL_DEFAULT_COMPONENT_NAME})
endif()

foreach(targetName ${INSTALL_CLR_TARGETS})
list(FIND CLR_CROSS_COMPONENTS_LIST ${targetName} INDEX)
if (NOT DEFINED CLR_CROSS_COMPONENTS_LIST OR NOT ${INDEX} EQUAL -1)
if (NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS)
strip_symbols(${targetName} symbol_file)
endif()
if (NOT "${INSTALL_CLR_COMPONENT}" STREQUAL "${targetName}")
get_property(definedComponents GLOBAL PROPERTY CLR_CMAKE_COMPONENTS)
list(FIND definedComponents "${INSTALL_CLR_COMPONENT}" componentIdx)
if (${componentIdx} EQUAL -1)
message(FATAL_ERROR "The ${INSTALL_CLR_COMPONENT} component is not defined. Add a call to `add_component(${INSTALL_CLR_COMPONENT})` to define the component in the build.")
endif()
add_dependencies(${INSTALL_CLR_COMPONENT} ${targetName})
endif()
get_target_property(targetType ${targetName} TYPE)
if (NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS AND NOT "${targetType}" STREQUAL "STATIC")
get_symbol_file_name(${targetName} symbol_file)
endif()

foreach(destination ${destinations})
# We don't need to install the export libraries for our DLLs
# since they won't be directly linked against.
install(PROGRAMS $<TARGET_FILE:${targetName}> DESTINATION ${destination})
if (NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS)
install_symbols(${symbol_file} ${destination})
endif()
foreach(destination ${destinations})
# We don't need to install the export libraries for our DLLs
# since they won't be directly linked against.
install(PROGRAMS $<TARGET_FILE:${targetName}> DESTINATION ${destination} COMPONENT ${INSTALL_CLR_COMPONENT})
if (NOT "${symbolFile}" STREQUAL "")
install_symbol_file(${symbol_file} ${destination} COMPONENT ${INSTALL_CLR_COMPONENT})
endif()

if(CLR_CMAKE_PGO_INSTRUMENT)
if(WIN32)
get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(is_multi_config)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${targetName}.pgd DESTINATION ${destination}/PGD OPTIONAL)
else()
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${targetName}.pgd DESTINATION ${destination}/PGD OPTIONAL)
endif()
endif()
if(CLR_CMAKE_PGO_INSTRUMENT)
if(WIN32)
get_property(is_multi_config GLOBAL PROPERTY GENERATOR_IS_MULTI_CONFIG)
if(is_multi_config)
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/$<CONFIG>/${targetName}.pgd DESTINATION ${destination}/PGD OPTIONAL COMPONENT ${INSTALL_CLR_COMPONENT})
else()
install(FILES ${CMAKE_CURRENT_BINARY_DIR}/${targetName}.pgd DESTINATION ${destination}/PGD OPTIONAL COMPONENT ${INSTALL_CLR_COMPONENT})
endif()
endforeach()
endif()
endif()
endif()
endforeach()
endforeach()
endfunction()

Expand Down Expand Up @@ -427,45 +467,29 @@ if (CMAKE_VERSION VERSION_LESS "3.16")
endfunction()
endif()

function(_add_executable)
function(add_executable_clr)
if(NOT WIN32)
add_executable(${ARGV} ${VERSION_FILE_PATH})
disable_pax_mprotect(${ARGV})
else()
add_executable(${ARGV})
endif(NOT WIN32)
list(FIND CLR_CROSS_COMPONENTS_LIST ${ARGV0} INDEX)
if (DEFINED CLR_CROSS_COMPONENTS_LIST AND ${INDEX} EQUAL -1)
set_target_properties(${ARGV0} PROPERTIES EXCLUDE_FROM_ALL 1)
if(NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS)
strip_symbols(${ARGV0} symbolFile)
endif()
endfunction()

function(_add_library)
function(add_library_clr)
if(NOT WIN32 AND "${ARGV1}" STREQUAL "SHARED")
add_library(${ARGV} ${VERSION_FILE_PATH})
else()
add_library(${ARGV})
endif(NOT WIN32 AND "${ARGV1}" STREQUAL "SHARED")
list(FIND CLR_CROSS_COMPONENTS_LIST ${ARGV0} INDEX)
if (DEFINED CLR_CROSS_COMPONENTS_LIST AND ${INDEX} EQUAL -1)
set_target_properties(${ARGV0} PROPERTIES EXCLUDE_FROM_ALL 1)
endif()
endfunction()

function(_install)
if(NOT DEFINED CLR_CROSS_COMPONENTS_BUILD)
install(${ARGV})
if("${ARGV1}" STREQUAL "SHARED" AND NOT CLR_CMAKE_KEEP_NATIVE_SYMBOLS)
strip_symbols(${ARGV0} symbolFile)
endif()
endfunction()

function(add_library_clr)
_add_library(${ARGV})
endfunction()

function(add_executable_clr)
_add_executable(${ARGV})
endfunction()

function(generate_module_index Target ModuleIndexFile)
if(CLR_CMAKE_HOST_WIN32)
set(scriptExt ".cmd")
Expand Down
Loading