External cmake project integration (#193)

Work to make it easier/possible to integrate the C++ components of MLOS with an external C++ project (e.g. LevelDB, MySql, etc.) via `cmake`.

- Fixes some issues with our own `cmake` config assumptions.
- Adds examples patterns and documentation for using in an "external project" (currently just a minimal example).
- Hooks those into the CI pipelines.

This branch was split off of the work from #177 (since merged) to build NuGet packages for some of the C# components, so many of the original commits are shared.

.github/workflows/main.yml

@@ -300,9 +300,9 @@ jobs:
  shell: bash
  run: |
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
- make -C source/Mlos.NetCore.Components.Packages/
+ dotnet build -c ${{ matrix.configuration }} source/Mlos.NetCore.Components.Packages/
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
- dotnet build external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/
+ dotnet build -c ${{ matrix.configuration }} external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/
  - name: Run ${{ matrix.configuration }} cmake build (CXX=${{ matrix.cxx }})
  timeout-minutes: 5
  shell: bash
@@ -312,7 +312,7 @@ jobs:
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
  test -e target/bin/${{ matrix.configuration }}/x86_64/Mlos.UnitTest
  - name: Run ${{ matrix.configuration }} cmake test
- timeout-minutes: 5
+ timeout-minutes: 10
  shell: bash
  run: |
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
@@ -323,6 +323,12 @@ jobs:
  run: |
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
  ./build.linux.sh --Configuration=${{ matrix.configuration }}
+ - name: Run ${{ matrix.configuration }} external cmake integration build/test
+ timeout-minutes: 5
+ shell: bash
+ run: |
+ docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
+ make -C external/ExternalIntegrationExample all test
  - name: Cleanup ${{ matrix.configuration }} docker instance for Ubuntu ${{ matrix.UbuntuVersion }}
  shell: bash
  run: |

.gitignore

@@ -431,3 +431,4 @@ website/python_api
 
 # python code coverage
 coverage
+.coverage.*

Dockerfile

@@ -93,13 +93,15 @@ RUN mkdir -p \
  /src/MLOS/out \
  /src/MLOS/target \
  /src/MLOS/temp \
- /src/MLOS/tools && \
+ /src/MLOS/tools \
+ /src/MLOS/external/ExternalIntegrationExample/build && \
  chgrp -R src /src/MLOS && \
  chmod 0775 \
  /src/MLOS/out \
  /src/MLOS/target \
  /src/MLOS/temp \
- /src/MLOS/tools
+ /src/MLOS/tools \
+ /src/MLOS/external/ExternalIntegrationExample/build
 
 # Declare a volume that we can bind mount the current MLOS repo into in-place
 # instead of the default copy.
@@ -113,6 +115,8 @@ VOLUME /src/MLOS/target
 VOLUME /src/MLOS/tools
 VOLUME /src/MLOS/temp
 
+VOLUME /src/MLOS/external/ExternalIntegrationExample/build
+
 WORKDIR /src/MLOS
 
 # Create directory for our scripts to go.

Makefile

@@ -15,6 +15,7 @@ handledtargets += cmake-build cmake-install cmake-test cmake-check \
  python-checks python-test python-clean \
  website website-clean \
  grpc-clean mlos-codegen-clean \
+ external-integration-test external-integration-test-clean external-integration-test-distclean \
  docker-image
 
 # Build using dotnet and the Makefile produced by cmake.
@@ -23,7 +24,7 @@ all: dotnet-build cmake-build python-checks
  @ echo "make all target finished."
 
 .PHONY: test
-test: dotnet-test cmake-test python-test
+test: dotnet-test cmake-test python-test #external-integration-test
  @ echo "make test target finished."
 
 .PHONY: check
@@ -38,10 +39,10 @@ pack:
  $(MAKE) -C source/Mlos.NetCore.Components.Packages
 
 .PHONY: clean
-clean: cmake-clean dotnet-clean grpc-clean mlos-codegen-clean website-clean python-clean
+clean: cmake-clean dotnet-clean grpc-clean mlos-codegen-clean website-clean python-clean external-integration-test-clean
 
 .PHONY: distclean
-distclean: clean dotnet-distclean dotnet-pkgs-clean cmake-distclean
+distclean: clean dotnet-distclean dotnet-pkgs-clean cmake-distclean external-integration-test-distclean
 
 .PHONY: rebuild
 rebuild: clean all
@@ -134,6 +135,18 @@ python-test:
  @ ./scripts/run-python-tests.sh
  @ echo "make python-test target finished."
 
+.PHONY: external-integration-test
+external-integration-test:
+ @ $(MAKE) -C external/ExternalIntegrationExample all test
+
+.PHONY: external-integration-test-clean
+external-integration-test-clean:
+ @ $(MAKE) -C external/ExternalIntegrationExample clean
+
+.PHONY: external-integration-test-distclean
+external-integration-test-distclean:
+ @ $(MAKE) -C external/ExternalIntegrationExample distclean
+
 # Don't force cmake regen every time we run ctags - only if it doesn't exist
 .PHONY: ctags
 ctags:

build/CMakeHelpers/ParseCsProjForCsFiles.sh

@@ -7,7 +7,7 @@
 # CMakeLists.txt wrappers around .csproj files.
 
 cat *.csproj 2>/dev/null \
- | egrep '<(Compile|SettingsRegistryDef) Include=".*\.cs"' \
+ | egrep '<(Compile|SettingsRegistryDef|MlosSettingsRegistryDefinition) Include=".*\.cs"' \
  | sed -r -e 's/.*Include="([^"]+\.cs)".*/\1/' -e 's|\\|/|g' \
  | tr '\n' ';' | sed 's/;$//'
 

build/CMakeHelpers/ParseCsProjForCsProjs.sh

@@ -6,7 +6,7 @@
 # We use this to help dynamically build up the set of dependencies for our
 # CMakeLists.txt wrappers around .csproj files.
 
-cat *.csproj 2>/dev/null \
+cat *.csproj *.proj 2>/dev/null \
  | grep '<ProjectReference Include=".*\.csproj"' \
  | sed -r -e 's|.*[/\\)"]([^"]+)\.csproj".*|\1|' \
  | tr '\n' ';' | sed 's/;$//'

build/Mlos.Common.cmake

@@ -18,34 +18,51 @@ if(NOT DEFINED MLOS_ROOT)
  "CMakeLists.txt error: MLOS_ROOT is not defined.")
 endif()
 
+# Set the possible values of build type for cmake-gui
+# Note: these options need to match the ones we support in the dotnet build.
+set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
+ "Debug" "Release") # "MinSizeRel" "RelWithDebInfo")
 # Set a default build type if none was specified.
 # (used in the codegen output determination)
 set(default_build_type "Release")
-if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES)
+if(NOT CMAKE_BUILD_TYPE) # AND NOT CMAKE_CONFIGURATION_TYPES)
  message(STATUS "Setting build type to '${default_build_type}' since none was specified.")
  set(CMAKE_BUILD_TYPE "${default_build_type}" CACHE
  STRING "Choose the type of build." FORCE)
- # Set the possible values of build type for cmake-gui
- set_property(CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
- "Debug" "Release") # "MinSizeRel" "RelWithDebInfo")
+endif()
+# Allow an easy way of overwritting the build type when MLOS is used with FetchContent.
+if(DEFINED MLOS_CMAKE_BUILD_TYPE)
+ set(CMAKE_BUILD_TYPE "${MLOS_CMAKE_BUILD_TYPE}")
+endif()
+# Check to make sure we're using an appropriate build type.
+if(NOT ((${CMAKE_BUILD_TYPE} STREQUAL "Release") OR (${CMAKE_BUILD_TYPE} STREQUAL "Debug")))
+ message(FATAL_ERROR
+ "Unsupported CMAKE_BUILD_TYPE: ${CMAKE_BUILD_TYPE}")
 endif()
 
-if(NOT (${CMAKE_SOURCE_DIR} STREQUAL ${MLOS_ROOT}))
+# When MLOS is included in another project using FetchContent, then the CMAKE_SOURCE_DIR is from the parent.
+# In that case, we expect the MLOS_ROOT to be nested under that.
+# If we're just building MLOS on its own, then we expect the variables to be the same,
+# or at least CMAKE_SOURCE_DIR not to be nested under MLOS_ROOT for the case of
+# attempting to build one of its subcomponents standalone).
+string(FIND "${MLOS_ROOT}" "${CMAKE_SOURCE_DIR}" CMAKE_SOURCE_DIR_IndexIn_MLOS_ROOT)
+if(NOT(CMAKE_SOURCE_DIR_IndexIn_MLOS_ROOT EQUAL 0))
  message(FATAL_ERROR
- "Please run 'cmake' from '${MLOS_ROOT}'.")
+ "CMAKE_SOURCE_DIR ('${CMAKE_SOURCE_DIR}') is not contained in or equal to MLOS_ROOT.\n"
+ "Please run 'cmake' from MLOS_ROOT ('${MLOS_ROOT}').")
 endif()
 
 # Prevent in-source builds as well as the default build/ directory
 # - it conflicts with our MSBuild config location.
-if((${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) OR (${CMAKE_BINARY_DIR} STREQUAL "build"))
+if((${CMAKE_SOURCE_DIR} STREQUAL ${CMAKE_BINARY_DIR}) OR (${CMAKE_BINARY_DIR} STREQUAL "${MLOS_ROOT}/build"))
  message(FATAL_ERROR
  "In-source builds not allowed. Please run\n"
  "# make\n"
  "or\n"
  "# rm -f CMakeCache.txt && cmake -DCMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE} -S ${MLOS_ROOT} -B ${MLOS_ROOT}/out/cmake/${CMAKE_BUILD_TYPE}\n"
  "to place CMake build outputs in the out/cmake/${CMAKE_BUILD_TYPE}/ directory.\n")
 endif()
-set(CMAKE_BINARY_DIR "${MLOS_ROOT}/out/cmake/${CMAKE_BUILD_TYPE}")
+#set(CMAKE_BINARY_DIR "${MLOS_ROOT}/out/cmake/${CMAKE_BUILD_TYPE}")
 
 # Set a binplace dir to match the msbuild rules.
 # We will use this in our install() definition rules.

build/Mlos.Cpp.UnitTest.cmake

@@ -85,6 +85,7 @@ function(add_mlos_agent_server_exe_test_run)
  # This test conflicts with any other test using the MlosSharedMemories
  # (no parallel test runs for the moment).
  RESOURCE_LOCK MlosSharedMemories)
+
  if(${WITH_OPTIMIZER})
  # Add some test fixtures to use for setup/tear down of other Mlos unit tests.
  #

build/Mlos.NetCore.cmake

@@ -114,11 +114,10 @@ function(add_mlos_dotnet_project)
  # we compose the dependency graph already above, so we can skip
  # building project references here in order to avoid some parallel
  # dotnet processes accessing the same files.
- #
+ COMMAND ${DOTNET} build -m --configuration ${CMAKE_BUILD_TYPE} /p:BuildProjectReferences=false "${CSPROJ}"
  # Also, "dotnet build" doesn't update timestamps in a make compatible
  # way, so we also mark the projects as having been built using touch.
- COMMAND ${DOTNET} build -m --configuration ${CMAKE_BUILD_TYPE} /p:BuildProjectReferences=false "${CSPROJ}" &&
- touch -c "${CSPROJ}" "${OUTPUT_DLL}" "${BINPLACE_DLL}"
+ COMMAND ${CMAKE_COMMAND} -E touch_nocreate "${CSPROJ}" "${OUTPUT_DLL}" "${BINPLACE_DLL}"
  DEPENDS "${DEPENDENCIES}"
  WORKING_DIRECTORY "${DIRECTORY}"
  COMMENT "Building dotnet assembly ${NAME}.dll")

documentation/RepoOrganization.md

@@ -27,6 +27,13 @@ Some notes on the directory layout organization in this repo.
 - [`test/`](../test/#mlos-github-tree-view) contains a directory and project to invoke each of the unit tests.
  - i.e. running `msbuild` or `make` in the `test/` directory will also run all of the tests.
 - [`scripts/`](../scripts/#mlos-github-tree-view) contains some helper scripts to initialize development environments, install tools, invoke build pipelines, run tests, etc.
+- [`external/`](../external/#mlos-github-tree-view) contains code, config, and examples for build integration for use of MLOS with external projects.
+
+ For instance:
+
+ - [`external/cmake/`](../external/cmake/) contains an `MLOS` module for use in projects using the `cmake` build system.
+ - [`ExternalIntegrationExample`](../external/ExternalIntegrationExample/) contains a stripped down example showing how to use it.
+ - [`leveldb`](../external/leveldb/) contains examples of how to use portions of MLOS to tune [LevelDb](https://github.com/google/leveldb).
 
 Auto generated content:
 

external/ExternalIntegrationExample/.gitignore

@@ -1,3 +1,4 @@
 # Ignore the local MLOS codegen output for this example:
 ./Mlos.CodeGen.out/
 .nuget/
+build/

external/ExternalIntegrationExample/.vscode/launch.json

@@ -0,0 +1,23 @@
+{
+ // Use IntelliSense to learn about possible attributes.
+ // Hover to view descriptions of existing attributes.
+ // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+ "version": "0.2.0",
+ "configurations": [
+ {
+ "name": "(gdb) Attach - ExternalIntegrationExample",
+ "type": "cppdbg",
+ "request": "launch",
+ "program": "${workspaceFolder}/build/ExternalIntegrationExample",
+ "cwd": "${workspaceFolder}/build",
+ "MIMode": "gdb",
+ "setupCommands": [
+ {
+ "description": "Enable pretty-printing for gdb",
+ "text": "-enable-pretty-printing",
+ "ignoreFailures": true
+ }
+ ]
+ }
+ ]
+}

external/ExternalIntegrationExample/CMakeLists.txt

@@ -0,0 +1,29 @@
+cmake_minimum_required(VERSION 3.15)
+
+# Here we assume some existing external build system setup, using cmake:
+
+project(ExternalIntegrationExample)
+
+# Set some general compile options.
+include(./cmake/Common.cmake)
+
+# This is a simple existing project definition.
+#
+add_executable(${PROJECT_NAME}
+ ExternalIntegrationExample.cpp
+)
+
+# Now, we start adding MLOS integrations.
+
+# First, include some cross component/target cmake rules for
+# fetching the MLOS source code and registering its cmake targets.
+#
+include(./cmake/MlosBuildIntegrations.cmake)
+
+target_link_libraries(${PROJECT_NAME} Mlos.Core)
+
+# SettingsRegistry projects produce C++ codegen artifacts, that this project
+# consumes, so we mark that project as a dependency.
+#
+add_subdirectory(ExternalIntegrationExample.SettingsRegistry)
+add_dependencies(${PROJECT_NAME} ExternalIntegrationExample.SettingsRegistry)

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/.gitignore

@@ -0,0 +1,3 @@
+!build/
+bin/
+obj/

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt

@@ -0,0 +1,25 @@
+# This is a sub component of the ExternalIntegrationExample.cpp
+# As such, some additional setup work to fetch and prepare the MLOS upstream
+# source repo for use is expected to have been done already.
+# See Also: MlosBuildIntegrations.cmake
+
+include(MLOS)
+
+# Use an MLOS cmake module provided function to create a small wrapper target
+# around the dotnet build for the SettingsRegistry csproj file.
+#
+add_mlos_settings_registry(
+ # The name of the .csproj for this MLOS SettingsRegistry:
+ NAME ExternalIntegrationExample.SettingsRegistry
+ # Where to find it:
+ DIRECTORY "${CMAKE_CURRENT_LIST_DIR}"
+ # Optionally override some settings in the .csproj file.
+ #
+ # This is equivalent to "dotnet build /p:MlosSettingsSystemCodeGenOutputDirectory=/basepath-to-all-codegen-output/specific-component"
+ CODEGEN_OUTPUT_DIR "${MlosCodeGenBaseOutDir}/ExternalIntegrationExample"
+ # This is equivalent to "dotnet build /p:MlosSettingsRegistryAssemblyOutputDirectory=/path-to-all-settings-registry-dll-outputs"
+ BINPLACE_DIR "${MlosSettingsRegistryDllDir}"
+ # Optionally build, pack, and restore the nugets from the local Mlos repo checkout.
+ # (since we aren't yet publishing these nugets, this is currently required)
+ USE_LOCAL_MLOS_NUGETS
+)

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/CodeGen/SmartComponentExampleConfig.cs

@@ -32,6 +32,6 @@ internal partial struct SmartComponentExampleConfig
  /// The size for the smart component example.
  /// </summary>
  [ScalarSetting]
- internal int Size;
+ internal long Size;
  }
 }

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/Makefile

@@ -1,17 +1,34 @@
+# This is a simple Makefile used mostly for local interactive development.
+
 CONFIGURATION ?= Release
 
 all: dotnet-build
 
 clean: dotnet-clean
+ rm -rf ../.nuget/packages/mlos.* || true
+ rm -rf ../Mlos.CodeGen.out/ExternalIntegrationExample
+
+distclean: clean
+ rm -rf ./bin/
+ rm -rf ./obj/
+ rm -rf ../.nuget/
+ rm -rf ../Mlos.CodeGen.out/
 
 dotnet-build:
  dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) /p:PackMlosDependencies=true
 
 dotnet-build-quick:
  dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore
 
+# A convenience helper for local package development:
 dotnet-rebuild-all:
  dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) /p:PackMlosDependencies=true /p:CleanMlosPackages=true
 
 dotnet-clean:
- dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore /t:clean
+ dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore /t:clean 2>/dev/null >/dev/null || true
+
+# A quick wrapper for invoking testing the CMakeLists.txt wrapper.
+cmake-build:
+ mkdir -p ../build
+ cd ../build/ && cmake ..
+ cd ../build/ && make -j ExternalIntegrationExample.SettingsRegistry

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/README.md

@@ -0,0 +1,3 @@
+# ExternalIntegrationExample.SettingsRegistry
+
+[This directory](./#mlos-github-tree-view) contains the *SettingsRegistry* definitions and build configuration files for the [`ExternalIntegrationExample`](../) project.

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/Common.props

@@ -14,7 +14,7 @@
  Note: This should match the cmake include_directory() specficiation so that the
  output SettingsProvider_gen_base.h and related files can be found.
  -->
- <MlosSettingsSystemCodeGenOutputBaseDir>$(MSBuildProjectDirectory)/../Mlos.CodeGen.out/$(Configuration)</MlosSettingsSystemCodeGenOutputBaseDir>
+ <MlosSettingsSystemCodeGenOutputBaseDir>$(MSBuildProjectDirectory)/../Mlos.CodeGen.out/</MlosSettingsSystemCodeGenOutputBaseDir>
 
  <!--
  Specify the version of the Mlos.SettingsSystem.* nuget packages to use.

external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/ExternalIntegrationExample.Stylecop.ruleset

@@ -12,5 +12,3 @@
  <!--Rule Id="CS1591" Action="Hidden" /-->
  </Rules>
 </RuleSet>
-
-