diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index ab1ecdad46..09667476b8 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -300,9 +300,9 @@ jobs: shell: bash run: | docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ - make -C source/Mlos.NetCore.Components.Packages/ + dotnet build -c ${{ matrix.configuration }} source/Mlos.NetCore.Components.Packages/ docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ - dotnet build external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/ + dotnet build -c ${{ matrix.configuration }} external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/ - name: Run ${{ matrix.configuration }} cmake build (CXX=${{ matrix.cxx }}) timeout-minutes: 5 shell: bash @@ -312,7 +312,7 @@ jobs: docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ test -e target/bin/${{ matrix.configuration }}/x86_64/Mlos.UnitTest - name: Run ${{ matrix.configuration }} cmake test - timeout-minutes: 5 + timeout-minutes: 10 shell: bash run: | docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ @@ -323,6 +323,12 @@ jobs: run: | docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ ./build.linux.sh --Configuration=${{ matrix.configuration }} + - name: Run ${{ matrix.configuration }} external cmake integration build/test + timeout-minutes: 5 + shell: bash + run: | + docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \ + make -C external/ExternalIntegrationExample all test - name: Cleanup ${{ matrix.configuration }} docker instance for Ubuntu ${{ matrix.UbuntuVersion }} shell: bash run: | diff --git a/.gitignore b/.gitignore index cbb47d5941..73bd7d2b38 100644 --- a/.gitignore +++ b/.gitignore @@ -431,3 +431,4 @@ website/python_api # python code coverage coverage +.coverage.* diff --git a/Dockerfile b/Dockerfile index 059b02c06f..b59e2db004 100644 --- a/Dockerfile +++ b/Dockerfile @@ -93,13 +93,15 @@ RUN mkdir -p \ /src/MLOS/out \ /src/MLOS/target \ /src/MLOS/temp \ - /src/MLOS/tools && \ + /src/MLOS/tools \ + /src/MLOS/external/ExternalIntegrationExample/build && \ chgrp -R src /src/MLOS && \ chmod 0775 \ /src/MLOS/out \ /src/MLOS/target \ /src/MLOS/temp \ - /src/MLOS/tools + /src/MLOS/tools \ + /src/MLOS/external/ExternalIntegrationExample/build # Declare a volume that we can bind mount the current MLOS repo into in-place # instead of the default copy. @@ -113,6 +115,8 @@ VOLUME /src/MLOS/target VOLUME /src/MLOS/tools VOLUME /src/MLOS/temp +VOLUME /src/MLOS/external/ExternalIntegrationExample/build + WORKDIR /src/MLOS # Create directory for our scripts to go. diff --git a/Makefile b/Makefile index 8be9bb7874..9e7c1931d0 100644 --- a/Makefile +++ b/Makefile @@ -15,6 +15,7 @@ handledtargets += cmake-build cmake-install cmake-test cmake-check \ python-checks python-test python-clean \ website website-clean \ grpc-clean mlos-codegen-clean \ + external-integration-test external-integration-test-clean external-integration-test-distclean \ docker-image # Build using dotnet and the Makefile produced by cmake. @@ -23,7 +24,7 @@ all: dotnet-build cmake-build python-checks @ echo "make all target finished." .PHONY: test -test: dotnet-test cmake-test python-test +test: dotnet-test cmake-test python-test #external-integration-test @ echo "make test target finished." .PHONY: check @@ -38,10 +39,10 @@ pack: $(MAKE) -C source/Mlos.NetCore.Components.Packages .PHONY: clean -clean: cmake-clean dotnet-clean grpc-clean mlos-codegen-clean website-clean python-clean +clean: cmake-clean dotnet-clean grpc-clean mlos-codegen-clean website-clean python-clean external-integration-test-clean .PHONY: distclean -distclean: clean dotnet-distclean dotnet-pkgs-clean cmake-distclean +distclean: clean dotnet-distclean dotnet-pkgs-clean cmake-distclean external-integration-test-distclean .PHONY: rebuild rebuild: clean all @@ -134,6 +135,18 @@ python-test: @ ./scripts/run-python-tests.sh @ echo "make python-test target finished." +.PHONY: external-integration-test +external-integration-test: + @ $(MAKE) -C external/ExternalIntegrationExample all test + +.PHONY: external-integration-test-clean +external-integration-test-clean: + @ $(MAKE) -C external/ExternalIntegrationExample clean + +.PHONY: external-integration-test-distclean +external-integration-test-distclean: + @ $(MAKE) -C external/ExternalIntegrationExample distclean + # Don't force cmake regen every time we run ctags - only if it doesn't exist .PHONY: ctags ctags: diff --git a/build/CMakeHelpers/ParseCsProjForCsFiles.sh b/build/CMakeHelpers/ParseCsProjForCsFiles.sh index 6b6117c553..08c49297dc 100755 --- a/build/CMakeHelpers/ParseCsProjForCsFiles.sh +++ b/build/CMakeHelpers/ParseCsProjForCsFiles.sh @@ -7,7 +7,7 @@ # CMakeLists.txt wrappers around .csproj files. cat *.csproj 2>/dev/null \ - | egrep '<(Compile|SettingsRegistryDef) Include=".*\.cs"' \ + | egrep '<(Compile|SettingsRegistryDef|MlosSettingsRegistryDefinition) Include=".*\.cs"' \ | sed -r -e 's/.*Include="([^"]+\.cs)".*/\1/' -e 's|\\|/|g' \ | tr '\n' ';' | sed 's/;$//' diff --git a/build/CMakeHelpers/ParseCsProjForCsProjs.sh b/build/CMakeHelpers/ParseCsProjForCsProjs.sh index 72f338c337..e978b4bbf7 100755 --- a/build/CMakeHelpers/ParseCsProjForCsProjs.sh +++ b/build/CMakeHelpers/ParseCsProjForCsProjs.sh @@ -6,7 +6,7 @@ # We use this to help dynamically build up the set of dependencies for our # CMakeLists.txt wrappers around .csproj files. -cat *.csproj 2>/dev/null \ +cat *.csproj *.proj 2>/dev/null \ | grep ' [ScalarSetting] - internal int Size; + internal long Size; } } diff --git a/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/Makefile b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/Makefile index 9ad323ab16..c082d607b9 100644 --- a/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/Makefile +++ b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/Makefile @@ -1,8 +1,18 @@ +# This is a simple Makefile used mostly for local interactive development. + CONFIGURATION ?= Release all: dotnet-build clean: dotnet-clean + rm -rf ../.nuget/packages/mlos.* || true + rm -rf ../Mlos.CodeGen.out/ExternalIntegrationExample + +distclean: clean + rm -rf ./bin/ + rm -rf ./obj/ + rm -rf ../.nuget/ + rm -rf ../Mlos.CodeGen.out/ dotnet-build: dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) /p:PackMlosDependencies=true @@ -10,8 +20,15 @@ dotnet-build: dotnet-build-quick: dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore +# A convenience helper for local package development: dotnet-rebuild-all: dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) /p:PackMlosDependencies=true /p:CleanMlosPackages=true dotnet-clean: - dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore /t:clean + dotnet build $(MSBUILD_ARGS) -c $(CONFIGURATION) --no-restore /t:clean 2>/dev/null >/dev/null || true + +# A quick wrapper for invoking testing the CMakeLists.txt wrapper. +cmake-build: + mkdir -p ../build + cd ../build/ && cmake .. + cd ../build/ && make -j ExternalIntegrationExample.SettingsRegistry diff --git a/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/README.md b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/README.md new file mode 100644 index 0000000000..5fda466346 --- /dev/null +++ b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/README.md @@ -0,0 +1,3 @@ +# ExternalIntegrationExample.SettingsRegistry + +[This directory](./#mlos-github-tree-view) contains the *SettingsRegistry* definitions and build configuration files for the [`ExternalIntegrationExample`](../) project. diff --git a/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/Common.props b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/Common.props index 048e97954a..caae486b38 100644 --- a/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/Common.props +++ b/external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/build/Common.props @@ -14,7 +14,7 @@ Note: This should match the cmake include_directory() specficiation so that the output SettingsProvider_gen_base.h and related files can be found. --> - $(MSBuildProjectDirectory)/../Mlos.CodeGen.out/$(Configuration) + $(MSBuildProjectDirectory)/../Mlos.CodeGen.out/ - - diff --git a/external/ExternalIntegrationExample/ExternalIntegrationExample.cpp b/external/ExternalIntegrationExample/ExternalIntegrationExample.cpp new file mode 100644 index 0000000000..e3504d29a3 --- /dev/null +++ b/external/ExternalIntegrationExample/ExternalIntegrationExample.cpp @@ -0,0 +1,121 @@ +//********************************************************************* +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root +// for license information. +// +// @File: ExternalIntegrationExample.cpp +// +// Purpose: +// This is a simple external C++ project main function file. +// +// Notes: +// +// +//********************************************************************* + +#include "include/Common.h" + +// Include platform specific implementations of some MLOS functions. +// Only needed in one compilation unit for a given target. +// +#include "MlosPlatform.Std.inl" + +void ThrowIfFail(HRESULT hr) +{ + if (FAILED(hr)) + { + throw std::exception(); + } +} + +int main() +{ + // Create the MlosContext. + // + DefaultMlosContextFactory mlosContextFactory; + HRESULT hr = mlosContextFactory.Create(); + ThrowIfFail(hr); + + Mlos::Core::MlosContext& mlosContext = mlosContextFactory.m_context; + + // Create a feedback channel receiver thread. + // + ISharedChannel& feedbackChannel = mlosContext.FeedbackChannel(); + + // This background thread uses a lambda to monitor the feedback channel for + // new messages and process them using the callbacks registered for each + // message type in the global dispatch table. + // + std::future feedbackChannelReader = std::async( + std::launch::async, + [&feedbackChannel] + { + // GlobalDispatchTable defines the set of recognized messages by this + // application. + // See GlobalDispatchTable.h for details. + // + auto globalDispatchTable = GlobalDispatchTable(); + + // This starts a loop to handle reading messages from the feedback + // channel, looking them up in the dispatch table, and calling the + // callbacks associated with them. + // + feedbackChannel.ProcessMessages(globalDispatchTable.data(), globalDispatchTable.size()); + + return true; + }); + + // Register the SettingsRegistry assembly with the external agent. + // + hr = mlosContext.RegisterSettingsAssembly( + "ExternalIntegrationExample.SettingsRegistry.dll", + ExternalIntegrationExample::ObjectDeserializationHandler::DispatchTableBaseIndex()); + ThrowIfFail(hr); + + // Create a component configuration object. + // This will be stored in a shared memory region below for use by both the + // component and the external agent. + // + Mlos::Core::ComponentConfig config(mlosContext); + + // Initialize the config with default values. + // + config.ActiveConfigId = 1; + config.NewConfigId = 1; + config.Size = 100; + + // Checks to see if there's already a shared memory region for storing the + // config for this component and if not creates it. + // + hr = mlosContext.RegisterComponentConfig(config); + ThrowIfFail(hr); + + // Pretend we did something with the component. + + // Place an example message onto the shared memory ring buffer. + // + ExternalIntegrationExample::SmartComponentExampleTelemetryMessage msg = { }; + msg.RequestKey = 42; + msg.RequestType = ExternalIntegrationExample::ComponentRequestType::Put; + msg.RequestSize = sizeof(msg.RequestKey); + msg.RequestDuration = 7.0; + msg.ResponseType = ExternalIntegrationExample::ComponentResponseType::Success; + msg.Size = config.Size; + + mlosContext.SendTelemetryMessage(msg); + + // For now we aren't bothering to try and receive any message from an agent. + + std::cout << "Hello World!" << std::endl; + + // Terminate the feedback channel. + // + mlosContext.TerminateFeedbackChannel(); + + // At this point there are no active feedback channel reader threads. + // Now, terminate the control channel. + // + mlosContext.TerminateControlChannel(); + + return 0; +} diff --git a/external/ExternalIntegrationExample/Makefile b/external/ExternalIntegrationExample/Makefile new file mode 100644 index 0000000000..91dff9ddab --- /dev/null +++ b/external/ExternalIntegrationExample/Makefile @@ -0,0 +1,22 @@ +# A very simple make file to separate the rest of the source tree of our example from its build outputs. + +CONFIGURATION ?= Release + +all: cmake-build + +cmake-build: + mkdir -p ./build + cd ./build/ && cmake .. + cd ./build/ && make -j + +test: cmake-build + # Run the test program. + ./build/ExternalIntegrationExample + +clean: + make -C ExternalIntegrationExample.SettingsRegistry/ clean + cd ./build/ 2>/dev/null && make clean 2>/dev/null || true + +distclean: clean + make -C ExternalIntegrationExample.SettingsRegistry/ distclean + rm -rf ./build/ diff --git a/external/ExternalIntegrationExample/README.md b/external/ExternalIntegrationExample/README.md index 07c22aee31..089836d060 100644 --- a/external/ExternalIntegrationExample/README.md +++ b/external/ExternalIntegrationExample/README.md @@ -1,8 +1,9 @@ # ExternalIntegrationExample -This directory serves as a minimal example for testing integration of MLOS with an external C++ example. +[This directory](./#mlos-github-tree-view) serves as a minimal example for testing integration of MLOS with an external C++ example that uses `cmake` for its build system. -It generally does not depend on anything in the rest of the MLOS repo other than the header files and C# NuGet packages that its build produces. +It generally does not depend on anything in the rest of the MLOS repo other than the `Mlos.Core` header files (both static and codegen outputs) and C# NuGet packages that the main MLOS build can produce. -TODO: Extend this to work for a real example (e.g. LevelDB, MySQL, etc.) +## See Also +- [`ExternalIntegrationExample.SettingsRegistry`](./ExternalIntegrationExample.SettingsRegistry/) diff --git a/external/ExternalIntegrationExample/cmake/Common.cmake b/external/ExternalIntegrationExample/cmake/Common.cmake new file mode 100644 index 0000000000..51732ba9f3 --- /dev/null +++ b/external/ExternalIntegrationExample/cmake/Common.cmake @@ -0,0 +1,12 @@ +# This file would contain whatever common build definitions the external project needed. + +set(CMAKE_CXX_STANDARD 14) + +# In our example, we want to enforce very strict compiler warnings to make sure +# the MLOS code we include will work easily in most places. +add_compile_options(-Wall -Wextra -Wpedantic -Werror) +add_link_options(-Wall -Wextra -Wpedantic -Werror) + +# Include debug symbols. +add_compile_options(-g) +add_link_options(-g) diff --git a/external/ExternalIntegrationExample/cmake/MlosBuildIntegrations.cmake b/external/ExternalIntegrationExample/cmake/MlosBuildIntegrations.cmake new file mode 100644 index 0000000000..cfa7972454 --- /dev/null +++ b/external/ExternalIntegrationExample/cmake/MlosBuildIntegrations.cmake @@ -0,0 +1,95 @@ +# In this file we add our extensions to the existing build definitions for this example external project. + +# First, start by making sure that MLOS is available for this project. +# +# Since it is not available in packaged form yet, we simply grab the sources for now. + +# Normally we'd fetch directly from the upstream repo, +#set(MLOS_GIT_URL "https://github.com/microsoft/MLOS.git") +# and would specify a particular stable version. +#set(MLOS_GIT_TAG v0.1.2) + +# However for local (and CI pipeline) testing, we just reference the current checkout. +set(MLOS_GIT_URL "file://${CMAKE_CURRENT_LIST_DIR}/../../..") +set(MLOS_GIT_TAG "HEAD") + +# Instruct cmake how to find the Mlos source code. +include(FetchContent) +#set(FETCHCONTENT_QUIET OFF) +FetchContent_Declare( + # The name of the dependency (used to form variable names later on). By convention listed in lowercase. + mlos + # Where to fetch it from. + GIT_REPOSITORY "${MLOS_GIT_URL}" + # and which version. + GIT_TAG "${MLOS_GIT_TAG}" + # Since we use GitVersionTask for versioning nugets we need a non-shallow fetch. + GIT_SHALLOW OFF +) +FetchContent_GetProperties(mlos) +# Allow building the MLOS projects with a different build type than the rest parent project, +# but by default use a Release build. +set(MLOS_CMAKE_BUILD_TYPE Release) +if(NOT mlos_POPULATED) + # Actually fetch the mlos code (at generation time). + FetchContent_Populate(mlos) + + # In case we checked out an unnamed version (e.g. in Github Actions CI), + # give it a local branch name for GitVersionTask to compute from. + if("${MLOS_GIT_TAG}" STREQUAL "HEAD") + execute_process( + COMMAND git checkout --quiet --detach + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + execute_process( + COMMAND git branch -f local-cmake-checkout + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + execute_process( + COMMAND git branch --no-track -f local-cmake-checkout origin/HEAD + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + execute_process( + COMMAND git checkout local-cmake-checkout + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + + # Also make the upstream/main available for comparison. + execute_process( + COMMAND git remote add upstream https://github.com/microsoft/MLOS + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + execute_process( + COMMAND git fetch -q upstream + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + execute_process( + COMMAND git branch -f main upstream/main + WORKING_DIRECTORY "${mlos_SOURCE_DIR}") + endif() + + # Make the MLOS project targets available for other cmake targets to reference as dependencies. + add_subdirectory("${mlos_SOURCE_DIR}" "${mlos_BINARY_DIR}" EXCLUDE_FROM_ALL) + + # Instruct those other targets how to find the Mlos cmake module. + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${mlos_SOURCE_DIR}/external/cmake") +endif() + +# Include the MLOS cmake module (from the path above). +include(MLOS) + +# Now, we instruct all the external project component targets to look in the +# following path for their Mlos codegen outputs. +# +# Note: This should match the path set in the C# build props. +# See Also: +# - ExternalIntegrationExample.SettingsRegistry/ExternalIntegrationExample.SettingsRegistry.csproj +# - ExternalIntegrationExample.SettingsRegistry/build/Common.props +# +# It can also be set in the add_mlos_settings_registry() definition. +# See Also: +# - ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt +# +set(MlosCodeGenBaseOutDir "${CMAKE_SOURCE_DIR}/Mlos.CodeGen.out") +include_directories(${MlosCodeGenBaseOutDir}) + +# The path to place the Mlos SettingRegistry dlls. +# +# See Also: +# - ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt +# +set(MlosSettingsRegistryDllDir "${CMAKE_BINARY_DIR}/SettingsRegistryDlls") diff --git a/external/ExternalIntegrationExample/include/Common.h b/external/ExternalIntegrationExample/include/Common.h new file mode 100644 index 0000000000..266c9ceb1c --- /dev/null +++ b/external/ExternalIntegrationExample/include/Common.h @@ -0,0 +1,42 @@ +//********************************************************************* +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root +// for license information. +// +// @File: Common.h +// +// Purpose: +// This is supposed to represent a typical header file for an external project. +// +// Notes: +// +// +//********************************************************************* + +#pragma once + +// Existing project includes: +#include +#include +#include + +// Now we start adding Mlos items: + +// Include Mlos.Core shared memory channel APIs. +// +// This also includes the the core message headers code generated from the +// Mlos.NetCore project for registering new assemblies for application specific +// smart components. +// +#include "Mlos.Core.h" + +// Include application specific codegen files and sets up the global dispatch table. +// +#include "GlobalDispatchTable.h" + +// Include Mlos.Core inline implementations. +// +// Note: This should be included after the application specific code gen +// included in GlobalDispatchTable.h +// +#include "Mlos.Core.inl" diff --git a/external/ExternalIntegrationExample/include/GlobalDispatchTable.h b/external/ExternalIntegrationExample/include/GlobalDispatchTable.h new file mode 100644 index 0000000000..42fd723542 --- /dev/null +++ b/external/ExternalIntegrationExample/include/GlobalDispatchTable.h @@ -0,0 +1,75 @@ +//********************************************************************* +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. See License.txt in the project root +// for license information. +// +// @File: GlobalDispatchTable.h +// +// Purpose: +// This file adds the code generated files for this smart component's settings +// and messages. Once the messages structures are defined, the message +// processing loop needs to be made aware of them by concatenating them into +// the GlobalDispatchTable (a lookup table that tells the message processing +// loop which callback to execute for which message type). +// Note that this is a compile time constant operation for effeciency. +// +// Notes: +// This is expected to be included *after* Mlos.Core.h and *before* +// Mlos.Core.inl. See stdafx.h for an example. +// +//********************************************************************* + +#pragma once + +// Include the codegen files from the ExternalIntegrationExample settings registry. +// +// Note: this expects out/Mlos.CodeGen.out/ to be listed in the default include +// search path for the compiler. +// +// Note: the codegen files for Mlos::Core are expected to have been included +// before these (typically via inclusion of Mlos.Core.h in stdafx.h or Common.h). +// +#include "ExternalIntegrationExample/SettingsProvider_gen_base.h" +#include "ExternalIntegrationExample/SettingsProvider_gen_callbacks.h" +#include "ExternalIntegrationExample/SettingsProvider_gen_dispatch.h" + +// Compute the base index offset for each settings registry's local dispatch +// table within the global dispatch table. +// +// See MLOS/source/Examples/SmartCache/GlobalDispatchTable.h for a more detailed explanation. +// +constexpr uint32_t ExternalIntegrationExample::ObjectDeserializationHandler::DispatchTableBaseIndex() +{ + // Since the settings registry codegen for Mlos::Core were included first, + // the ExternalIntegrationExample dispatch table starts after that one. + // + // If additional settings registries were included here, they would + // similarly add the previously included settings registry + // DispatchTableElementCounts to compute their own relative starting offset. + // + // See SmartSharedChannel/GlobalDispatchTable.h for an example. + // + return static_cast(Mlos::Core::ObjectDeserializationHandler::DispatchTableElementCount()); +} + +// Concatenate all of the local settings registry dispatch tables together to +// form the global dispatch table, fixing up their individual message ids +// relative to their new location within the global dispatch table. +// +// This effectively registers each of the code generated messages for the +// channel message reader loops to be able to process. +// +// Note: messages still need to have a handler function assigned to their +// Callback function pointer so that application specific code can be executed +// when its messages are received. +// +// See MLOS/source/Examples/SmartCache/Main.cpp for an example. +// +constexpr auto GlobalDispatchTable() +{ + auto globalDispatchTable = Mlos::Core::DispatchTable<0>() + .concatenate(Mlos::Core::ObjectDeserializationHandler::DispatchTable) + .concatenate(ExternalIntegrationExample::ObjectDeserializationHandler::DispatchTable); + + return globalDispatchTable; +} diff --git a/external/README.md b/external/README.md new file mode 100644 index 0000000000..a0c5f54845 --- /dev/null +++ b/external/README.md @@ -0,0 +1,9 @@ +# External Integration Code and Examples + +[This directory](./#mlos-github-tree-view) contains some code, configuration, and examples of how to integrate MLOS with external projects. + +For instance: + +- The [`cmake`](./cmake/) directory contains an `MLOS` module for use in projects using the `cmake` build system. +- [`ExternalIntegrationExample`](./ExternalIntegrationExample/) contains a stripped down example showing how to use it. +- [`leveldb`](./leveldb/) contains a notebook and other examples of using portions of the MLOS toolkit to tune [LevelDb](https://github.com/google/leveldb) parameters. diff --git a/external/cmake/MLOS.cmake b/external/cmake/MLOS.cmake new file mode 100644 index 0000000000..efde7d8875 --- /dev/null +++ b/external/cmake/MLOS.cmake @@ -0,0 +1,112 @@ +# A cmake module to help integrating MLOS codegen into other cmake based projects. +# Currently expected to be used with FetchContent() +# See the ExternalIntegrationExample for a detailed example. + +cmake_minimum_required(VERSION 3.15) + +get_filename_component(MLOS_ROOT "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) + +if(NOT MLOS_CMAKE_BUILD_TYPE) + set(MLOS_CMAKE_BUILD_TYPE "${CMAKE_BUILD_TYPE}") +endif() +if(NOT ((${MLOS_CMAKE_BUILD_TYPE} STREQUAL "Release") OR (${MLOS_CMAKE_BUILD_TYPE} STREQUAL "Debug"))) + message(FATAL_ERROR "Unsupported MLOS_CMAKE_BUILD_TYPE: ${MLOS_CMAKE_BUILD_TYPE}.") +endif() + +# TODO: Convert these to target_include_directories() using some macros? +# Make sure to include the Mlos project's codegen output directories to the include search path. +include_directories("${MLOS_ROOT}/out/Mlos.CodeGen.out/${MLOS_CMAKE_BUILD_TYPE}") +# Also include the Mlos.Core source in the headers search path. +include_directories("${MLOS_ROOT}/source/Mlos.Core") + +if((NOT DEFINED CMAKE_CXX_STANDARD) OR (CMAKE_CXX_STANDARD LESS 14)) + message(SEND_ERROR "CMAKE_CXX_STANDARD >= 14 is required by MLOS codegen output.") +endif() + +# add_mlos_settings_registry() +# +# A wrapper function build an MLOS C# SettingsRegistry .csproj using "dotnet build". +# +# See ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt for an example. +# +function(add_mlos_settings_registry) + set(options USE_LOCAL_MLOS_NUGETS) + set(oneValueArgs NAME DIRECTORY CODEGEN_OUTPUT_DIR BINPLACE_DIR) + cmake_parse_arguments(add_mlos_settings_registry "${options}" "${oneValueArgs}" "${multiValueArgs}" ${ARGN}) + + if(NOT DEFINED MLOS_ROOT) + message(FATAL_ERROR "Missing MLOS_ROOT.") + endif() + + set(NAME ${add_mlos_settings_registry_NAME}) + set(DIRECTORY ${add_mlos_settings_registry_DIRECTORY}) + set(CODEGEN_OUTPUT_DIR ${add_mlos_settings_registry_CODEGEN_OUTPUT_DIR}) + set(BINPLACE_DIR ${add_mlos_settings_registry_BINPLACE_DIR}) + set(USE_LOCAL_MLOS_NUGETS ${add_mlos_settings_registry_USE_LOCAL_MLOS_NUGETS}) + + # In the "dotnet build" command update some properites + # - BinplaceDir + # - Codegen Output Dir + + if(CODEGEN_OUTPUT_DIR) + get_filename_component(CODEGEN_OUTPUT_DIR "${CODEGEN_OUTPUT_DIR}" ABSOLUTE) + set(CODEGEN_OUTPUT_DIR_ARGS "'/p:MlosSettingsSystemCodeGenOutputDirectory=${CODEGEN_OUTPUT_DIR}'") + else() + set(CODEGEN_OUTPUT_DIR_ARGS "") + endif() + + if(BINPLACE_DIR) + get_filename_component(BINPLACE_DIR "${BINPLACE_DIR}" ABSOLUTE) + set(BINPLACE_DIR_ARGS "'/p:MlosSettingsRegistryAssemblyOutputDirectory=${BINPLACE_DIR}'") + else() + set(BINPLACE_DIR_ARGS "") + endif() + + if(${USE_LOCAL_MLOS_NUGETS}) + set(MlosLocalPkgOutput "${MLOS_ROOT}/target/pkg/${MLOS_CMAKE_BUILD_TYPE}") + set(NUGET_RESTORE_ARGS "'/p:RestoreSources=${MlosLocalPkgOutput}\;https://api.nuget.org/v3/index.json'") + set(MLOS_PKGVERS_ARGS "'/p:MlosPackageVersion=*-*'") + # Add a dependency on the local nuget packaging build step in the main MLOS repo. + set(MlosLocalPkgTargetDeps Mlos.NetCore.Components.Packages) + else() + set(NUGET_RESTORE_ARGS "") + endif() + + # Parse the csproj files in the directory to determine the *.cs file dependencies. + execute_process( + COMMAND "${MLOS_ROOT}/build/CMakeHelpers/ParseCsProjForCsFiles.sh" + WORKING_DIRECTORY "${DIRECTORY}" + OUTPUT_VARIABLE CS_SOURCES + #COMMAND_ECHO STDERR + ) + + set(CSPROJ ${NAME}.csproj) + + set(DEPENDENCIES + ${DIRECTORY}/${CSPROJ} + ${CS_SOURCES} + ${MlosLocalPkgOutput} ${MlosLocalPkgTargetDeps}) + + # Rather than track the outputs from the "dotnet build" (which could change according to how the build was authored), + # we'll simply use a build.stamp file in the cmake output dir to mark when the "dotnet build" last succeeded. + get_filename_component(DIRECTORY "${DIRECTORY}" ABSOLUTE) + file(RELATIVE_PATH DIRECTORY_RELATIVE_TO_SOURCE_ROOT "${CMAKE_SOURCE_DIR}" "${DIRECTORY}") + set(OUTDIR "${CMAKE_BINARY_DIR}/${DIRECTORY_RELATIVE_TO_SOURCE_ROOT}") + set(BUILD_STAMP "${OUTDIR}/build.stamp") + + add_custom_command(OUTPUT "${BUILD_STAMP}" "${CODEGEN_OUTPUT_DIR}" + # we compose the dependency graph already above, so we can skip + # building project references here in order to avoid some parallel + # dotnet processes accessing the same files. + COMMAND ${DOTNET} build -m --configuration ${CMAKE_BUILD_TYPE} ${NUGET_RESTORE_ARGS} ${MLOS_PKGVERS_ARGS} ${CODEGEN_OUTPUT_DIR_ARGS} ${BINPLACE_DIR_ARGS} "${CSPROJ}" + # Also, "dotnet build" doesn't update timestamps in a make compatible + # way, so we also mark the projects as having been built using touch. + COMMAND ${CMAKE_COMMAND} -E make_directory "${OUTDIR}" + COMMAND ${CMAKE_COMMAND} -E touch "${BUILD_STAMP}" + DEPENDS "${DEPENDENCIES}" + WORKING_DIRECTORY "${DIRECTORY}" + COMMENT "Building dotnet assembly and MLOS CodeGen for ${NAME}.") + + add_custom_target(${NAME} ALL + DEPENDS "${BUILD_STAMP}" "${CODEGEN_OUTPUT_DIR}") +endfunction() diff --git a/external/cmake/README.md b/external/cmake/README.md new file mode 100644 index 0000000000..eb738904a8 --- /dev/null +++ b/external/cmake/README.md @@ -0,0 +1,83 @@ +# External CMake Modules + +This directory contains `cmake` modules for external project integration. + +## Overview + +The general usage pattern is to + +1. Use [`FetchContent`](https://cmake.org/cmake/help/v3.15/module/FetchContent.html) to obtain the MLOS source tree as a dependency + for the external project's `cmake` build system and add this directory to their `CMAKE_MODULE_PATH` so that `include(MLOS)` can + find the [`MLOS.cmake`](./MLOS.cmake#mlos-github-tree-view) file in this directory. + + Note: This can happen at a common level for an entire external project's `cmake` build system. + + ```cmake + # Make MLOS source code available in the external project. + include(FetchContent) + FetchContent_Declare( + mlos + GIT_REPOSITORY "https://github.com/microsoft/MLOS" + GIT_TAG "main" + GIT_SHALLOW OFF + ) + FetchContent_GetProperties(mlos) + set(MLOS_CMAKE_BUILD_TYPE Release) + if(NOT mlos_POPULATED) + FetchContent_Populate(mlos) + add_subdirectory("${mlos_SOURCE_DIR}" "${mlos_BINARY_DIR}" EXCLUDE_FROM_ALL) + + # Instruct other targets how to find the Mlos cmake module. + set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH} "${mlos_SOURCE_DIR}/external/cmake") + endif() + + # Include the MLOS cmake module (from the path above). + # (this includes the add_mlos_settings_registry() function used next) + include(MLOS) + + # Define some common settings used in C# SettingsRegistry and C++ builds. + set(MlosCodeGenBaseOutDir "${CMAKE_SOURCE_DIR}/Mlos.CodeGen.out") + include_directories(${MlosCodeGenBaseOutDir}) + + set(MlosSettingsRegistryDllDir "${CMAKE_BINARY_DIR}/SettingsRegistryDlls") + ``` + + See Also: [`ExternalIntegrationExample/MlosBuildIntegrations.cmake`](../ExternalIntegrationExample/cmake/MlosBuildIntegrations.cmake#mlos-github-tree-view) + +2. Use the `add_mlos_settings_registry()` function provided there to provide a `cmake` target wrapper for `dotnet build` of various + *`SmartComponent`*`.SettingsRegistry.csproj` files the external project will add. + + ```cmake + # MySmartComponent.SettingsRegistry/CMakeLists.txt: + # A wrapper for the MySmartComponent.csproj file in the same directory. + + include(MLOS) + + add_mlos_settings_registry( + NAME MySmartComponent.SettingsRegistry + DIRECTORY "${CMAKE_CURRENT_LIST_DIR}" + CODEGEN_OUTPUT_DIR "${MlosCodeGenBaseOutDir}/MySmartComponent" + BINPLACE_DIR "${MlosSettingsRegistryDllDir}" + USE_LOCAL_MLOS_NUGETS + ) + ``` + + See Also: [`ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt`](../ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/CMakeLists.txt#mlos-github-tree-view) + +3. Reference those using `add_dependencies()` in the C/C++ components. + + ```cmake + # MySmartComponent/CMakeLists.txt: + + # ... existing cmake definitions + + target_link_libraries(${PROJECT_NAME} Mlos.Core) + + # SettingsRegistry projects produce C++ codegen artifacts, that this project + # consumes, so we mark that project as a dependency. + # + add_subdirectory(MySmartComponent.SettingsRegistry) + add_dependencies(${PROJECT_NAME} MySmartComponent.SettingsRegistry) + ``` + + See Also: [`ExternalIntegrationExample/CMakeLists.txt`](../ExternalIntegrationExample/CMakeLists.txt#mlos-github-tree-view) diff --git a/external/leveldb/README.md b/external/leveldb/README.md new file mode 100644 index 0000000000..e5bfe5c164 --- /dev/null +++ b/external/leveldb/README.md @@ -0,0 +1,5 @@ +# LevelDb Tuning Examples with MLOS + +This [directory](./#mlos-github-tree-view) contains some examples of using MLOS to tune an external project like LevelDb. + +A rendered notebook can be found [here](https://microsoft.github.io/MLOS/notebooks/LevelDbTuning). diff --git a/source/CMakeLists.txt b/source/CMakeLists.txt index 477149cca9..e9c2f6d176 100644 --- a/source/CMakeLists.txt +++ b/source/CMakeLists.txt @@ -17,6 +17,7 @@ add_subdirectory(Mlos.Agent EXCLUDE_FROM_ALL) add_subdirectory(Mlos.Agent.GrpcClient EXCLUDE_FROM_ALL) add_subdirectory(Mlos.Agent.GrpcServer EXCLUDE_FROM_ALL) add_subdirectory(Mlos.Agent.Server EXCLUDE_FROM_ALL) +add_subdirectory(Mlos.NetCore.Components.Packages EXCLUDE_FROM_ALL) # Start listing our other C++ sub projects. # See Also: dirs.proj diff --git a/source/Mlos.NetCore.Components.Packages/CMakeLists.txt b/source/Mlos.NetCore.Components.Packages/CMakeLists.txt new file mode 100644 index 0000000000..ecc7055c07 --- /dev/null +++ b/source/Mlos.NetCore.Components.Packages/CMakeLists.txt @@ -0,0 +1,44 @@ +# This is a CMakeLists for the dotnet meta-project that handles nuget packaging. +# We use this to allow other CMakeLists (e.g. from external projects referencing +# MLOS via FetchContent) a dependency to reference so they can obtain local +# packages. + +project(Mlos.NetCore.Components.Packages LANGUAGES NONE) + +get_filename_component(MLOS_ROOT "${CMAKE_CURRENT_LIST_DIR}/../.." ABSOLUTE) +include("${MLOS_ROOT}/build/Mlos.Common.cmake") + +# Note: for this particular project, we don't use the build/Mlos.NetCore.cmake +# and instead provide our own custom targets. + +# Parse the csproj files in the directory to determine the *.csproj project +# dependencies and turn them into target dependencies. +execute_process( + COMMAND "${MLOS_ROOT}/build/CMakeHelpers/ParseCsProjForCsProjs.sh" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + OUTPUT_VARIABLE CS_PROJS + #COMMAND_ECHO STDERR +) + +set(PROJ "${PROJECT_SOURCE_DIR}/${PROJECT_NAME}.proj") +set(PKG_OUTPUT_DIR "${MLOS_ROOT}/target/pkg/${CMAKE_BUILD_TYPE}") + +set(DEPENDENCIES + ${PROJ} + ${CS_PROJS} +) + +file(RELATIVE_PATH RelativeProjectSourcePath "${MLOS_ROOT}" "${PROJECT_SOURCE_DIR}") +set(OUTDIR "${MLOS_ROOT}/out/dotnet/${RelativeProjectSourcePath}") +set(ProjectBuildStampFile "${OUTDIR}/build.stamp") + +add_custom_command(OUTPUT "${ProjectBuildStampFile}" "${PKG_OUTPUT_DIR}" + COMMAND dotnet build -c ${CMAKE_BUILD_TYPE} "${PROJ}" + COMMAND ${CMAKE_COMMAND} -E make_directory "${OUTDIR}" + COMMAND ${CMAKE_COMMAND} -E touch "${ProjectBuildStampFile}" + DEPENDS "${DEPENDENCIES}" + WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" + COMMENT "Creating MLOS Nuget Packages") + +add_custom_target(${PROJECT_NAME} ALL + DEPENDS "${ProjectBuildStampFile}" "${PKG_OUTPUT_DIR}") diff --git a/source/Mlos.NetCore.Components.Packages/Makefile b/source/Mlos.NetCore.Components.Packages/Makefile index 1c80e81d9e..5b000db866 100644 --- a/source/Mlos.NetCore.Components.Packages/Makefile +++ b/source/Mlos.NetCore.Components.Packages/Makefile @@ -1,3 +1,3 @@ RelativePathToProjectRoot := ../.. -include $(RelativePathToProjectRoot)/build/DotnetWrapper.mk -#include $(RelativePathToProjectRoot)/build/CMakeWrapper.mk +#include $(RelativePathToProjectRoot)/build/DotnetWrapper.mk +include $(RelativePathToProjectRoot)/build/CMakeWrapper.mk diff --git a/source/Mlos.SettingsSystem.CodeGen/build/Mlos.SettingsSystem.CodeGen.targets b/source/Mlos.SettingsSystem.CodeGen/build/Mlos.SettingsSystem.CodeGen.targets index bb1e162879..5835280251 100644 --- a/source/Mlos.SettingsSystem.CodeGen/build/Mlos.SettingsSystem.CodeGen.targets +++ b/source/Mlos.SettingsSystem.CodeGen/build/Mlos.SettingsSystem.CodeGen.targets @@ -92,7 +92,7 @@ - + diff --git a/website/Makefile b/website/Makefile index 8a3fbe2153..157b5a9c3a 100644 --- a/website/Makefile +++ b/website/Makefile @@ -26,6 +26,7 @@ clean-hugo-site: rm -rf content/source rm -rf content/test rm -rf content/notebooks + rm -rf content/external rm -rf public .PHONY: diff --git a/website/build_site.sh b/website/build_site.sh index dd90a9eac7..91720cf59c 100755 --- a/website/build_site.sh +++ b/website/build_site.sh @@ -92,7 +92,9 @@ cp ../LICENSE.txt content/ -name '*.md' -or \ -name '*.png' -or \ -name '*.svg' \ - | grep -v -e '\.nuget/packages/' \ + | grep -v \ + -e '/\.nuget/packages/' \ + -e '/build/_deps/' \ ) | while read path; do dir=$(dirname "$path") file=$(basename "$path") diff --git a/website/content/menu/index.md b/website/content/menu/index.md index 073b5f8437..548a9837e6 100644 --- a/website/content/menu/index.md +++ b/website/content/menu/index.md @@ -27,6 +27,8 @@ headless = true - [Mlos.Core Shared Memory Communication Channel]({{< relref "/source/Mlos.Core/doc" >}}) - [Mlos.Agent.Server]({{< relref "/source/Mlos.Agent.Server" >}}) +- [External Integration]({{< relref "/external" >}}) + - [Code of Conduct]({{< relref "/CODE_OF_CONDUCT" >}}) - [Contributing]({{< relref "/CONTRIBUTING" >}}) - [MLOS Github Repository](https://github.com/Microsoft/MLOS)