Create Nuget Packages for C#

.github/workflows/main.yml

@@ -33,13 +33,21 @@ jobs:
  steps:
  # Checks-out your repository under $GITHUB_WORKSPACE, so your job can access it
  - uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
  # Runs multiple commands using the runners shell
- - name: Run ${{ matrix.configuration }} msbuild
+ - name: Msbuild/test ${{ matrix.configuration }}
  timeout-minutes: 15
  shell: cmd
  run: |
  set Configuration=${{ matrix.configuration }}
  scripts\build.windows.cmd
+ - name: Pack/test ${{ matrix.configuration }}
+ timeout-minutes: 15
+ shell: cmd
+ run: |
+ set Configuration=${{ matrix.configuration }}
+ scripts\package.windows.cmd
 
  # Docker build process overview:
  # - We currently use the docker.pkg.github.com package registry as a staging
@@ -224,6 +232,8 @@ jobs:
  steps:
  - name: Checkout
  uses: actions/checkout@v2
+ with:
+ fetch-depth: 0
  # "Fetch" the docker image precached by the "docker-image-cached-build" job that
  # we depend on by re-executing (roughly) its same steps.
  - name: Set up Docker Buildx
@@ -279,6 +289,14 @@ jobs:
  run: |
  docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
  make dotnet-test
+ - name: Run ${{ matrix.configuration }} dotnet pack test
+ timeout-minutes: 5
+ shell: bash
+ run: |
+ docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
+ make -C source/Mlos.NetCore.Components.Packages/
+ docker exec mlos-${{ matrix.configuration }}-build-ubuntu-${{ matrix.UbuntuVersion }} \
+ dotnet build external/ExternalIntegrationExample/ExternalIntegrationExample.SettingsRegistry/
  - name: Run ${{ matrix.configuration }} cmake build
  timeout-minutes: 5
  shell: bash

GitVersion.yml

@@ -0,0 +1,22 @@
+# This is the config file for the GitVersionTask tool used in build/versions/Mlos.NetCore.Version.props
+# To see the resulting config use the following commands
+# dotnet tool install --global gitversion.tool
+# dotnet-gitversion /showconfig # should be run from this directory
+
+# We currently merge everything to main and don't yet have release/{major.minor} branches
+mode: Mainline
+# Note: GitVersionTask will still search the history for v{major.minor.patch} tags to automatically
+# increment the .patch portion of the versions it produces and calculate the number of commits since
+# branch for the SemVers .{branchName}+{commits} suffix.
+# So, to do releases in this way, from time to time we need to decide that certain commits should be
+# tagged with a new v{major.minor.0} version.
+increment: Patch
+branches:
+ master:
+ # Remove the "master" branch name from consideration.
+ # Also allow matching the "main" branch from "origin" remote on CI pipelines with shallow fetches.
+ regex: ^((origin|upstream|github)/)?main$
+ # Mark any packages produced directly from main as "prerelease" for now.
+ tag: beta
+ # We don't currently have develop or release/{vers} branches to look for.
+ source-branches: []

Makefile

@@ -33,11 +33,15 @@ check: all test
 install: dotnet-install cmake-install
  @ echo "make install target finished."
 
+.PHONY: pack
+pack:
+ $(MAKE) -C source/Mlos.NetCore.Components.Packages
+
 .PHONY: clean
 clean: cmake-clean dotnet-clean grpc-clean mlos-codegen-clean website-clean python-clean
 
 .PHONY: distclean
-distclean: clean cmake-distclean
+distclean: clean dotnet-distclean dotnet-pkgs-clean cmake-distclean
 
 .PHONY: rebuild
 rebuild: clean all

build/Common.mk

@@ -75,7 +75,7 @@ ifeq ($(filter $(CONFIGURATION),$(SupportedConfigurations)),)
 endif
 
 # Variables tracking additional targets to be added to by later Makefile wrappers.
-handledtargets = all test check install clean distclean rebuild ctags
+handledtargets = all test check install pack clean distclean rebuild ctags
 
 # Mark this file as imported.
 MlosCommonMkImported := true

build/DotnetWrapper.mk

@@ -14,7 +14,11 @@ install: dotnet-install
 clean: dotnet-clean
 
 .PHONY: distclean
-distclean: clean
+distclean: dotnet-distclean
+
+.PHONY: pack
+pack: dotnet-pack
+ @ echo "make pack target finished."
 
 .PHONY: test
 test: dotnet-test

build/DotnetWrapperRules.mk

@@ -17,17 +17,27 @@ endif
 # Needed for "clean" target.
 # See Also: Mlos.Common.props
 DotnetBaseOutDir := $(MLOS_ROOT)/out/dotnet
+DotnetBasePkgDir := $(MLOS_ROOT)/target/pkg/$(CONFIGURATION)
 DotnetOutputPath := $(DotnetBaseOutDir)/$(RelativeSourceDir)
 
 # Find all the *.csproj, dirs.proj files in this directory and make some
 # corresponding fake targets for the "all" target to depend on.
 Csprojs := $(wildcard *.csproj)
 CsprojBuildTargets := $(Csprojs:.csproj=.csproj.fake-build-target)
 CsprojBuildQuickTargets := $(Csprojs:.csproj=.csproj.fake-build-quick-target)
+CsprojPackTargets := $(Csprojs:.csproj=.csproj.fake-pack-target)
 CsprojTestTargets := $(Csprojs:.csproj=.csproj.fake-test-target)
-DirsProj := $(wildcard dirs.proj)
+CsprojCleanTargets := $(Csprojs:.csproj=.csproj.fake-clean-target)
+CsprojCleanQuickTargets := $(Csprojs:.csproj=.csproj.fake-clean-quick-target)
+# Actually, include other *.proj files as well.
+DirsProj := $(wildcard *.proj)
 DirsProjBuildTarget := $(DirsProj:.proj=.proj.fake-build-target)
+DirsProjPackTarget := $(DirsProj:.proj=.proj.fake-pack-target)
 DirsProjTestTarget := $(DirsProj:.proj=.proj.fake-test-target)
+DirsProjCleanTarget := $(DirsProj:.proj=.proj.fake-clean-target)
+
+# Increase log verbosity slightly.
+#MSBUILD_ARGS := -v:normal
 
 # To be added to the including Makefile's all target.
 .PHONY: dotnet-build
@@ -40,10 +50,22 @@ dotnet-build: $(CsprojBuildTargets) $(DirsProjBuildTarget)
 dotnet-build-quick: $(CsprojBuildQuickTargets)
  @ echo "make dotnet-build-quick target finished."
 
+.PHONY: dotnet-pack
+dotnet-pack: $(CsprojPackTargets) $(DirsProjPackTarget)
+ @ echo "make dotnet-pack target finished."
+
 .PHONY: dotnet-test
 dotnet-test: $(CsprojTestTargets) $(DirsProjTestTarget)
  @ echo "make dotnet-test target finished."
 
+.PHONY: dotnet-clean
+dotnet-clean: $(CsprojCleanTargets) $(DirsProjCleanTarget)
+ @ echo "make dotnet-clean target finished."
+
+.PHONY: dotnet-clean-quick
+dotnet-clean-quick: $(CsprojCleanQuickTargets)
+ @ echo "make dotnet-clean-quick target finished."
+
 .PHONY: dotnet-install
 dotnet-install:
  @ echo "make dotnet-install is currently a no-op."
@@ -56,11 +78,17 @@ dotnet-install:
  @ # Note: This command currently also does a "dotnet restore" first,
  @ # which can be slow, however is difficult to check when it is unnecessary.
  @ # Note: -m tells it to build in parallel.
- @ $(DOTNET) build -m --configuration $(CONFIGURATION) $(@:.fake-build-target=)
+ @ $(DOTNET) build $(MSBUILD_ARGS) -m --configuration $(CONFIGURATION) $(@:.fake-build-target=)
 
 %.fake-build-quick-target:
  @ # A target to allow quickly rebuilding just a given project and none of its dependencies.
- @ $(DOTNET) build -m --configuration $(CONFIGURATION) --no-restore /p:BuildProjectReferences=false $(@:.fake-build-quick-target=)
+ @ $(DOTNET) build $(MSBUILD_ARGS) -m --configuration $(CONFIGURATION) --no-restore /p:BuildProjectReferences=false $(@:.fake-build-quick-target=)
+
+# For each of the fake test targets, just call "dotnet pack" on its
+# corresponding *.csproj file
+# For now, don't force a rebuild first.
+%.fake-pack-target: #%.fake-build-target
+ @ $(DOTNET) pack $(MSBUILD_ARGS) --no-build -m --configuration $(CONFIGURATION) $(@:.fake-pack-target=)
 
 # By default don't run certain tests.
 # To override, run with:
@@ -74,14 +102,31 @@ endif
 # corresponding *.csproj file
 # For now, don't force a rebuild first.
 %.fake-test-target: #%.fake-build-target
- $(DOTNET) test --no-build -m --configuration $(CONFIGURATION) $(DOTNET_TEST_FILTER) $(@:.fake-test-target=)
+ $(DOTNET) test $(MSBUILD_ARGS) --no-build -m --configuration $(CONFIGURATION) $(DOTNET_TEST_FILTER) $(@:.fake-test-target=)
+
+%.fake-clean-target:
+ $(DOTNET) build $(MSBUILD_ARGS) /t:clean --no-restore -m --configuration $(CONFIGURATION) $(@:.fake-clean-target=)
+
+%.fake-clean-quick-target:
+ $(DOTNET) build $(MSBUILD_ARGS) /t:clean /p:BuildProjectReferences=false --no-restore -m --configuration $(CONFIGURATION) $(@:.fake-clean-quick-target=)
 
 # Note: this clean method somewhat lazily removes both Debug and Release build outputs.
 # To be added to the including Makefile's clean target.
-.PHONY: dotnet-clean
-dotnet-clean:
- @ $(RM) $(DotnetOutputPath)
+.PHONY: dotnet-distclean
+dotnet-distclean:
+ $(RM) $(DotnetOutputPath)
+ @ $(MKDIR) $(DotnetOutputPath)
+
+.PHONY: dotnet-pkgs-clean
+dotnet-pkgs-clean:
+ $(RM) $(DotnetBasePkgDir)
+ @ $(MKDIR) $(DotnetBasePkgDir)
 
-handledtargets += $(Csprojs) $(CsProjBuildTargets) $(CsProjTestTargets) \
- $(DirsProj) $(DirsProjBuildTarget) $(DirsProjTestTarget) \
- dotnet-build dotnet-build-quick dotnet-install dotnet-test dotnet-clean
+handledtargets += $(Csprojs) $(DirsProj) \
+ dotnet-build $(CsprojBuildTargets) $(DirsProjBuildTarget) \
+ dotnet-build-quick $(CsprojBuildQuickTargets) \
+ dotnet-pack $(CsprojPackTargets) $(DirsProjPackTarget) \
+ dotnet-test $(CsprojTestTargets) $(DirsProjTestTarget) \
+ dotnet-clean $(CsprojCleanTargets) $(DirsProjCleanTarget) \
+ dotnet-clean-quick $(CsprojCleanQuickTargets) \
+ dotnet-install dotnet-pkgs-clean dotnet-distclean

build/Mlos.Common.props

@@ -32,6 +32,12 @@
  <PropertyGroup>
  <MlosCodeGenOutputPathRoot Condition="'$(MlosCodeGenOutputPathRoot)' == ''">$(BaseDir)\out\Mlos.CodeGen.out\$(BuildType)</MlosCodeGenOutputPathRoot>
  </PropertyGroup>
+ <!-- Convenience properties to detect when we're in certain build environments. -->
+ <PropertyGroup>
+ <!-- https://onebranch.visualstudio.com/Pipeline/_wiki/wikis/Pipeline.wiki/325/Versioning -->
+ <CdpxBuildPipeline Condition="'$(CDP_COMMIT_ID)' != '' Or '$(CDP_BUILD_NUMBER)' != ''">true</CdpxBuildPipeline>
+ <HasAdoPipeline Condition="Exists('$(BaseDir)\.pipelines\pipeline.user.windows.yml')">true</HasAdoPipeline>
+ </PropertyGroup>
  <!-- Configure warning levels and errors. -->
  <PropertyGroup>
  <WarningLevel>4</WarningLevel>

build/Mlos.MetaProj.targets

@@ -0,0 +1,21 @@
+<?xml version="1.0" encoding="utf-8"?>
+<Project DefaultTargets="Build" ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <!-- Some targets for basic .proj files to include for test and package proj files to build their dependencies properly. -->
+ <Target Name="Build" BeforeTargets="Test">
+ <MSBuild Projects="@(ProjectReference)" Targets="Build" RunEachTargetSeparately="true"
+ Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
+ </Target>
+ <Target Name="Clean">
+ <MSBuild Projects="@(ProjectReference)" Targets="Clean" RunEachTargetSeparately="true"
+ Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
+ </Target>
+ <Target Name="Restore">
+ <MSBuild Projects="@(ProjectReference)" Targets="Restore" RunEachTargetSeparately="true"
+ Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
+ </Target>
+ <Target Name="VerifyAlteredTargetsUsed">
+ <MSBuild Projects="@(ProjectReference)" Targets="VerifyAlteredTargetsUsed" RunEachTargetSeparately="true"
+ Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
+ </Target>
+ <Import Project="$(BaseDir)\build\Mlos.Common.targets" />
+</Project>

build/Mlos.NetCore.Grpc.props

@@ -11,6 +11,10 @@
  <PackageReference Include="Grpc.AspNetCore" Version="$(GrpcVersion)" Condition="'$(GrpcServices)' == 'Server' Or '$(GrpcServices)' == 'Both'" />
  <PackageReference Include="Grpc.Core" Version="$(GrpcVersion)" />
  <PackageReference Include="Grpc.Net.Client" Version="$(GrpcVersion)" Condition="'$(GrpcServices)' == 'Client' OR '$(GrpcServices)' == 'Both'" />
- <PackageReference Include="Grpc.Tools" Version="$(GrpcVersion)" />
+ <PackageReference Include="Grpc.Tools" Version="$(GrpcVersion)">
+ <!-- Grpc.Tools is a build time dependency only - don't include it in any nugets we create. -->
+ <PrivateAssets>all</PrivateAssets>
+ <IncludeAssets>runtime; build; native; contentfiles; analyzers</IncludeAssets>
+ </PackageReference>
  </ItemGroup>
 </Project>

build/Mlos.NetCore.Package.props

@@ -0,0 +1,42 @@
+<?xml version="1.0" encoding="utf-8"?>
+<!--
+ This file effectively takes the place of a number of .nuspec declarations
+ so that we can build directly with .csproj files using "dotnet pack".
+-->
+<Project ToolsVersion="Current" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+ <!--
+ Set some .nuspec style properties
+ Note: The Copyright property also comes from the Mlos.NetCore.Version.props file.
+ -->
+ <PropertyGroup>
+ <PackageProjectUrl>https://github.com/Microsoft/MLOS</PackageProjectUrl>
+ <RepositoryUrl>https://github.com/Microsoft/MLOS.git</RepositoryUrl>
+ <RepositoryType>git</RepositoryType>
+ <Authors>Microsoft</Authors>
+ <PackageRequireLicenseAcceptance>false</PackageRequireLicenseAcceptance>
+ <PackageLicenseExpression>MIT</PackageLicenseExpression>
+ <PackageTags>MLOS;Machine Learning;Optimization;Systems</PackageTags>
+ </PropertyGroup>
+
+ <!-- Unless specified on the commandline, set the default package output to the target/ dir. -->
+ <PropertyGroup>
+ <MlosPkgOutputBaseDir>$(BaseDir)/target/pkg</MlosPkgOutputBaseDir>
+ <MlosPkgOutputDir>$(MlosPkgOutputBaseDir)/$(Configuration)</MlosPkgOutputDir>
+ <PackageOutputPath>$(MlosPkgOutputDir)/</PackageOutputPath>
+ </PropertyGroup>
+
+ <!-- Also create a .snupkg along side the package for eventual publishing of symbols to nuget.org -->
+ <PropertyGroup>
+ <IncludeSymbols>true</IncludeSymbols>
+ <!--
+ When this property is omitted a separate .symbols.nupkg file is created
+ that includes both the .dll and .pdb.
+ -->
+ <SymbolPackageFormat>snupkg</SymbolPackageFormat>
+ </PropertyGroup>
+
+ <!-- Include the .pdb symbols file in the standard package to make debugging easier. -->
+ <PropertyGroup Condition="'$(Configuration)' == 'Debug'">
+ <AllowedOutputExtensionsInPackageBuildOutputFolder>$(AllowedOutputExtensionsInPackageBuildOutputFolder);.pdb</AllowedOutputExtensionsInPackageBuildOutputFolder>
+ </PropertyGroup>
+</Project>

build/Mlos.NetCore.props

@@ -30,6 +30,7 @@
  <Optimize>true</Optimize>
  </ProjectConfiguration>
  </ItemGroup>
+ <Import Project="$(BaseDir)/build/Mlos.NetCore.Package.props" />
  <ItemGroup>
  <None Remove="Makefile" />
  <None Remove="CMakeLists.txt" />

build/Mlos.SettingsSystem.CodeGen.targets

@@ -33,7 +33,6 @@
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_callbacks.h" />
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_dispatch.h" />
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_base.h" />
- <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_python.h" />
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_base.cs" />
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_deserialize.cs" />
  <MlosCodeGenOutput Include="$(SettingsSystemCodeGenAbsoluteOutputPath)\$(SettingsSystemCodeGenOutputBasename)_dispatch.cs" />
@@ -49,7 +48,15 @@
  so that they always run and the CodeGen process itself avoids overwriting files if their output hasn't changed.
  -->
  <!-- Codegen using internal compiler -->
- <Target Name="SettingsSystemRegistryDefCodeGen" AfterTargets="AfterResolveReferences" Outputs="@(MlosCodeGenOutput)" Condition="'@(SettingsRegistryDef->Count())' != 0">
+ <!--
+ Make sure that codegen is run whenever compile is.
+ Note: We don't use AfterTargets="AfterResolveReferences" since Pack targets also use it Restore dependencies,
+ but Pack may not actually need to re-build.
+ -->
+ <PropertyGroup>
+ <BuildDependsOn>$(BuildDependsOn);SettingsSystemRegistryDefCodeGen</BuildDependsOn>
+ </PropertyGroup>
+ <Target Name="SettingsSystemRegistryDefCodeGen" BeforeTargets="BeforeCompile;_GenerateCompileInputs" Outputs="@(MlosCodeGenOutput)" Condition="'@(SettingsRegistryDef->Count())' != 0">
  <PropertyGroup>
  <SettingsSystemCodeGenInputArgs>@(SettingsRegistryDef -> '--input-cs=&quot;%(FullPath)&quot;', ' ')</SettingsSystemCodeGenInputArgs>
  </PropertyGroup>

build/Mlos.TestRun.targets

@@ -3,17 +3,5 @@
  <!-- Include binplace properties. -->
  <Import Project="$(BaseDir)\build\Mlos.Binplace.props" />
  <!-- A basic .proj file to include for test proj files to run properly. -->
- <Target Name="Build" BeforeTargets="Test">
- <MSBuild Projects="@(ProjectReference)" Targets="Build" RunEachTargetSeparately="true" Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
- </Target>
- <Target Name="Clean">
- <MSBuild Projects="@(ProjectReference)" Targets="Clean" RunEachTargetSeparately="true" Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
- </Target>
- <Target Name="Restore">
- <MSBuild Projects="@(ProjectReference)" Targets="Restore" RunEachTargetSeparately="true" Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
- </Target>
- <Target Name="VerifyAlteredTargetsUsed">
- <MSBuild Projects="@(ProjectReference)" Targets="VerifyAlteredTargetsUsed" RunEachTargetSeparately="true" Condition="'$(QBUILD)' != '1' And '$(BuildProjectReferences)' != 'false'" />
- </Target>
- <Import Project="$(BaseDir)\build\Mlos.Common.targets" />
+ <Import Project="$(BaseDir)\build\Mlos.MetaProj.targets" />
 </Project>

build/versions/Mlos.NetCore.Version.props

@@ -15,4 +15,36 @@
  <Link>Properties\PipelineAssemblyInfo.cs</Link>
  </Compile>
  </ItemGroup>
+
+ <!-- If we're not using the pipeline version, then use the git version by default. -->
+ <PropertyGroup>
+ <PackageUsesGitVersionInfo Condition="'$(UsePipelineVersion)' == 'true'">false</PackageUsesGitVersionInfo>
+ <PackageUsesGitVersionInfo Condition="'$(PackageUsesGitVersionInfo)' == '' And '$(BUILD_COREXT)' != '1' And '$(CdpxBuildPipeline)' != 'true' And '$(HasAdoPipeline)' != 'true'">true</PackageUsesGitVersionInfo>
+ <PackageUsesGitVersionInfo Condition="'$(PackageUsesGitVersionInfo)' != 'true'">false</PackageUsesGitVersionInfo>
+ </PropertyGroup>
+ <!--
+ When packaging, we need to be able to reference consistent and unique versions.
+ The easiest way to do that is to let GitVersionTask update them in the assemblies for us.
+ Note: We only do this when not using the pipeline version.
+ -->
+ <PropertyGroup>
+ <DisableGitVersionTask>true</DisableGitVersionTask>
+ <DisableGitVersionTask Condition="'$(PackageUsesGitVersionInfo)' == 'true'">false</DisableGitVersionTask>
+ <GitVersion_NoFetchEnabled>true</GitVersion_NoFetchEnabled>
+ <WriteVersionInfoToBuildLog>false</WriteVersionInfoToBuildLog>
+ <!--
+ <GetVersion>$(PackageUsesGitVersionInfo)</GetVersion>
+ <UpdateAssemblyInfo>$(PackageUsesGitVersionInfo)</UpdateAssemblyInfo>
+ <UpdateVersionProperties>$(PackageUsesGitVersionInfo)</UpdateVersionProperties>
+ <UseFullSemVerForNuGet>$(PackageUsesGitVersionInfo)</UseFullSemVerForNuGet>
+ <GenerateGitVersionInformation>$(PackageUsesGitVersionInfo)</GenerateGitVersionInformation>
+ -->
+ </PropertyGroup>
+ <!-- Add a build tool for discovering version info from the repository. -->
+ <ItemGroup Condition="'$(PackageUsesGitVersionInfo)' == 'true' And '$(DisableGitVersionTask)' != 'true'">
+ <PackageReference Include="GitVersionTask" Version="5.5.0">
+ <!-- Don't include its assets in any build output or dependency references. -->
+ <PrivateAssets>All</PrivateAssets>
+ </PackageReference>
+ </ItemGroup>
 </Project>