Merge branch 'main' into cpp_notebook

build/Mlos.NetCore.Grpc.props

@@ -4,34 +4,41 @@
  <!-- Protobuf gen output path. -->
  <GrpcOutputDir>$(BaseDir)\out\Grpc.out\$(GrpcServices)\$(BuildType)</GrpcOutputDir>
  <GrpcVersion>2.29.0</GrpcVersion>
+ <ProtobufVersion>3.12.0</ProtobufVersion>
  </PropertyGroup>
  <ItemGroup>
- <PackageReference Include="Google.Protobuf" Version="3.12.0" GeneratePathProperty="true" />
- <BinplaceFile Include="$(PkgGoogle_Protobuf)/lib/netstandard2.0/Google.Protobuf.dll" />
+ <PackageReference Include="Google.Protobuf" Version="$(ProtobufVersion)" GeneratePathProperty="true" />
 
  <PackageReference Include="Grpc.Core" Version="$(GrpcVersion)" GeneratePathProperty="true" />
  <PackageReference Include="Grpc.Core.Api" Version="$(GrpcVersion)" GeneratePathProperty="true" />
- <BinplaceFile Include="$(PkgGrpc_Core)/lib/netstandard2.0/Grpc.Core.dll" />
- <BinplaceFile Include="$(PkgGrpc_Core_Api)/lib/netstandard2.0/Grpc.Core.Api.dll" />
 
  <PackageReference Include="Grpc.Net.Common" Version="$(GrpcVersion)" GeneratePathProperty="true" />
- <BinplaceFile Include="$(PkgGrpc_Net_Common)/lib/netstandard2.1/Grpc.Net.Common.dll" />
 
  <PackageReference Include="Grpc.Tools" Version="$(GrpcVersion)" GeneratePathProperty="true" />
  </ItemGroup>
 
- <ItemGroup Condition="'$(GrpcServices)' == 'Server'">
+ <ItemGroup Condition="'$(GrpcServices)' == 'Server' Or '$(GrpcServices)' == 'Both'">
  <PackageReference Include="Grpc.AspNetCore" Version="$(GrpcVersion)" GeneratePathProperty="true" />
  <PackageReference Include="Grpc.AspNetCore.Server" Version="$(GrpcVersion)" GeneratePathProperty="true" />
  <PackageReference Include="Grpc.AspNetCore.Server.ClientFactory" Version="$(GrpcVersion)" GeneratePathProperty="true" />
- <BinplaceFile Include="$(PkgGrpc_AspNetCore_Server)/lib/netcoreapp3.0/Grpc.AspNetCore.Server.dll" />
- <BinplaceFile Include="$(PkgGrpc_AspNetCore_Server_ClientFactory)/lib/netcoreapp3.0/Grpc.AspNetCore.Server.ClientFactory.dll" />
  </ItemGroup>
 
- <ItemGroup Condition="'$(GrpcServices)' == 'Client'">
+ <ItemGroup Condition="'$(GrpcServices)' == 'Client' Or '$(GrpcServices)' == 'Both'">
  <PackageReference Include="Grpc.Net.Client" Version="$(GrpcVersion)" GeneratePathProperty="true" />
  <PackageReference Include="Grpc.Net.ClientFactory" Version="$(GrpcVersion)" GeneratePathProperty="true" />
- <BinplaceFile Include="$(PkgGrpc_Net_Client)/lib/netstandard2.1/Grpc.Net.Client.dll" />
- <BinplaceFile Include="$(PkgGrpc_Net_ClientFactory)/lib/netstandard2.1/Grpc.Net.ClientFactory.dll" />
+ </ItemGroup>
+
+ <!--
+ Add a reference to a separate project that consolidates the binplace rules
+ for those packages in order to avoid duplicate copy operations which upsets
+ some build systems.
+ -->
+ <PropertyGroup>
+ <MlosGrpcBinplaceProj>$([System.IO.Path]::GetFullPath('$(BaseDir)\source\Mlos.Grpc.Binplace\Mlos.Grpc.Binplace.csproj'))</MlosGrpcBinplaceProj>
+ <NormalizedProjectPath>$([System.IO.Path]::GetFullPath('$(MSBuildProjectFullPath)'))</NormalizedProjectPath>
+ </PropertyGroup>
+ <!-- Prevent circular dependencies -->
+ <ItemGroup Condition="'$(NormalizedProjectPath)' != '$(MlosGrpcBinplaceProj)'">
+ <ProjectReference Include="$(MlosGrpcBinplaceProj)" />
  </ItemGroup>
 </Project>

source/Examples/SmartCache/Main.cpp

@@ -43,7 +43,7 @@ using namespace SmartCache;
 // HRESULTs are an error code encoding mechanism typically used in Windows environments.
 // See Also: https://en.wikipedia.org/wiki/HRESULT
 //
-void CheckHR(HRESULT hr)
+void ThrowIfFail(HRESULT hr)
 {
  if (FAILED(hr))
  {
@@ -74,7 +74,7 @@ main(
  //
  Mlos::Core::InterProcessMlosContextInitializer mlosContextInitializer;
  HRESULT hr = mlosContextInitializer.Initialize();
- CheckHR(hr);
+ ThrowIfFail(hr);
 
  Mlos::Core::InterProcessMlosContext mlosContext(std::move(mlosContextInitializer));
 
@@ -124,7 +124,7 @@ main(
  hr = mlosContext.RegisterSettingsAssembly(
  "SmartCache.SettingsRegistry.dll",
  SmartCache::ObjectDeserializationHandler::DispatchTableBaseIndex());
- CheckHR(hr);
+ ThrowIfFail(hr);
 
  // Create a component configuration object.
  // This will be stored in a shared memory region below for use by both the
@@ -145,7 +145,7 @@ main(
  // config for this component and if not creates it.
  //
  hr = mlosContext.RegisterComponentConfig(config);
- CheckHR(hr);
+ ThrowIfFail(hr);
 
  // Create an instance of our SmartCache component to tune.
  //

source/Examples/SmartCache/SmartCache.sln

@@ -6,6 +6,7 @@ MinimumVisualStudioVersion = 10.0.40219.1
 Project("{8BC9CEB8-8B4A-11D0-8D11-00A0C91BC942}") = "SmartCache", "SmartCache.vcxproj", "{0CE12F0A-E7B5-4E52-B6DF-E9BA1F525030}"
  ProjectSection(ProjectDependencies) = postProject
  {E4407270-6E64-4E87-A4B7-3B932121BC81} = {E4407270-6E64-4E87-A4B7-3B932121BC81}
+ {5DB4C179-4A97-40F9-8155-8C44B2BE416D} = {5DB4C179-4A97-40F9-8155-8C44B2BE416D}
  EndProjectSection
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "SmartCache.SettingsRegistry", "SmartCache.SettingsRegistry\SmartCache.SettingsRegistry.csproj", "{E4407270-6E64-4E87-A4B7-3B932121BC81}"

source/Mlos.Agent.Server/MlosAgentServer.cs

@@ -104,10 +104,15 @@ public static void Main(string[] args)
  MlosContext.OptimizerFactory = new MlosOptimizer.BayesianOptimizerFactory(optimizerAddressUri);
  }
 
- // Create (or open) the circular buffer shared memory before running the target process.
+ // In the active learning mode, create a new shared memory map before running the target process.
+ // On Linux, we unlink existing shared memory map, if they exist.
+ // If the agent is not in the active learning mode, create new or open existing to communicate with the target process.
  //
+ using MlosContext mlosContext = (executableFilePath != null)
+ ? InterProcessMlosContext.Create()
+ : InterProcessMlosContext.CreateOrOpen();
  using var mainAgent = new MainAgent();
- mainAgent.InitializeSharedChannel();
+ mainAgent.InitializeSharedChannel(mlosContext);
 
  // Active learning mode.
  //
@@ -153,7 +158,9 @@ public static void Main(string[] args)
  Console.WriteLine("Starting Mlos.Agent");
  Task mlosAgentTask = Task.Factory.StartNew(
  () => mainAgent.RunAgent(),
- TaskCreationOptions.LongRunning);
+ CancellationToken.None,
+ TaskCreationOptions.LongRunning,
+ TaskScheduler.Current);
 
  Task waitForTargetProcessTask = Task.Factory.StartNew(
  () =>
@@ -165,13 +172,15 @@ public static void Main(string[] args)
  mainAgent.UninitializeSharedChannel();
  }
  },
- TaskCreationOptions.LongRunning);
+ CancellationToken.None,
+ TaskCreationOptions.LongRunning,
+ TaskScheduler.Current);
 
  Console.WriteLine("Waiting for Mlos.Agent to exit");
 
  while (true)
  {
- Task.WaitAny(new[] { mlosAgentTask, waitForTargetProcessTask });
+ Task.WaitAny(mlosAgentTask, waitForTargetProcessTask);
 
  if (mlosAgentTask.IsFaulted && targetProcessManager != null && !waitForTargetProcessTask.IsCompleted)
  {

source/Mlos.Agent.Server/TargetProcessManager.cs

@@ -7,9 +7,7 @@
 // -----------------------------------------------------------------------
 
 using System;
-using System.Collections.Generic;
 using System.Diagnostics;
-using System.Text;
 
 namespace Mlos.Agent.Server
 {
@@ -22,9 +20,10 @@ namespace Mlos.Agent.Server
  internal class TargetProcessManager : IDisposable
  {
  private readonly string executableFilePath;
+
  private Process targetProcess;
 
- private bool disposed = false;
+ private bool isDisposed;
 
  public TargetProcessManager(string executableFilePath)
  {
@@ -40,18 +39,15 @@ public void Dispose()
 
  private void Dispose(bool disposing)
  {
- if (disposed)
+ if (isDisposed || !disposing)
  {
  return;
  }
 
- if (disposing)
- {
- targetProcess?.Dispose();
- targetProcess = null;
- }
+ targetProcess?.Dispose();
+ targetProcess = null;
 
- disposed = true;
+ isDisposed = true;
  }
 
  public void StartTargetProcess()

source/Mlos.Agent/MainAgent.cs

@@ -13,7 +13,6 @@
 using System.Runtime.InteropServices;
 
 using Mlos.Core;
-using Proxy.Mlos.Core;
 
 using MlosProxy = Proxy.Mlos.Core;
 using MlosProxyInternal = Proxy.Mlos.Core.Internal;
@@ -28,18 +27,6 @@ namespace Mlos.Agent
  /// </remarks>
  public class MainAgent : IDisposable
  {
- /// <remarks>
- /// Shared memory mapping name must start with "Host_" prefix, to be accessible from certain applications.
- /// TODO: Make these config regions configurable to support multiple processes.
- /// </remarks>
- private const string GlobalMemoryMapName = "Host_Mlos.GlobalMemory";
- private const string ControlChannelMemoryMapName = "Host_Mlos.ControlChannel";
- private const string FeedbackChannelMemoryMapName = "Host_Mlos.FeedbackChannel";
- private const string ControlChannelSemaphoreName = @"Global\ControlChannel_Event"; //// FIXME: Use non-backslashes for Linux environments.
- private const string FeedbackChannelSemaphoreName = @"Global\FeedbackChannel_Event";
- private const string SharedConfigMemoryMapName = "Host_Mlos.Config.SharedMemory";
- private const int SharedMemorySize = 65536;
-
  private readonly SettingsAssemblyManager settingsAssemblyManager = new SettingsAssemblyManager();
 
  private readonly Dictionary<uint, SharedMemoryMapView> memoryRegions = new Dictionary<uint, SharedMemoryMapView>();
@@ -53,64 +40,27 @@ public class MainAgent : IDisposable
  private bool isDisposed;
 
  #region Shared objects
- private SharedMemoryRegionView<MlosProxyInternal.GlobalMemoryRegion> globalMemoryRegionView;
-
- private SharedMemoryMapView controlChannelMemoryMapView;
- private SharedMemoryMapView feedbackChannelMemoryMapView;
 
- private NamedEvent controlChannelNamedEvent;
- private NamedEvent feedbackChannelNamedEvent;
+ private MlosContext mlosContext;
 
- private SharedMemoryRegionView<MlosProxyInternal.SharedConfigMemoryRegion> sharedConfigMemoryMapView;
  #endregion
 
  #region Mlos.Agent setup
 
  /// <summary>
  /// Initialize shared channel.
  /// </summary>
- public void InitializeSharedChannel()
+ /// <param name="mlosContext">Mlos context instance.</param>
+ public void InitializeSharedChannel(MlosContext mlosContext)
  {
- // Create or open the memory mapped files.
- //
- Console.WriteLine("Create or open memory mapped files");
- globalMemoryRegionView = SharedMemoryRegionView.CreateOrOpen<MlosProxyInternal.GlobalMemoryRegion>(GlobalMemoryMapName, SharedMemorySize);
- controlChannelMemoryMapView = SharedMemoryMapView.CreateOrOpen(ControlChannelMemoryMapName, SharedMemorySize);
- feedbackChannelMemoryMapView = SharedMemoryMapView.CreateOrOpen(FeedbackChannelMemoryMapName, SharedMemorySize);
- sharedConfigMemoryMapView = SharedMemoryRegionView.CreateOrOpen<MlosProxyInternal.SharedConfigMemoryRegion>(SharedConfigMemoryMapName, SharedMemorySize);
-
- // Create channel synchronization primitives.
- //
- Console.WriteLine("Create channel synchronization primitives");
- controlChannelNamedEvent = NamedEvent.CreateOrOpen(ControlChannelSemaphoreName);
- feedbackChannelNamedEvent = NamedEvent.CreateOrOpen(FeedbackChannelSemaphoreName);
-
- // Setup feedback channel.
- //
- Console.WriteLine("Setup feedback channel");
-
- MlosProxyInternal.GlobalMemoryRegion globalMemoryRegion = globalMemoryRegionView.MemoryRegion();
-
- Console.WriteLine("Setup feedback channel");
-
-
- var feedbackChannel = new SharedChannel<InterProcessSharedChannelPolicy, SharedChannelSpinPolicy>(
- buffer: feedbackChannelMemoryMapView.Buffer,
- size: (uint)feedbackChannelMemoryMapView.MemSize,
- sync: globalMemoryRegion.FeedbackChannelSynchronization)
- {
- ChannelPolicy = { NotificationEvent = feedbackChannelNamedEvent },
- };
+ this.mlosContext = mlosContext;
 
  // Set SharedConfig memory region.
  //
- Console.WriteLine("Set SharedConfig memory region.");
-
- sharedConfigManager.SetMemoryRegion(new MlosProxyInternal.SharedConfigMemoryRegion { Buffer = sharedConfigMemoryMapView.MemoryRegion().Buffer });
+ sharedConfigManager.SetMemoryRegion(new MlosProxyInternal.SharedConfigMemoryRegion { Buffer = mlosContext.SharedConfigMemoryRegion.Buffer });
 
  // Setup MlosContext.
  //
- MlosContext.FeedbackChannel = feedbackChannel;
  MlosContext.SharedConfigManager = sharedConfigManager;
 
  // Initialize callbacks.
@@ -129,7 +79,7 @@ public void InitializeSharedChannel()
  // Register assemblies from the shared config.
  // Assembly Mlos.NetCore does not have a config, as it is always registered first.
  //
- for (uint index = 1; index < globalMemoryRegion.RegisteredSettingsAssemblyCount.Load(); index++)
+ for (uint index = 1; index < mlosContext.GlobalMemoryRegion.RegisteredSettingsAssemblyCount.Load(); index++)
  {
  RegisterSettingsAssembly(assemblyIndex: index);
  }
@@ -144,8 +94,8 @@ public void UninitializeSharedChannel()
 
  // Signal named event to close any waiter threads.
  //
- controlChannelNamedEvent.Signal();
- feedbackChannelNamedEvent.Signal();
+ mlosContext.TerminateControlChannel();
+ mlosContext.TerminateFeedbackChannel();
  }
 
  /// <summary>
@@ -301,15 +251,13 @@ private void RegisterSharedConfigMemoryRegionRequestMessageCallback(MlosProxyInt
 
  /// <summary>
  /// #TODO remove, this is not required.
- /// set the terminate channel in sync object and signal.
  /// </summary>
  /// <param name="msg"></param>
- private void TerminateReaderThreadRequestMessageCallback(TerminateReaderThreadRequestMessage msg)
+ private void TerminateReaderThreadRequestMessageCallback(MlosProxy.TerminateReaderThreadRequestMessage msg)
  {
  // Terminate the channel.
  //
- MlosProxyInternal.GlobalMemoryRegion globalMemoryRegion = globalMemoryRegionView.MemoryRegion();
- ChannelSynchronization controlChannelSync = globalMemoryRegion.ControlChannelSynchronization;
+ MlosProxy.ChannelSynchronization controlChannelSync = mlosContext.GlobalMemoryRegion.ControlChannelSynchronization;
  controlChannelSync.TerminateChannel.Store(true);
  }
 
@@ -320,20 +268,9 @@ private void TerminateReaderThreadRequestMessageCallback(TerminateReaderThreadRe
  /// </summary>
  public void RunAgent()
  {
- // Create the shared memory control channel.
- //
- var globalMemoryRegion = globalMemoryRegionView.MemoryRegion();
- var controlChannel = new SharedChannel<InterProcessSharedChannelPolicy, SharedChannelSpinPolicy>(
- buffer: controlChannelMemoryMapView.Buffer,
- size: (uint)controlChannelMemoryMapView.MemSize,
- sync: globalMemoryRegion.ControlChannelSynchronization)
- {
- ChannelPolicy = { NotificationEvent = controlChannelNamedEvent },
- };
-
  // Process the messages from the control channel.
  //
- controlChannel.ProcessMessages(dispatchTable: ref globalDispatchTable);
+ MlosContext.ControlChannel.ProcessMessages(dispatchTable: ref globalDispatchTable);
  }
 
  protected virtual void Dispose(bool disposing)
@@ -343,32 +280,16 @@ protected virtual void Dispose(bool disposing)
  return;
  }
 
- // Close shared memory.
+ // Dispose MlosContext.
  //
- globalMemoryRegionView?.Dispose();
- globalMemoryRegionView = null;
-
- controlChannelMemoryMapView?.Dispose();
- controlChannelMemoryMapView = null;
-
- feedbackChannelMemoryMapView?.Dispose();
- feedbackChannelMemoryMapView = null;
-
- sharedConfigMemoryMapView?.Dispose();
- sharedConfigMemoryMapView = null;
-
- controlChannelNamedEvent?.Dispose();
- controlChannelNamedEvent = null;
-
- feedbackChannelNamedEvent?.Dispose();
- feedbackChannelNamedEvent = null;
+ mlosContext?.Dispose();
+ mlosContext = null;
 
  isDisposed = true;
  }
 
  public void Dispose()
  {
- // Do not change this code. Put cleanup code in 'Dispose(bool disposing)' method
  Dispose(disposing: true);
  GC.SuppressFinalize(this);
  }

source/Mlos.Core/CMakeLists.txt

@@ -6,6 +6,8 @@ include("${MLOS_ROOT}/build/Mlos.Cpp.cmake")
 
 add_library(${PROJECT_NAME} STATIC
  GlobalMemoryRegion.cpp
+ InternalMlosContext.cpp
+ InterProcessMlosContext.cpp
  Mlos.Core.cpp
  MlosContext.cpp
  NamedEvent.Linux.cpp