Merge pull request #774 from dolittle/sparse-eventlog

Support sparse event logs
dolittle · Sep 18, 2024 · 1197809 · 1197809
2 parents 5f2f838 + 6a88d9b
commit 1197809
Show file tree

Hide file tree

Showing 68 changed files with 588 additions and 223 deletions.
diff --git a/Directory.Packages.props b/Directory.Packages.props
@@ -0,0 +1,76 @@
+<Project>
+  <!--    <Import Project="versions.props"/>-->
+  <PropertyGroup>
+    <ManagePackageVersionsCentrally>true</ManagePackageVersionsCentrally>
+  </PropertyGroup>
+  <PropertyGroup>
+    <ContractsVersion>7.8.0</ContractsVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageVersion Include="Dolittle.Contracts" Version="$(ContractsVersion)" />
+    <PackageVersion Include="Autofac" Version="8.1.0" />
+    <PackageVersion Include="Autofac.Extensions.DependencyInjection" Version="10.0.0" />
+    <PackageVersion Include="BenchmarkDotNet" Version="0.14.0" />
+    <PackageVersion Include="ConsoleTables" Version="2.6.1" />
+    <PackageVersion Include="coverlet.collector" Version="6.0.2" />
+    <PackageVersion Include="Docker.DotNet" Version="3.125.15" />
+    <PackageVersion Include="Grpc.AspNetCore" Version="2.65.0" />
+    <PackageVersion Include="Grpc.AspNetCore.Web" Version="2.65.0" />
+    <PackageVersion Include="Grpc.AspNetCore.Server.Reflection" Version="2.65.0" />
+    <PackageVersion Include="Grpc.AspNetCore.HealthChecks" Version="2.65.0" />
+    <PackageVersion Include="Grpc.Core.Testing" Version="2.46.6" />
+    <PackageVersion Include="Grpc.Tools" Version="2.66.0">
+      <PrivateAssets>all</PrivateAssets>
+      <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
+    </PackageVersion>
+    <PackageVersion Include="Grpc.Net.Client" Version="2.65.0" />
+    <PackageVersion Include="Machine.Specifications" Version="1.1.2" />
+    <PackageVersion Include="Machine.Specifications.Should" Version="1.0.0" />
+    <PackageVersion Include="Machine.Specifications.Runner.VisualStudio" Version="2.10.2" />
+    <PackageVersion Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
+    <PackageVersion Include="Microsoft.NET.Test.Sdk" Version="17.11.1" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Configuration.Json" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.FileProviders.Abstractions" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.FileProviders.Physical" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Hosting" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Hosting.Abstractions" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.1" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Configuration" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Logging.Console" Version="8.0.0" />
+    <PackageVersion Include="Microsoft.Extensions.Options" Version="8.0.2" />
+    <PackageVersion Include="MongoDB.Driver" Version="2.28.0" />
+    <PackageVersion Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="1.5.0" />
+    <PackageVersion Include="Moq" Version="4.18.4" />
+    <PackageVersion Include="FluentAssertions" Version="6.12.1" />
+    <PackageVersion Include="NetEscapades.Configuration.Yaml" Version="3.1.0" />
+    <PackageVersion Include="Newtonsoft.Json" Version="13.0.3" />
+    <PackageVersion Include="Nito.AsyncEx" Version="5.1.2" />
+    <PackageVersion Include="Nito.AsyncEx.Tasks" Version="5.1.2" />
+    <PackageVersion Include="Polly" Version="8.4.1" />
+    <PackageVersion Include="prometheus-net" Version="8.2.1" />
+    <PackageVersion Include="prometheus-net.AspNetCore" Version="8.2.1" />
+    <PackageVersion Include="prometheus-net.DotNetRuntime" Version="4.4.0" />
+    <PackageVersion Include="Proto.Actor" Version="1.7.1-alpha.0.1" />
+    <PackageVersion Include="Proto.Remote" Version="1.7.1-alpha.0.1" />
+    <PackageVersion Include="Proto.OpenTelemetry" Version="1.7.1-alpha.0.1" />
+    <PackageVersion Include="Proto.Cluster" Version="1.7.1-alpha.0.1" />
+    <PackageVersion Include="Proto.Cluster.CodeGen" Version="1.7.1-alpha.0.1" />
+    <PackageVersion Include="Google.Protobuf" Version="3.28.1" />
+    <PackageVersion Include="Swashbuckle.AspNetCore" Version="6.7.3" />
+    <PackageVersion Include="System.Linq.Async" Version="6.0.1" />
+    <PackageVersion Include="System.Security.Claims" Version="4.3.0" />
+    <PackageVersion Include="OpenTelemetry.Api" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.9.0-beta.1" />
+    <PackageVersion Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
+    <PackageVersion Include="Testcontainers.MongoDb" Version="3.10.0" />
+    <PackageVersion Include="xunit" Version="2.9.0" />
+    <PackageVersion Include="xunit.runner.visualstudio" Version="2.8.2" />
+  </ItemGroup>
+</Project>
diff --git a/Docker/Development/Dockerfile b/Docker/Development/Dockerfile
@@ -6,6 +6,7 @@ ARG TARGETARCH
 
 COPY default.props /app/
 COPY versions.props /app/
+COPY Directory.Packages.props /app/
 COPY Runtime.sln /app/
 COPY Source /app/Source/
 COPY Specifications /app/Specifications/

diff --git a/Docker/Production/Dockerfile b/Docker/Production/Dockerfile
@@ -6,6 +6,7 @@ ARG TARGETARCH
 
 COPY default.props /app/
 COPY versions.props /app/
+COPY Directory.Packages.props /app/
 COPY Runtime.sln /app/
 COPY Source /app/Source/
 COPY Specifications /app/Specifications/

diff --git a/Integration/Benchmarks/Benchmarks.csproj b/Integration/Benchmarks/Benchmarks.csproj
@@ -13,6 +13,6 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="BenchmarkDotNet" Version="0.14.0" />
+        <PackageReference Include="BenchmarkDotNet"/>
     </ItemGroup>
 </Project>
diff --git a/Integration/Shared/Shared.csproj b/Integration/Shared/Shared.csproj
@@ -11,7 +11,7 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="MongoDB.Driver" Version="$(MongoDBDriverVersion)" />
-        <PackageReference Include="Moq" Version="$(MoqVersion)" />
+        <PackageReference Include="MongoDB.Driver"/>
+        <PackageReference Include="Moq"/>
     </ItemGroup>
 </Project>
diff --git a/Integration/Tests/Services/Services.csproj b/Integration/Tests/Services/Services.csproj
@@ -11,9 +11,9 @@
         <ProjectReference Include="..\..\Shared\Shared.csproj" />
     </ItemGroup>
     <ItemGroup>
-        <PackageReference Include="Google.Protobuf" Version="$(ProtobufVersion)" />
-        <PackageReference Include="Dolittle.Contracts" Version="$(ContractsVersion)" />
-        <PackageReference Include="Grpc.Tools" Version="$(GrpcToolsVersion)">
+        <PackageReference Include="Google.Protobuf"/>
+        <PackageReference Include="Dolittle.Contracts"/>
+        <PackageReference Include="Grpc.Tools">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>

diff --git a/Runtime.sln b/Runtime.sln
@@ -115,6 +115,7 @@ Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".meta", ".meta", "{CAF1D781
 		specs.props = specs.props
 		tests.props = tests.props
 		README.md = README.md
+		Directory.Packages.props = Directory.Packages.props
 	EndProjectSection
 EndProject
 Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = ".github", ".github", "{89A4E435-785F-4CDF-91B4-A14EAB91084B}"

diff --git a/Source/Actors/Actors.csproj b/Source/Actors/Actors.csproj
@@ -6,16 +6,15 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Grpc.Tools" Version="$(GrpcToolsVersion)">
+        <PackageReference Include="Grpc.Tools">
             <PrivateAssets>all</PrivateAssets>
             <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
         </PackageReference>
-        <PackageReference Include="Proto.Actor" Version="$(ProtoActorVersion)" />
-        <PackageReference Include="Proto.Remote" Version="$(ProtoActorVersion)" />
-        <PackageReference Include="Proto.OpenTelemetry" Version="$(ProtoActorVersion)" />
-        <PackageReference Include="Proto.Cluster" Version="$(ProtoActorVersion)" />
-        <PackageReference Include="Proto.Cluster.CodeGen" Version="$(ProtoActorVersion)" />
-        <PackageReference Include="Proto.Cluster.TestProvider" Version="$(ProtoActorVersion)" />
+        <PackageReference Include="Proto.Actor"/>
+        <PackageReference Include="Proto.Remote"/>
+        <PackageReference Include="Proto.OpenTelemetry"/>
+        <PackageReference Include="Proto.Cluster"/>
+        <PackageReference Include="Proto.Cluster.CodeGen"/>
     </ItemGroup>
 
     <ItemGroup>

diff --git a/Source/CLI/CLI.csproj b/Source/CLI/CLI.csproj
@@ -21,13 +21,13 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="ConsoleTables" Version="$(ConsoleTablesVersion)" />
-        <PackageReference Include="Docker.DotNet" Version="3.125.15" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="McMaster.Extensions.CommandLineUtils" Version="4.1.1" />
-        <PackageReference Include="Microsoft.Extensions.Logging.Console" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" Version="$(MicrosoftExtensionsVersion)" />
+        <PackageReference Include="ConsoleTables"/>
+        <PackageReference Include="Docker.DotNet"/>
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection" />
+        <PackageReference Include="McMaster.Extensions.CommandLineUtils"/>
+        <PackageReference Include="Microsoft.Extensions.Logging.Console" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" />
     </ItemGroup>
 
 </Project>
diff --git a/Source/Client/Client.csproj b/Source/Client/Client.csproj
@@ -7,8 +7,8 @@
     </PropertyGroup>
 
     <ItemGroup>
-        <PackageReference Include="Dolittle.Contracts" Version="$(ContractsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsVersion)" />
+        <PackageReference Include="Dolittle.Contracts"/>
+        <PackageReference Include="Microsoft.Extensions.Logging" />
     </ItemGroup>
 
     <ItemGroup>

diff --git a/Source/Configuration.Management/Configuration.Management.csproj b/Source/Configuration.Management/Configuration.Management.csproj
@@ -12,12 +12,12 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Configuration" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="NetEscapades.Configuration.Yaml" Version="$(NetEscapadesYaml)" />
-        <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftVersion)" />
+        <PackageReference Include="Microsoft.Extensions.Configuration" />
+        <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" />
+        <PackageReference Include="NetEscapades.Configuration.Yaml"/>
+        <PackageReference Include="Newtonsoft.Json"/>
     </ItemGroup>
 </Project>
diff --git a/Source/Configuration/Configuration.csproj b/Source/Configuration/Configuration.csproj
@@ -12,13 +12,13 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Autofac" Version="$(AutofacVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Configuration" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Configuration.Json" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="NetEscapades.Configuration.Yaml" Version="$(NetEscapadesYaml)" />
-        <PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftVersion)" />
+        <PackageReference Include="Autofac" />
+        <PackageReference Include="Microsoft.Extensions.Configuration" />
+        <PackageReference Include="Microsoft.Extensions.Configuration.Json" />
+        <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.FileProviders.Physical" />
+        <PackageReference Include="NetEscapades.Configuration.Yaml"/>
+        <PackageReference Include="Newtonsoft.Json"/>
     </ItemGroup>
 </Project>
diff --git a/Source/DependencyInversion/DependencyInversion.csproj b/Source/DependencyInversion/DependencyInversion.csproj
@@ -11,11 +11,11 @@
   </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Autofac" Version="$(AutofacVersion)" />
-        <PackageReference Include="Autofac.Extensions.DependencyInjection" Version="$(AutofacExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Options" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="$(MicrosoftExtensionsVersion)" />
+        <PackageReference Include="Autofac" />
+        <PackageReference Include="Autofac.Extensions.DependencyInjection"/>
+        <PackageReference Include="Microsoft.Extensions.Hosting.Abstractions" />
+        <PackageReference Include="Microsoft.Extensions.Options" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" />
   </ItemGroup>
 
 </Project>
diff --git a/Source/Diagnostics/Diagnostics.csproj b/Source/Diagnostics/Diagnostics.csproj
@@ -11,16 +11,16 @@
     </ItemGroup>
 
     <ItemGroup>
-        <PackageReference Include="Microsoft.Extensions.Logging" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="Microsoft.Extensions.Logging.Configuration" Version="$(MicrosoftExtensionsVersion)" />
-        <PackageReference Include="MongoDB.Driver" Version="$(MongoDBDriverVersion)" />
-        <PackageReference Include="MongoDB.Driver.Core.Extensions.DiagnosticSources" Version="$(MongoDBDiagnosticSourcesVersion)" />
-        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol" Version="1.9.0" />
-        <PackageReference Include="OpenTelemetry.Extensions.Hosting" Version="1.9.0" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore" Version="1.9.0" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.GrpcNetClient" Version="1.9.0-beta.1" />
-        <PackageReference Include="OpenTelemetry.Instrumentation.Http" Version="1.9.0" />
-        <PackageReference Include="Proto.OpenTelemetry" Version="$(ProtoActorVersion)" />
+        <PackageReference Include="Microsoft.Extensions.Logging" />
+        <PackageReference Include="Microsoft.Extensions.Logging.Configuration" />
+        <PackageReference Include="MongoDB.Driver"/>
+        <PackageReference Include="MongoDB.Driver.Core.Extensions.DiagnosticSources"/>
+        <PackageReference Include="OpenTelemetry.Exporter.OpenTelemetryProtocol"/>
+        <PackageReference Include="OpenTelemetry.Extensions.Hosting"/>
+        <PackageReference Include="OpenTelemetry.Instrumentation.AspNetCore"/>
+        <PackageReference Include="OpenTelemetry.Instrumentation.GrpcNetClient"/>
+        <PackageReference Include="OpenTelemetry.Instrumentation.Http"/>
+        <PackageReference Include="Proto.OpenTelemetry"/>
     </ItemGroup>
 
 </Project>
diff --git a/Source/EventHorizon/Consumer/Connections/EventHorizonConnection.cs b/Source/EventHorizon/Consumer/Connections/EventHorizonConnection.cs
@@ -4,14 +4,14 @@
 using System;
 using System.Diagnostics;
 using System.Threading;
+using System.Threading.Channels;
 using System.Threading.Tasks;
 using Dolittle.Runtime.EventHorizon.Contracts;
 using Dolittle.Runtime.EventHorizon.UnBreaking;
 using Dolittle.Runtime.Events.Store.Streams;
 using Dolittle.Runtime.Protobuf;
 using Dolittle.Runtime.Services.Clients;
 using Microsoft.Extensions.Logging;
-using Nito.AsyncEx;
 using ExecutionContext = Dolittle.Runtime.Execution.ExecutionContext;
 
 namespace Dolittle.Runtime.EventHorizon.Consumer.Connections;
@@ -92,7 +92,7 @@ public async Task<SubscriptionResponse> Connect(
 
     /// <inheritdoc/>
     public Task StartReceivingEventsInto(
-        AsyncProducerConsumerQueue<StreamEvent> connectionToStreamProcessorQueue,
+        Channel<StreamEvent> connectionToStreamProcessorQueue,
         CancellationToken cancellationToken)
     {
         return _reverseCallClient.Handle(
@@ -105,7 +105,7 @@ public Task StartReceivingEventsInto(
     }
 
     async Task<ConsumerResponse> HandleEventFromEventHorizon(
-        AsyncProducerConsumerQueue<StreamEvent> connectionToStreamProcessorQueue,
+        Channel<StreamEvent> connectionToStreamProcessorQueue,
         EventHorizonEvent @event,
         CancellationToken cancellationToken)
     {
@@ -120,7 +120,7 @@ async Task<ConsumerResponse> HandleEventFromEventHorizon(
                 PartitionId.None,
                 false);
 
-            await connectionToStreamProcessorQueue.EnqueueAsync(streamEvent, cancellationToken).ConfigureAwait(false);
+            await connectionToStreamProcessorQueue.Writer.WriteAsync(streamEvent, cancellationToken).ConfigureAwait(false);
             return CreateSuccessfulResponse();
         }
         catch (Exception exception)

diff --git a/Source/EventHorizon/Consumer/Connections/IEventHorizonConnection.cs b/Source/EventHorizon/Consumer/Connections/IEventHorizonConnection.cs
@@ -3,9 +3,9 @@
 
 using System;
 using System.Threading;
+using System.Threading.Channels;
 using System.Threading.Tasks;
 using Dolittle.Runtime.Events.Store.Streams;
-using Nito.AsyncEx;
 
 namespace Dolittle.Runtime.EventHorizon.Consumer.Connections;
 
@@ -22,7 +22,7 @@ public interface IEventHorizonConnection : IDisposable
     /// <param name="cancellationToken">A cancellation token that can be used to cancel the connection attempt.</param>
     /// <returns>
     /// A task that, when resolved returns the <see cref="SubscriptionResponse"/> from the connection to the producer Runtime.
-    /// If <see cref="SubscriptionResponse.Success"/> is true, the connection is started and <see cref="StartRecevingEventsInto(AsyncProducerConsumerQueue{StreamEvent}, CancellationToken)"/> should be called.
+    /// If <see cref="SubscriptionResponse.Success"/> is true, the connection is started and <see cref="StartRecevingEventsInto(Channel{StreamEvent}, CancellationToken)"/> should be called.
     /// Else, the connection failed and should it should not be used.
     /// </returns>
     Task<SubscriptionResponse> Connect(
@@ -37,6 +37,6 @@ Task<SubscriptionResponse> Connect(
     /// <param name="cancellationToken">A cancellation token that can be used to close the connection.</param>
     /// <returns>A task that represents the asynchronous operation.</returns>
     Task StartReceivingEventsInto(
-        AsyncProducerConsumerQueue<StreamEvent> connectionToStreamProcessorQueue,
+        Channel<StreamEvent> connectionToStreamProcessorQueue,
         CancellationToken cancellationToken);
 }