Disable twostep gossip

ProtoActor.sln

@@ -237,6 +237,14 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proto.Cluster.Dashboard", "
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Proto.Cluster.Dashboard.Host", "examples\Proto.Cluster.Dashboard.Host\Proto.Cluster.Dashboard.Host.csproj", "{2A909620-2AA9-4038-A138-D0B135DD4DC5}"
 EndProject
+Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "GossipBenchmark", "GossipBenchmark", "{3AD895C6-2E1D-40FE-9055-DD4F40E7FFFA}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node1", "benchmarks\GossipBenchmark\Node1\Node1.csproj", "{048ACA91-191B-4B37-93DF-CC603D8E1736}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node2", "benchmarks\GossipBenchmark\Node2\Node2.csproj", "{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Messages", "benchmarks\GossipBenchmark\Messages\Messages.csproj", "{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}"
+EndProject
 Global
 	GlobalSection(SolutionConfigurationPlatforms) = preSolution
 		Debug|Any CPU = Debug|Any CPU
@@ -1171,6 +1179,42 @@ Global
 		{2A909620-2AA9-4038-A138-D0B135DD4DC5}.Release|x64.Build.0 = Release|Any CPU
 		{2A909620-2AA9-4038-A138-D0B135DD4DC5}.Release|x86.ActiveCfg = Release|Any CPU
 		{2A909620-2AA9-4038-A138-D0B135DD4DC5}.Release|x86.Build.0 = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|x64.Build.0 = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Debug|x86.Build.0 = Debug|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|Any CPU.Build.0 = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|x64.ActiveCfg = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|x64.Build.0 = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|x86.ActiveCfg = Release|Any CPU
+		{048ACA91-191B-4B37-93DF-CC603D8E1736}.Release|x86.Build.0 = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|x64.Build.0 = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Debug|x86.Build.0 = Debug|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|Any CPU.Build.0 = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|x64.ActiveCfg = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|x64.Build.0 = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|x86.ActiveCfg = Release|Any CPU
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062}.Release|x86.Build.0 = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|x64.ActiveCfg = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|x64.Build.0 = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|x86.ActiveCfg = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Debug|x86.Build.0 = Debug|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|Any CPU.Build.0 = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|x64.ActiveCfg = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|x64.Build.0 = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|x86.ActiveCfg = Release|Any CPU
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110}.Release|x86.Build.0 = Release|Any CPU
 	EndGlobalSection
 	GlobalSection(SolutionProperties) = preSolution
 		HideSolutionNode = FALSE
@@ -1279,6 +1323,10 @@ Global
 		{7D13CCDD-6B93-45B8-BA88-761744601726} = {010B4228-6236-4111-B76E-90BD83780E35}
 		{58D27164-2426-4DBC-8826-A9048CC33E9D} = {010B4228-6236-4111-B76E-90BD83780E35}
 		{2A909620-2AA9-4038-A138-D0B135DD4DC5} = {59DCCC96-DDAF-469F-9E8E-9BC733285082}
+		{3AD895C6-2E1D-40FE-9055-DD4F40E7FFFA} = {0F3AB331-C042-4371-A2F0-0AFDFA13DC9F}
+		{048ACA91-191B-4B37-93DF-CC603D8E1736} = {3AD895C6-2E1D-40FE-9055-DD4F40E7FFFA}
+		{DCB884F7-18D4-4E22-A2D2-91DEC7F9B062} = {3AD895C6-2E1D-40FE-9055-DD4F40E7FFFA}
+		{2D6F8A17-00B5-41C0-BCF3-1DD733E7C110} = {3AD895C6-2E1D-40FE-9055-DD4F40E7FFFA}
 	EndGlobalSection
 	GlobalSection(ExtensibilityGlobals) = postSolution
 		SolutionGuid = {CD0D1E44-8118-4682-8793-6B20ABFA824C}

benchmarks/GossipBenchmark/GossipBenchmark.sln

@@ -0,0 +1,28 @@
+
+Microsoft Visual Studio Solution File, Format Version 12.00
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Messages", "Messages\Messages.csproj", "{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node2", "Node2\Node2.csproj", "{95685292-A5C9-48DA-871A-EB00A78A39A5}"
+EndProject
+Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "Node1", "Node1\Node1.csproj", "{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}"
+EndProject
+Global
+	GlobalSection(SolutionConfigurationPlatforms) = preSolution
+		Debug|Any CPU = Debug|Any CPU
+		Release|Any CPU = Release|Any CPU
+	EndGlobalSection
+	GlobalSection(ProjectConfigurationPlatforms) = postSolution
+		{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{A5E3B3E1-5517-4E08-9E50-50EF17215AB5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{95685292-A5C9-48DA-871A-EB00A78A39A5}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{95685292-A5C9-48DA-871A-EB00A78A39A5}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{95685292-A5C9-48DA-871A-EB00A78A39A5}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{95685292-A5C9-48DA-871A-EB00A78A39A5}.Release|Any CPU.Build.0 = Release|Any CPU
+		{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Debug|Any CPU.ActiveCfg = Debug|Any CPU
+		{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Debug|Any CPU.Build.0 = Debug|Any CPU
+		{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Release|Any CPU.ActiveCfg = Release|Any CPU
+		{B8D93ECE-0203-41C2-B845-7BBDDBA222DE}.Release|Any CPU.Build.0 = Release|Any CPU
+	EndGlobalSection
+EndGlobal

benchmarks/GossipBenchmark/Messages/Messages.csproj

@@ -0,0 +1,21 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <PropertyGroup>
+    <TargetFramework>net6.0</TargetFramework>
+  </PropertyGroup>
+  <ItemGroup>
+    <PackageReference Include="Google.Protobuf" Version="3.19.4" />
+    <PackageReference Include="Grpc.Tools" Version="2.44.0-pre2" PrivateAssets="All" />
+    <PackageReference Include="Proto.Cluster.CodeGen" Version="0.26.1-alpha.0.56" />
+  </ItemGroup>
+  <ItemGroup>
+    <Protobuf Include="Protos.proto" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProtoGrain Include="Protos.proto" />
+  </ItemGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Proto.Cluster\Proto.Cluster.csproj" />
+  </ItemGroup>
+
+</Project>

benchmarks/GossipBenchmark/Messages/Protos.proto

@@ -0,0 +1,14 @@
+syntax = "proto3";
+package HelloHelloWorld;
+option csharp_namespace = "ClusterHelloWorld.Messages";
+
+//dotnet run -p ../../../protobuf/ProtoGrainGenerator/ProtoGrainGenerator.csproj --framework net5.0 many *.proto
+
+message HelloRequest {}
+message HelloResponse {
+    string Message=1;
+}
+
+service HelloGrain {
+	rpc SayHello(HelloRequest) returns (HelloResponse) {}
+}

benchmarks/GossipBenchmark/Node1/Node1.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ServerGarbageCollection>true</ServerGarbageCollection>
+    <ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
+    <LangVersion>10</LangVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Proto.Cluster\Proto.Cluster.csproj" />
+    <ProjectReference Include="..\..\..\src\Proto.Remote\Proto.Remote.csproj" />
+    <ProjectReference Include="..\Messages\Messages.csproj" />
+  </ItemGroup>
+</Project>

benchmarks/GossipBenchmark/Node1/Program.cs

@@ -0,0 +1,64 @@
+// -----------------------------------------------------------------------
+// <copyright file="Program.cs" company="Asynkron AB">
+//      Copyright (C) 2015-2022 Asynkron AB All rights reserved
+// </copyright>
+// -----------------------------------------------------------------------
+using System;
+using System.Threading.Tasks;
+using ClusterHelloWorld.Messages;
+using Microsoft.Extensions.Logging;
+using Proto;
+using Proto.Cluster;
+using Proto.Cluster.Partition;
+using Proto.Cluster.Seed;
+using Proto.Context;
+using Proto.Remote;
+using Proto.Remote.GrpcNet;
+using static Proto.CancellationTokens;
+using ProtosReflection =ClusterHelloWorld.Messages.ProtosReflection;
+
+class Program
+{
+    private static async Task Main()
+    {
+        Proto.Log.SetLoggerFactory(
+            LoggerFactory.Create(l => l.AddConsole().SetMinimumLevel(LogLevel.Information)));
+        var logger = Log.CreateLogger("benchmark");
+
+        // Required to allow unencrypted GrpcNet connections
+        AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
+        var system = new ActorSystem(new ActorSystemConfig()
+                .WithDeveloperSupervisionLogging(true)
+                .WithDeadLetterRequestLogging(true)
+                .WithDeadLetterResponseLogging(true)
+                .WithConfigureProps(p => p.WithDeadlineDecorator(TimeSpan.FromSeconds(1), logger).WithLoggingContextDecorator(logger)))
+
+            .WithRemote(GrpcNetRemoteConfig.BindToLocalhost().WithProtoMessages(ProtosReflection.Descriptor))
+            .WithCluster(ClusterConfig
+                .Setup("MyCluster", new SeedNodeClusterProvider(new(("127.0.0.1", 8090))), new PartitionIdentityLookup()));
+
+        system.EventStream.Subscribe<ClusterTopology>(e => {
+                Console.WriteLine($"{DateTime.Now:O} My members {e.TopologyHash}");
+            }
+        );
+
+        await system
+            .Cluster()
+            .StartMemberAsync();
+
+        Console.WriteLine("Started");
+
+        var helloGrain = system.Cluster().GetHelloGrain("MyGrain");
+
+        var res = await helloGrain.SayHello(new HelloRequest(), FromSeconds(5));
+        Console.WriteLine(res.Message);
+
+        res = await helloGrain.SayHello(new HelloRequest(), FromSeconds(5));
+        Console.WriteLine(res.Message);
+
+        Console.WriteLine("Press enter to exit");
+        Console.ReadLine();
+        Console.WriteLine("Shutting Down...");
+        await system.Cluster().ShutdownAsync();
+    }
+}

benchmarks/GossipBenchmark/Node2/Node2.csproj

@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk" ToolsVersion="15.0">
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net6.0</TargetFramework>
+    <ServerGarbageCollection>true</ServerGarbageCollection>
+    <ConcurrentGarbageCollection>true</ConcurrentGarbageCollection>
+    <LangVersion>10</LangVersion>
+  </PropertyGroup>
+  <ItemGroup>
+    <ProjectReference Include="..\..\..\src\Proto.Cluster\Proto.Cluster.csproj" />
+    <ProjectReference Include="..\..\..\src\Proto.Remote\Proto.Remote.csproj" />
+    <ProjectReference Include="..\Messages\Messages.csproj" />
+  </ItemGroup>
+</Project>

benchmarks/GossipBenchmark/Node2/Program.cs

@@ -0,0 +1,84 @@
+// -----------------------------------------------------------------------
+// <copyright file="Program.cs" company="Asynkron AB">
+//      Copyright (C) 2015-2022 Asynkron AB All rights reserved
+// </copyright>
+// -----------------------------------------------------------------------
+using System;
+using System.Threading.Tasks;
+using ClusterHelloWorld.Messages;
+using Microsoft.Extensions.Logging;
+using Proto;
+using Proto.Cluster;
+using Proto.Cluster.Partition;
+using Proto.Cluster.Seed;
+using Proto.Context;
+using Proto.Remote;
+using Proto.Remote.GrpcNet;
+using static System.Threading.Tasks.Task;
+using ProtosReflection = ClusterHelloWorld.Messages.ProtosReflection;
+
+namespace Node2;
+
+public class HelloGrain : HelloGrainBase
+{
+    private readonly string _identity;
+
+    public HelloGrain(IContext ctx, string identity) : base(ctx) => _identity = identity;
+
+    public override Task<HelloResponse> SayHello(HelloRequest request)
+    {
+        Console.WriteLine("Got request!!");
+        var res = new HelloResponse
+        {
+            Message = $"Hello from typed grain {_identity}"
+        };
+
+        return FromResult(res);
+    }
+}
+
+class Program
+{
+    private static async Task Main()
+    {
+        Proto.Log.SetLoggerFactory(
+            LoggerFactory.Create(l => l.AddConsole().SetMinimumLevel(LogLevel.Information)));
+
+        var logger = Log.CreateLogger("benchmark");
+
+        // Required to allow unencrypted GrpcNet connections
+        AppContext.SetSwitch("System.Net.Http.SocketsHttpHandler.Http2UnencryptedSupport", true);
+        var system = new ActorSystem(new ActorSystemConfig()
+                .WithDeveloperSupervisionLogging(true)
+                .WithDeadLetterRequestLogging(true)
+                .WithDeadLetterResponseLogging(true)
+                .WithConfigureProps(p => p.WithDeadlineDecorator(TimeSpan.FromSeconds(1), logger).WithLoggingContextDecorator(logger)))
+            .WithRemote(GrpcNetRemoteConfig.BindToLocalhost(8090).WithProtoMessages(ProtosReflection.Descriptor))
+            .WithCluster(ClusterConfig
+                .Setup("MyCluster", new SeedNodeClusterProvider(), new PartitionIdentityLookup())
+                .WithClusterKind(HelloGrainActor.GetClusterKind((ctx, identity) => new HelloGrain(ctx, identity.Identity)))
+            );
+
+        system.EventStream.Subscribe<ClusterTopology>(e => {
+                Console.WriteLine($"{DateTime.Now:O} My members {e.TopologyHash}");
+            }
+        );
+
+        await system
+            .Cluster()
+            .StartMemberAsync();
+
+        Console.WriteLine("Started...");
+
+        Console.CancelKeyPress += async (e, y) => {
+            Console.WriteLine("Shutting Down...");
+            await system
+                .Cluster()
+                .ShutdownAsync();
+        };
+
+
+
+        await Delay(-1);
+    }
+}

benchmarks/GossipBenchmark/docker-compose.yml

@@ -0,0 +1,48 @@
+version: '3.7'
+services:
+
+  consul-agent-1: &consul-agent
+    image: consul:latest
+    networks:
+      - consul
+    command: "agent -retry-join consul-server-bootstrap -client 0.0.0.0"
+
+  consul-agent-2:
+    <<: *consul-agent
+
+  consul-agent-3:
+    <<: *consul-agent
+
+  consul-server-1: &consul-server
+    <<: *consul-agent
+    command: "agent -server -retry-join consul-server-bootstrap -client 0.0.0.0"
+
+  consul-server-2:
+    <<: *consul-server
+
+  consul-server-bootstrap:
+    <<: *consul-agent
+    ports:
+      - "8400:8400"
+      - "8500:8500"
+      - "8600:8600"
+      - "8600:8600/udp"
+    command: "agent -server -bootstrap-expect 3 -ui -client 0.0.0.0"
+
+  mongodb:
+    image: mongo:latest
+    ports:
+      - 127.0.0.1:27017:27017
+    volumes:
+      - mongodb_data:/data/db
+
+  redis:
+    image: redis:latest
+    ports:
+      - 127.0.0.1:6379:6379
+
+networks:
+  consul:
+
+volumes:
+  mongodb_data:

benchmarks/GossipBenchmark/run.sh

@@ -0,0 +1 @@
+docker-compose up --build

src/Proto.Cluster/Gossip/GossipActor.cs

@@ -111,12 +111,20 @@ private Task OnGossipRequest(IContext context, GossipRequest gossipRequest)
             return Task.CompletedTask;
         }
 
-        context.RequestReenter<GossipResponseAck>(context.Sender!, new GossipResponse
-            {
-                State = memberState.State.Clone(), //ensure we have a copy and not state that might mutate
-            }, task => ReenterAfterResponseAck(context, task, memberState), CancellationTokens.WithTimeout(_gossipRequestTimeout));
+        context.Respond(new GossipResponse(){
+            State = memberState.State.Clone(), //ensure we have a copy and not state that might mutate
+        });
 
         return Task.CompletedTask;
+
+        //this code is broken
+        //
+        // context.RequestReenter<GossipResponseAck>(context.Sender!, new GossipResponse
+        //     {
+        //         State = memberState.State.Clone(), //ensure we have a copy and not state that might mutate
+        //     }, task => ReenterAfterResponseAck(context, task, memberState), CancellationTokens.WithTimeout(_gossipRequestTimeout));
+        //
+        // return Task.CompletedTask;
     }
 
     private void ReceiveState(IContext context, GossipState remoteState)