From d2401209f513cbce50f89194d71cfddb93ff5030 Mon Sep 17 00:00:00 2001
From: MASES Public Developers Team
<94312179+masesdevelopers@users.noreply.github.com>
Date: Wed, 12 Jul 2023 02:17:05 +0200
Subject: [PATCH] Many updates and errors solved (#53)
* Added missing file
* #52: fix management of serialization
* #52: fix some errors, code cleanup and added new options
* #22: update to KNet 2.0.1
---
.../Internal/KafkaOptionsExtension.cs | 72 ++++++++++++----
src/KEFCore/Infrastructure/KafkaDbContext.cs | 85 +++++++++++++++++++
src/KEFCore/KEFCore.cs | 5 ++
src/KEFCore/KEFCore.csproj | 4 +-
.../Serdes/Internal/IKafkaSerdesEntityType.cs | 10 ++-
.../Serdes/Internal/KafkaSerdesEntityType.cs | 62 ++++++++++++--
.../Serdes/Internal/KafkaSerdesFactory.cs | 3 +-
src/KEFCore/Storage/Internal/IKafkaCluster.cs | 3 +-
src/KEFCore/Storage/Internal/IKafkaTable.cs | 8 +-
src/KEFCore/Storage/Internal/KafkaCluster.cs | 46 ++++++----
.../Internal/KafkaStreamsBaseRetriever.cs | 12 ++-
.../Internal/KafkaStreamsTableRetriever.cs | 4 +-
src/KEFCore/Storage/Internal/KafkaTable.cs | 17 ++--
test/KEFCore.Test/KEFCore.Test.csproj | 2 +-
test/KEFCore.Test/Program.cs | 44 ++++++----
15 files changed, 288 insertions(+), 89 deletions(-)
create mode 100644 src/KEFCore/Infrastructure/KafkaDbContext.cs
diff --git a/src/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs b/src/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs
index 0dd133a0..e08853fd 100644
--- a/src/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs
+++ b/src/KEFCore/Infrastructure/Internal/KafkaOptionsExtension.cs
@@ -18,6 +18,7 @@
#nullable enable
+using Java.Lang;
using Java.Util;
using MASES.JCOBridge.C2JBridge;
using MASES.KNet.Common;
@@ -46,6 +47,9 @@ public class KafkaOptionsExtension : IDbContextOptionsExtension
private TopicConfigBuilder? _topicConfigBuilder;
private DbContextOptionsExtensionInfo? _info;
+ static Java.Lang.ClassLoader _loader = Java.Lang.ClassLoader.SystemClassLoader;
+ static Java.Lang.ClassLoader SystemClassLoader => _loader;
+
public KafkaOptionsExtension()
{
}
@@ -199,18 +203,31 @@ public virtual Properties StreamsOptions(IEntityType entityType)
public virtual Properties StreamsOptions(string applicationId)
{
- var props = new Properties();
- var localCfg = StreamsConfigBuilder.CreateFrom(StreamsConfigBuilder).WithApplicationId(applicationId)
- .WithBootstrapServers(BootstrapServers)
- .WithDefaultKeySerdeClass(Org.Apache.Kafka.Common.Serialization.Serdes.String().Dyn().getClass())
- .WithDefaultValueSerdeClass(Org.Apache.Kafka.Common.Serialization.Serdes.String().Dyn().getClass());
-
-
+ Properties props = _streamsConfigBuilder ?? new();
+ if (props.ContainsKey(StreamsConfig.APPLICATION_ID_CONFIG))
+ {
+ props.Remove(StreamsConfig.APPLICATION_ID_CONFIG);
+ }
props.Put(StreamsConfig.APPLICATION_ID_CONFIG, applicationId);
+ if (props.ContainsKey(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG))
+ {
+ props.Remove(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG);
+ }
props.Put(StreamsConfig.BOOTSTRAP_SERVERS_CONFIG, BootstrapServers);
- props.Put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Org.Apache.Kafka.Common.Serialization.Serdes.String().Dyn().getClass());
- props.Put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Org.Apache.Kafka.Common.Serialization.Serdes.String().Dyn().getClass());
-
+ if (props.ContainsKey(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG))
+ {
+ props.Remove(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG);
+ }
+ props.Put(StreamsConfig.DEFAULT_KEY_SERDE_CLASS_CONFIG, Class.ForName("org.apache.kafka.common.serialization.Serdes$StringSerde", true, SystemClassLoader));
+ if (props.ContainsKey(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG))
+ {
+ props.Remove(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG);
+ }
+ props.Put(StreamsConfig.DEFAULT_VALUE_SERDE_CLASS_CONFIG, Class.ForName("org.apache.kafka.common.serialization.Serdes$ByteArraySerde", true, SystemClassLoader));
+ if (props.ContainsKey(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG))
+ {
+ props.Remove(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
+ }
props.Put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
return props;
@@ -218,13 +235,34 @@ public virtual Properties StreamsOptions(string applicationId)
public virtual Properties ProducerOptions()
{
- Properties props = new();
+ Properties props = _producerConfigBuilder ?? new();
+ if (props.ContainsKey(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG))
+ {
+ props.Remove(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG);
+ }
props.Put(ProducerConfig.BOOTSTRAP_SERVERS_CONFIG, BootstrapServers);
- props.Put(ProducerConfig.ACKS_CONFIG, "all");
- props.Put(ProducerConfig.RETRIES_CONFIG, 0);
- props.Put(ProducerConfig.LINGER_MS_CONFIG, 1);
- props.Put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
- props.Put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, "org.apache.kafka.common.serialization.StringSerializer");
+ if (!props.ContainsKey(ProducerConfig.ACKS_CONFIG))
+ {
+ props.Put(ProducerConfig.ACKS_CONFIG, "all");
+ }
+ if (!props.ContainsKey(ProducerConfig.RETRIES_CONFIG))
+ {
+ props.Put(ProducerConfig.RETRIES_CONFIG, 0);
+ }
+ if (!props.ContainsKey(ProducerConfig.LINGER_MS_CONFIG))
+ {
+ props.Put(ProducerConfig.LINGER_MS_CONFIG, 1);
+ }
+ if (props.ContainsKey(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG))
+ {
+ props.Remove(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG);
+ }
+ props.Put(ProducerConfig.KEY_SERIALIZER_CLASS_CONFIG, Class.ForName("org.apache.kafka.common.serialization.StringSerializer", true, SystemClassLoader));
+ if (props.ContainsKey(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG))
+ {
+ props.Remove(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG);
+ }
+ props.Put(ProducerConfig.VALUE_SERIALIZER_CLASS_CONFIG, Class.ForName("org.apache.kafka.common.serialization.StringSerializer", true, SystemClassLoader));
return props;
}
@@ -263,7 +301,7 @@ public override string LogFragment
{
if (_logFragment == null)
{
- var builder = new StringBuilder();
+ var builder = new System.Text.StringBuilder();
builder.Append("DataBaseName=").Append(Extension._databaseName).Append(' ');
diff --git a/src/KEFCore/Infrastructure/KafkaDbContext.cs b/src/KEFCore/Infrastructure/KafkaDbContext.cs
new file mode 100644
index 00000000..959ef782
--- /dev/null
+++ b/src/KEFCore/Infrastructure/KafkaDbContext.cs
@@ -0,0 +1,85 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+
+/*
+* Copyright 2022 MASES s.r.l.
+*
+* Licensed under the Apache License, Version 2.0 (the "License");
+* you may not use this file except in compliance with the License.
+* You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*
+* Refer to LICENSE for more information.
+*/
+
+using MASES.KNet.Common;
+using MASES.KNet.Producer;
+using MASES.KNet.Streams;
+
+namespace MASES.EntityFrameworkCore.KNet.Infrastructure;
+
+///
+/// Allows Kafka specific configuration to be performed on .
+///
+public class KafkaDbContext : DbContext
+{
+ ///
+ /// The bootstrap servers of the Apache Kafka cluster
+ ///
+ public string? BootstrapServers { get; set; }
+ ///
+ /// The application id
+ ///
+ public string ApplicationId { get; set; } = Guid.NewGuid().ToString();
+ ///
+ /// Database name
+ ///
+ public string? DbName { get; set; }
+ ///
+ /// Database number of partitions
+ ///
+ public int DefaultNumPartitions { get; set; } = 10;
+ ///
+ /// Database replication factor
+ ///
+ public short DefaultReplicationFactor { get; set; } = 1;
+ ///
+ /// Use persistent storage
+ ///
+ public bool UsePersistentStorage { get; set; } = false;
+ ///
+ /// Use a producer for each Entity
+ ///
+ public bool UseProducerByEntity { get; set; } = false;
+
+ public ProducerConfigBuilder? ProducerConfigBuilder { get; set; }
+
+ public StreamsConfigBuilder? StreamsConfigBuilder { get; set; }
+
+ public TopicConfigBuilder? TopicConfigBuilder { get; set; }
+
+ protected override void OnConfiguring(DbContextOptionsBuilder optionsBuilder)
+ {
+ if (BootstrapServers == null)
+ {
+ throw new ArgumentNullException(nameof(BootstrapServers));
+ }
+
+ if (DbName == null) throw new ArgumentNullException(nameof(DbName));
+
+ optionsBuilder.UseKafkaDatabase(ApplicationId, DbName, BootstrapServers, (o) =>
+ {
+ o.StreamsConfig(StreamsConfigBuilder??o.EmptyStreamsConfigBuilder).WithDefaultNumPartitions(DefaultNumPartitions);
+ o.WithUsePersistentStorage(UsePersistentStorage);
+ o.WithProducerByEntity(UseProducerByEntity);
+ o.WithDefaultReplicationFactor(DefaultReplicationFactor);
+ });
+ }
+}
diff --git a/src/KEFCore/KEFCore.cs b/src/KEFCore/KEFCore.cs
index 865549a2..f743e683 100644
--- a/src/KEFCore/KEFCore.cs
+++ b/src/KEFCore/KEFCore.cs
@@ -22,5 +22,10 @@ namespace MASES.EntityFrameworkCore.KNet
{
public class KEFCore : KNetCore
{
+#if DEBUG
+ public override bool EnableDebug => true;
+
+ public override bool LogClassPath => true;
+#endif
}
}
diff --git a/src/KEFCore/KEFCore.csproj b/src/KEFCore/KEFCore.csproj
index 0ff3ccf4..80288fe1 100644
--- a/src/KEFCore/KEFCore.csproj
+++ b/src/KEFCore/KEFCore.csproj
@@ -63,10 +63,10 @@
-
+
All
None
-
+
diff --git a/src/KEFCore/Serdes/Internal/IKafkaSerdesEntityType.cs b/src/KEFCore/Serdes/Internal/IKafkaSerdesEntityType.cs
index e29c5fbf..5fc790bf 100644
--- a/src/KEFCore/Serdes/Internal/IKafkaSerdesEntityType.cs
+++ b/src/KEFCore/Serdes/Internal/IKafkaSerdesEntityType.cs
@@ -16,17 +16,19 @@
* Refer to LICENSE for more information.
*/
+using Org.Apache.Kafka.Common.Header;
+
namespace MASES.EntityFrameworkCore.KNet.Serdes.Internal
{
public interface IKafkaSerdesEntityType
{
- string Serialize(params object?[]? args);
+ string Serialize(Headers headers, params object?[]? args);
- string Serialize(TKey key);
+ string Serialize(Headers headers, TKey key);
- object[] Deserialize(string arg);
+ object[] Deserialize(Headers headers, string arg);
- TKey Deserialize(string arg);
+ TKey Deserialize(Headers headers, string arg);
object[] ConvertData(object[]? input);
}
diff --git a/src/KEFCore/Serdes/Internal/KafkaSerdesEntityType.cs b/src/KEFCore/Serdes/Internal/KafkaSerdesEntityType.cs
index 52d0e6a7..ba5806da 100644
--- a/src/KEFCore/Serdes/Internal/KafkaSerdesEntityType.cs
+++ b/src/KEFCore/Serdes/Internal/KafkaSerdesEntityType.cs
@@ -16,8 +16,14 @@
* Refer to LICENSE for more information.
*/
+using Org.Apache.Kafka.Common.Header;
+using System.Text.Json;
+using System.Text.Json.Nodes;
+using System.Text.Json.Serialization;
+
namespace MASES.EntityFrameworkCore.KNet.Serdes.Internal
{
+ [JsonSerializable(typeof(KafkaSerdesEntityTypeData))]
public class KafkaSerdesEntityTypeData
{
public KafkaSerdesEntityTypeData() { }
@@ -27,9 +33,10 @@ public KafkaSerdesEntityTypeData(string tName, object[] rData)
typeName = tName;
data = rData;
}
-
- public string? typeName;
- public object[]? data;
+ [JsonInclude()]
+ public string typeName;
+ [JsonInclude()]
+ public object[] data;
}
public class KafkaSerdesEntityType : IKafkaSerdesEntityType
@@ -43,27 +50,64 @@ public KafkaSerdesEntityType(IEntityType type)
_properties = _type.GetProperties().ToArray();
}
- public object[] Deserialize(string arg)
+ public object[] Deserialize(Headers headers, string arg)
{
var des = GetFullType(arg);
return ConvertData(des!.data);
}
- public TKey Deserialize(string arg) => System.Text.Json.JsonSerializer.Deserialize(arg)!;
+ public TKey Deserialize(Headers headers, string arg) => System.Text.Json.JsonSerializer.Deserialize(arg)!;
- public string Serialize(params object?[]? args) => System.Text.Json.JsonSerializer.Serialize(new KafkaSerdesEntityTypeData(_type.Name, args!));
+ public string Serialize(Headers headers, params object?[]? args) => System.Text.Json.JsonSerializer.Serialize(new KafkaSerdesEntityTypeData(_type.Name, args!));
- public string Serialize(TKey key) => System.Text.Json.JsonSerializer.Serialize(key);
+ public string Serialize(Headers headers, TKey key) => System.Text.Json.JsonSerializer.Serialize(key);
public static KafkaSerdesEntityTypeData? GetFullType(string arg) => System.Text.Json.JsonSerializer.Deserialize(arg);
public object[] ConvertData(object[]? input)
{
+ if (input == null) return null;
+ List