Merge pull request #107 from microsoft/fix/sharding-policy-noise

Fix/sharding policy noise
microsoft · Jun 23, 2022 · f6007bc · f6007bc
2 parents 51daa52 + 37b127b
commit f6007bc
Show file tree

Hide file tree

Showing 6 changed files with 130 additions and 47 deletions.
diff --git a/code/DeltaKustoFileIntegrationTest/Policies/Sharding/ShardingPolicyDatabaseTest.cs b/code/DeltaKustoFileIntegrationTest/Policies/Sharding/ShardingPolicyDatabaseTest.cs
@@ -38,7 +38,7 @@ public async Task NoneToOne()
             Assert.Equal("mydb", policyCommand!.EntityName.Name);
             Assert.Equal(
                 750000,
-                policyCommand!.DeserializePolicy<ShardingPolicy>().MaxRowCount);
+                policyCommand!.MaxRowCount);
         }
 
         [Fact]
@@ -92,7 +92,7 @@ public async Task OneToOneWithChange()
             Assert.Equal("mydb", policyCommand!.EntityName.Name);
             Assert.Equal(
                1000000,
-               policyCommand!.DeserializePolicy<ShardingPolicy>().MaxRowCount);
+               policyCommand!.MaxRowCount);
         }
     }
 }
diff --git a/code/DeltaKustoFileIntegrationTest/Policies/Sharding/ShardingPolicyTableTest.cs b/code/DeltaKustoFileIntegrationTest/Policies/Sharding/ShardingPolicyTableTest.cs
@@ -38,7 +38,7 @@ public async Task NoneToOne()
             Assert.Equal("my-table", policyCommand!.EntityName.Name);
             Assert.Equal(
                 750000,
-                policyCommand!.DeserializePolicy<ShardingPolicy>().MaxRowCount);
+                policyCommand!.MaxRowCount);
         }
 
         [Fact]
@@ -92,7 +92,7 @@ public async Task OneToOneWithChange()
             Assert.Equal("my-table", policyCommand!.EntityName.Name);
             Assert.Equal(
                1000000,
-               policyCommand!.DeserializePolicy<ShardingPolicy>().MaxRowCount);
+               policyCommand!.MaxRowCount);
         }
     }
 }
diff --git a/code/DeltaKustoLib/CommandModel/Policies/AlterShardingPolicyCommand.cs b/code/DeltaKustoLib/CommandModel/Policies/AlterShardingPolicyCommand.cs
@@ -5,6 +5,7 @@
 using System.Linq;
 using System.Text;
 using System.Text.Json;
+using System.Text.Json.Serialization;
 using System.Threading.Tasks;
 
 namespace DeltaKustoLib.CommandModel.Policies
@@ -15,6 +16,19 @@ namespace DeltaKustoLib.CommandModel.Policies
     [Command(15100, "Alter Sharding Policies")]
     public class AlterShardingPolicyCommand : EntityPolicyCommandBase
     {
+        private static readonly ShardingPolicySerializerContext _serializerContext
+            = new ShardingPolicySerializerContext(
+                new JsonSerializerOptions
+                {
+                    WriteIndented = true
+                });
+
+        public int? MaxRowCount { get; }
+
+        public int? MaxExtentSizeInMb { get; }
+
+        public int? MaxOriginalSizeInMb { get; }
+
         public override string CommandFriendlyName => ".alter <entity> policy sharding";
 
         public override string ScriptPath => EntityType == EntityType.Database
@@ -24,26 +38,17 @@ public class AlterShardingPolicyCommand : EntityPolicyCommandBase
         public AlterShardingPolicyCommand(
             EntityType entityType,
             EntityName entityName,
-            JsonDocument policy) : base(entityType, entityName, policy)
-        {
-        }
-
-        public AlterShardingPolicyCommand(
-            EntityType entityType,
-            EntityName entityName,
-            int maxRowCount,
-            int maxExtentSizeInMb,
-            int maxOriginalSizeInMb)
-            : this(
+            int? maxRowCount,
+            int? maxExtentSizeInMb,
+            int? maxOriginalSizeInMb)
+            : base(
                   entityType,
                   entityName,
-                  ToJsonDocument(new
-                  {
-                      MaxRowCount = maxRowCount,
-                      MaxExtentSizeInMb = maxExtentSizeInMb,
-                      MaxOriginalSizeInMb = maxOriginalSizeInMb
-                  }))
+                  CreatePolicyText(maxRowCount, maxExtentSizeInMb, maxOriginalSizeInMb))
         {
+            MaxRowCount = maxRowCount;
+            MaxExtentSizeInMb = maxExtentSizeInMb;
+            MaxOriginalSizeInMb = maxOriginalSizeInMb;
         }
 
         internal static CommandBase FromCode(SyntaxElement rootElement)
@@ -66,23 +71,19 @@ internal static CommandBase FromCode(SyntaxElement rootElement)
                 rootElement.GetUniqueDescendant<LiteralExpression>(
                     "Sharding",
                     e => e.NameInParent == "ShardingPolicy"));
-            var policy = Deserialize<JsonDocument>(policyText.Text);
-
-            if (policy == null)
-            {
-                throw new DeltaException(
-                    $"Can't extract policy objects from {policyText.ToScript()}");
-            }
 
-            return new AlterShardingPolicyCommand(
-                entityType,
-                EntityName.FromCode(entityName.Name),
-                policy);
+            return CreateFromPolicyText(entityType, entityName, policyText);
         }
 
         public override string ToScript(ScriptingContext? context)
         {
             var builder = new StringBuilder();
+            var policy = new
+            {
+                MaxRowCount = MaxRowCount,
+                MaxExtentSizeInMb = MaxExtentSizeInMb,
+                MaxOriginalSizeInMb = MaxOriginalSizeInMb
+            };
 
             builder.Append(".alter ");
             builder.Append(EntityType == EntityType.Table ? "table" : "database");
@@ -95,12 +96,10 @@ public override string ToScript(ScriptingContext? context)
             {
                 builder.Append(EntityName.ToScript());
             }
-            builder.Append(" policy sharding");
-            builder.AppendLine();
-            builder.Append("```");
-            builder.Append(SerializePolicy());
-            builder.AppendLine();
-            builder.Append("```");
+            builder.AppendLine(" policy sharding");
+            builder.AppendLine("```");
+            builder.AppendLine(SerializePolicy());
+            builder.AppendLine("```");
 
             return builder.ToString();
         }
@@ -129,5 +128,93 @@ internal static IEnumerable<CommandBase> ComputeDelta(
             {   //  Both target and current are null:  no delta
             }
         }
+
+        private static JsonDocument CreatePolicyText(
+            int? maxRowCount,
+            int? maxExtentSizeInMb,
+            int? maxOriginalSizeInMb)
+        {
+            var map = new Dictionary<string, int>();
+
+            if (maxRowCount != null)
+            {
+                map[nameof(maxRowCount)] = maxRowCount.Value;
+            }
+            if (maxExtentSizeInMb != null)
+            {
+                map[nameof(maxExtentSizeInMb)] = maxExtentSizeInMb.Value;
+            }
+            if (maxOriginalSizeInMb != null)
+            {
+                map[nameof(maxOriginalSizeInMb)] = maxOriginalSizeInMb.Value;
+            }
+
+            var text = Serialize(map, _serializerContext);
+            var doc = Deserialize<JsonDocument>(text);
+
+            return doc!;
+        }
+
+        private static CommandBase CreateFromPolicyText(
+            EntityType entityType,
+            NameReference entityName,
+            QuotedText policyText)
+        {
+            var policy = Deserialize<JsonDocument>(policyText.Text);
+
+            if (policy == null)
+            {
+                throw new DeltaException(
+                    $"Can't extract policy objects from {policyText.ToScript()}");
+            }
+            else
+            {
+                int? maxRowCount = null;
+                int? maxExtentSizeInMb = null;
+                int? maxOriginalSizeInMb = null;
+                var validProperties = new[]
+                {
+                    (Name:nameof(maxRowCount), Action:(Action<int>)((value) => maxRowCount = value)),
+                    (Name:nameof(maxExtentSizeInMb), Action:(Action<int>)((value) => maxExtentSizeInMb = value)),
+                    (Name:nameof(maxOriginalSizeInMb), Action:(Action<int>)((value) => maxOriginalSizeInMb = value))
+                };
+
+                foreach (var property in policy.RootElement.EnumerateObject())
+                {
+                    foreach (var validProperty in validProperties)
+                    {
+                        if (property.Name.Equals(
+                            validProperty.Name,
+                            StringComparison.InvariantCultureIgnoreCase)
+                            && property.Value.ValueKind != JsonValueKind.Null)
+                        {
+                            int value;
+
+                            if (property.Value.TryGetInt32(out value))
+                            {
+                                validProperty.Action(value);
+                            }
+                            else
+                            {
+                                throw new DeltaException($"{validProperty.Name} should be an integer but is "
+                                    + $"a {property.Value.ValueKind}");
+                            }
+                        }
+                    }
+                }
+
+                return new AlterShardingPolicyCommand(
+                    entityType,
+                    EntityName.FromCode(entityName.Name),
+                    maxRowCount,
+                    maxExtentSizeInMb,
+                    maxOriginalSizeInMb);
+            }
+        }
+    }
+
+    [JsonSerializable(typeof(Dictionary<string, int>))]
+    internal partial class ShardingPolicySerializerContext : JsonSerializerContext
+    {
     }
 }
diff --git a/code/DeltaKustoUnitTest/CommandParsing/Policies/AlterShardingPolicyTest.cs b/code/DeltaKustoUnitTest/CommandParsing/Policies/AlterShardingPolicyTest.cs
@@ -26,7 +26,7 @@ public void DbComposedTableName()
         {
             var command = ParseOneCommand(
                 ".alter table mydb.mytable policy sharding "
-                + "@'{\"MaxRowCount\":\"200000\"}'");
+                + "@'{\"MaxRowCount\":200000}'");
 
             Assert.IsType<AlterShardingPolicyCommand>(command);
 
@@ -41,7 +41,7 @@ public void ClusterComposedTableName()
         {
             var command = ParseOneCommand(
                 ".alter table mycluster.['my db'].mytable policy sharding "
-                + "@'{\"MaxRowCount\":\"300000\"}'");
+                + "@'{\"MaxRowCount\":300000}'");
 
             Assert.IsType<AlterShardingPolicyCommand>(command);
 

diff --git a/code/DeltaKustoUnitTest/Delta/Policies/DeltaShardingPolicyTest.cs b/code/DeltaKustoUnitTest/Delta/Policies/DeltaShardingPolicyTest.cs
@@ -27,9 +27,7 @@ public void TableFromEmptyToSomething()
                 42,
                 c =>
                 {
-                    var policy = c.DeserializePolicy<ShardingPolicy>();
-
-                    Assert.Equal(42, policy.MaxRowCount);
+                    Assert.Equal(42, c.MaxRowCount);
                 },
                 null);
         }
@@ -54,9 +52,7 @@ public void TableDelta()
                 54,
                 c =>
                 {
-                    var policy = c.DeserializePolicy<ShardingPolicy>();
-
-                    Assert.Equal(54, policy.MaxRowCount);
+                    Assert.Equal(54, c.MaxRowCount);
                 },
                 null);
         }

diff --git a/code/delta-kusto/delta-kusto.csproj b/code/delta-kusto/delta-kusto.csproj
@@ -5,7 +5,7 @@
 		<TargetFramework>net6.0</TargetFramework>
 		<RootNamespace>delta_kusto</RootNamespace>
 		<Nullable>enable</Nullable>
-		<Version>0.10.3</Version>
+		<Version>0.10.4</Version>
 		<!--	Important to avoid the trimming warning hell ; since we automate-test everything, we do not need static analysis	-->
 		<ILLinkTreatWarningsAsErrors>false</ILLinkTreatWarningsAsErrors>
 	</PropertyGroup>