Skip to content

Commit

Permalink
Take min/max batch size into account again for seeding (#28955)
Browse files Browse the repository at this point in the history 
Fixes #28876
  • Loading branch information
@roji
roji authored Sep 7, 2022
1 parent 27345a2 commit d453d3f
Show file tree
Hide file tree
Showing 5 changed files with 221 additions and 83 deletions.
16 changes: 5 additions & 11 deletions src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2190,17 +2190,11 @@ private IEnumerable<MigrationOperation> GetDataOperations(
yield break;
}

var commands = identityMaps.Values.SelectMany(m => m.Rows).Where(r =>
{
return r.EntityState switch
{
EntityState.Added => true,
EntityState.Modified => true,
EntityState.Unchanged => false,
EntityState.Deleted => diffContext.FindDrop(r.Table!) == null,
_ => throw new InvalidOperationException($"Unexpected entity state: {r.EntityState}")
};
});
var commands = identityMaps.Values
.SelectMany(m => m.Rows)
.Where(
r => r.EntityState is EntityState.Added or EntityState.Modified
|| (r.EntityState is EntityState.Deleted && diffContext.FindDrop(r.Table!) == null));

var commandSets = new CommandBatchPreparer(CommandBatchPreparerDependencies)
.TopologicalSort(commands);
Expand Down
9 changes: 9 additions & 0 deletions src/EFCore.Relational/Update/ICommandBatchPreparer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,13 @@ public interface ICommandBatchPreparer
/// <param name="updateAdapter">The model data.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> BatchCommands(IList<IUpdateEntry> entries, IUpdateAdapter updateAdapter);

/// <summary>
/// Given a set of modification commands, returns one more ready-to-execute batches for those commands, taking into account e.g.
/// maximum batch sizes and other batching constraints.
/// </summary>
/// <param name="commandSet">The set of commands to be organized in batches.</param>
/// <param name="moreCommandSets">Whether more command sets are expected after this one within the same save operation.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> CreateCommandBatches(IEnumerable<IReadOnlyModificationCommand> commandSet, bool moreCommandSets);
}
165 changes: 101 additions & 64 deletions src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs
View file
Original file line number Diff line number Diff line change
@@ -1,7 +1,6 @@
// Licensed to the .NET Foundation under one or more agreements.
// The .NET Foundation licenses this file to you under the MIT license.

using System.Security.Principal;
using System.Text;
using Microsoft.EntityFrameworkCore.Metadata.Internal;

Expand Down Expand Up @@ -70,91 +69,129 @@ public virtual IEnumerable<ModificationCommandBatch> BatchCommands(

for (var commandSetIndex = 0; commandSetIndex < commandSets.Count; commandSetIndex++)
{
var commandSet = commandSets[commandSetIndex];
var batches = CreateCommandBatches(
commandSets[commandSetIndex],
commandSetIndex < commandSets.Count - 1,
assertColumnModification: true,
parameterNameGenerator);

var batch = Dependencies.ModificationCommandBatchFactory.Create();
foreach (var modificationCommand in commandSet)
foreach (var batch in batches)
{
yield return batch;
}
}
}

/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
public virtual IEnumerable<ModificationCommandBatch> CreateCommandBatches(
IEnumerable<IReadOnlyModificationCommand> commandSet,
bool moreCommandSets)
=> CreateCommandBatches(commandSet, moreCommandSets, assertColumnModification: false);

/// <summary>
/// This is an internal API that supports the Entity Framework Core infrastructure and not subject to
/// the same compatibility standards as public APIs. It may be changed or removed without notice in
/// any release. You should only use it directly in your code with extreme caution and knowing that
/// doing so can result in application failures when updating to a new Entity Framework Core release.
/// </summary>
private IEnumerable<ModificationCommandBatch> CreateCommandBatches(
IEnumerable<IReadOnlyModificationCommand> commandSet,
bool moreCommandSets,
bool assertColumnModification,
ParameterNameGenerator? parameterNameGenerator = null)
{
var batch = Dependencies.ModificationCommandBatchFactory.Create();

foreach (var modificationCommand in commandSet)
{
#if DEBUG
if (assertColumnModification)
{
(modificationCommand as ModificationCommand)?.AssertColumnsNotInitialized();
if (modificationCommand.EntityState == EntityState.Modified
&& !modificationCommand.ColumnModifications.Any(m => m.IsWrite))
{
continue;
}
}
#endif

if (modificationCommand.EntityState == EntityState.Modified
&& !modificationCommand.ColumnModifications.Any(m => m.IsWrite))
{
continue;
}

if (!batch.TryAddCommand(modificationCommand))
if (!batch.TryAddCommand(modificationCommand))
{
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
{
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
if (batch.ModificationCommands.Count > 1)
{
if (batch.ModificationCommands.Count > 1)
{
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}

batch.Complete(moreBatchesExpected: true);
batch.Complete(moreBatchesExpected: true);

yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);
yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);

foreach (var command in batch.ModificationCommands)
{
batch = StartNewBatch(parameterNameGenerator, command);
batch.Complete(moreBatchesExpected: true);
foreach (var command in batch.ModificationCommands)
{
batch = StartNewBatch(parameterNameGenerator, command);
batch.Complete(moreBatchesExpected: true);

yield return batch;
}
yield return batch;
}

batch = StartNewBatch(parameterNameGenerator, modificationCommand);
}
}

var hasMoreCommandSets = commandSetIndex < commandSets.Count - 1;
batch = StartNewBatch(parameterNameGenerator, modificationCommand);
}
}

if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
if (batch.ModificationCommands.Count == 1
|| batch.ModificationCommands.Count >= _minBatchSize)
{
if (batch.ModificationCommands.Count > 1)
{
if (batch.ModificationCommands.Count > 1)
{
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}
Dependencies.UpdateLogger.BatchReadyForExecution(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count);
}

batch.Complete(moreBatchesExpected: hasMoreCommandSets);
batch.Complete(moreBatchesExpected: moreCommandSets);

yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);
yield return batch;
}
else
{
Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize(
batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize);

for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++)
{
var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]);
singleCommandBatch.Complete(
moreBatchesExpected: hasMoreCommandSets || commandIndex < batch.ModificationCommands.Count - 1);
for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++)
{
var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]);
singleCommandBatch.Complete(
moreBatchesExpected: moreCommandSets || commandIndex < batch.ModificationCommands.Count - 1);

yield return singleCommandBatch;
}
yield return singleCommandBatch;
}
}
}

private ModificationCommandBatch StartNewBatch(
ParameterNameGenerator parameterNameGenerator,
IReadOnlyModificationCommand modificationCommand)
{
parameterNameGenerator.Reset();
var batch = Dependencies.ModificationCommandBatchFactory.Create();
batch.TryAddCommand(modificationCommand);
return batch;
ModificationCommandBatch StartNewBatch(
ParameterNameGenerator? parameterNameGenerator,
IReadOnlyModificationCommand modificationCommand)
{
parameterNameGenerator?.Reset();
var batch = Dependencies.ModificationCommandBatchFactory.Create();
batch.TryAddCommand(modificationCommand);
return batch;
}
}

/// <summary>
Expand Down
22 changes: 14 additions & 8 deletions src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@
using Microsoft.EntityFrameworkCore.SqlServer.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Metadata.Internal;
using Microsoft.EntityFrameworkCore.SqlServer.Update.Internal;
using Microsoft.EntityFrameworkCore.Update.Internal;

// ReSharper disable once CheckNamespace
namespace Microsoft.EntityFrameworkCore.Migrations;
Expand All @@ -33,17 +34,18 @@ public class SqlServerMigrationsSqlGenerator : MigrationsSqlGenerator
private IReadOnlyList<MigrationOperation> _operations = null!;
private int _variableCounter;

private readonly ICommandBatchPreparer _commandBatchPreparer;

/// <summary>
/// Creates a new <see cref="SqlServerMigrationsSqlGenerator" /> instance.
/// </summary>
/// <param name="dependencies">Parameter object containing dependencies for this service.</param>
/// <param name="migrationsAnnotations">Provider-specific Migrations annotations to use.</param>
/// <param name="commandBatchPreparer">The command batch preparer.</param>
public SqlServerMigrationsSqlGenerator(
MigrationsSqlGeneratorDependencies dependencies,
IRelationalAnnotationProvider migrationsAnnotations)
ICommandBatchPreparer commandBatchPreparer)
: base(dependencies)
{
}
=> _commandBatchPreparer = commandBatchPreparer;

/// <summary>
/// Generates commands from a list of operations.
Expand Down Expand Up @@ -1445,10 +1447,14 @@ protected override void Generate(
GenerateIdentityInsert(builder, operation, on: true, model);

var sqlBuilder = new StringBuilder();
((ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator).AppendBulkInsertOperation(
sqlBuilder,
GenerateModificationCommands(operation, model).ToList(),
0);

var modificationCommands = GenerateModificationCommands(operation, model).ToList();
var updateSqlGenerator = (ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator;

foreach (var batch in _commandBatchPreparer.CreateCommandBatches(modificationCommands, moreCommandSets: true))
{
updateSqlGenerator.AppendBulkInsertOperation(sqlBuilder, batch.ModificationCommands, commandPosition: 0);
}

if (Options.HasFlag(MigrationsSqlGenerationOptions.Idempotent))
{
Expand Down
92 changes: 92 additions & 0 deletions test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -788,6 +788,85 @@ IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name'
");
}

[ConditionalFact]
public virtual void InsertDataOperation_max_batch_size_is_respected()
{
// The SQL Server max batch size is 42 by default
var values = new object[50, 1];
for (var i = 0; i < 50; i++)
{
values[i, 0] = "Foo" + i;
}

Generate(
CreateGotModel,
new InsertDataOperation
{
Table = "People",
Columns = new[] { "First Name" },
Values = values
});

AssertSql(
@"IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] ON;
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo0'),
(N'Foo1'),
(N'Foo2'),
(N'Foo3'),
(N'Foo4'),
(N'Foo5'),
(N'Foo6'),
(N'Foo7'),
(N'Foo8'),
(N'Foo9'),
(N'Foo10'),
(N'Foo11'),
(N'Foo12'),
(N'Foo13'),
(N'Foo14'),
(N'Foo15'),
(N'Foo16'),
(N'Foo17'),
(N'Foo18'),
(N'Foo19'),
(N'Foo20'),
(N'Foo21'),
(N'Foo22'),
(N'Foo23'),
(N'Foo24'),
(N'Foo25'),
(N'Foo26'),
(N'Foo27'),
(N'Foo28'),
(N'Foo29'),
(N'Foo30'),
(N'Foo31'),
(N'Foo32'),
(N'Foo33'),
(N'Foo34'),
(N'Foo35'),
(N'Foo36'),
(N'Foo37'),
(N'Foo38'),
(N'Foo39'),
(N'Foo40'),
(N'Foo41');
INSERT INTO [dbo].[People] ([First Name])
VALUES (N'Foo42'),
(N'Foo43'),
(N'Foo44'),
(N'Foo45'),
(N'Foo46'),
(N'Foo47'),
(N'Foo48'),
(N'Foo49');
IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]'))
SET IDENTITY_INSERT [dbo].[People] OFF;
");
}

public override void InsertDataOperation_throws_for_unsupported_column_types()
=> base.InsertDataOperation_throws_for_unsupported_column_types();

Expand Down Expand Up @@ -1045,6 +1124,19 @@ public virtual void CreateIndex_generates_exec_when_legacy_filter_and_idempotent
");
}

private static void CreateGotModel(ModelBuilder b)
=> b.HasDefaultSchema("dbo").Entity(
"Person", pb =>
{
pb.ToTable("People");
pb.Property<string>("FirstName").HasColumnName("First Name");
pb.Property<string>("LastName").HasColumnName("Last Name");
pb.Property<string>("Birthplace").HasColumnName("Birthplace");
pb.Property<string>("Allegiance").HasColumnName("House Allegiance");
pb.Property<string>("Culture").HasColumnName("Culture");
pb.HasKey("FirstName", "LastName");
});

public SqlServerMigrationsSqlGeneratorTest()
: base(
SqlServerTestHelpers.Instance,
Expand Down

0 comments on commit d453d3f

Please sign in to comment.