diff --git a/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs b/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs index 04c93ffd454..eafacf1767c 100644 --- a/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs +++ b/src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs @@ -2190,17 +2190,11 @@ private IEnumerable GetDataOperations( yield break; } - var commands = identityMaps.Values.SelectMany(m => m.Rows).Where(r => - { - return r.EntityState switch - { - EntityState.Added => true, - EntityState.Modified => true, - EntityState.Unchanged => false, - EntityState.Deleted => diffContext.FindDrop(r.Table!) == null, - _ => throw new InvalidOperationException($"Unexpected entity state: {r.EntityState}") - }; - }); + var commands = identityMaps.Values + .SelectMany(m => m.Rows) + .Where( + r => r.EntityState is EntityState.Added or EntityState.Modified + || (r.EntityState is EntityState.Deleted && diffContext.FindDrop(r.Table!) == null)); var commandSets = new CommandBatchPreparer(CommandBatchPreparerDependencies) .TopologicalSort(commands); diff --git a/src/EFCore.Relational/Update/ICommandBatchPreparer.cs b/src/EFCore.Relational/Update/ICommandBatchPreparer.cs index cd1794f858c..09923953545 100644 --- a/src/EFCore.Relational/Update/ICommandBatchPreparer.cs +++ b/src/EFCore.Relational/Update/ICommandBatchPreparer.cs @@ -34,4 +34,13 @@ public interface ICommandBatchPreparer /// The model data. /// The list of batches to execute. IEnumerable BatchCommands(IList entries, IUpdateAdapter updateAdapter); + + /// + /// Given a set of modification commands, returns one more ready-to-execute batches for those commands, taking into account e.g. + /// maximum batch sizes and other batching constraints. + /// + /// The set of commands to be organized in batches. + /// Whether more command sets are expected after this one within the same save operation. + /// The list of batches to execute. + IEnumerable CreateCommandBatches(IEnumerable commandSet, bool moreCommandSets); } diff --git a/src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs b/src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs index f668f2b3ae6..6b3b4d2de69 100644 --- a/src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs +++ b/src/EFCore.Relational/Update/Internal/CommandBatchPreparer.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Security.Principal; using System.Text; using Microsoft.EntityFrameworkCore.Metadata.Internal; @@ -70,91 +69,129 @@ public virtual IEnumerable BatchCommands( for (var commandSetIndex = 0; commandSetIndex < commandSets.Count; commandSetIndex++) { - var commandSet = commandSets[commandSetIndex]; + var batches = CreateCommandBatches( + commandSets[commandSetIndex], + commandSetIndex < commandSets.Count - 1, + assertColumnModification: true, + parameterNameGenerator); - var batch = Dependencies.ModificationCommandBatchFactory.Create(); - foreach (var modificationCommand in commandSet) + foreach (var batch in batches) + { + yield return batch; + } + } + } + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + public virtual IEnumerable CreateCommandBatches( + IEnumerable commandSet, + bool moreCommandSets) + => CreateCommandBatches(commandSet, moreCommandSets, assertColumnModification: false); + + /// + /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to + /// the same compatibility standards as public APIs. It may be changed or removed without notice in + /// any release. You should only use it directly in your code with extreme caution and knowing that + /// doing so can result in application failures when updating to a new Entity Framework Core release. + /// + private IEnumerable CreateCommandBatches( + IEnumerable commandSet, + bool moreCommandSets, + bool assertColumnModification, + ParameterNameGenerator? parameterNameGenerator = null) + { + var batch = Dependencies.ModificationCommandBatchFactory.Create(); + + foreach (var modificationCommand in commandSet) + { +#if DEBUG + if (assertColumnModification) { (modificationCommand as ModificationCommand)?.AssertColumnsNotInitialized(); - if (modificationCommand.EntityState == EntityState.Modified - && !modificationCommand.ColumnModifications.Any(m => m.IsWrite)) - { - continue; - } + } +#endif + + if (modificationCommand.EntityState == EntityState.Modified + && !modificationCommand.ColumnModifications.Any(m => m.IsWrite)) + { + continue; + } - if (!batch.TryAddCommand(modificationCommand)) + if (!batch.TryAddCommand(modificationCommand)) + { + if (batch.ModificationCommands.Count == 1 + || batch.ModificationCommands.Count >= _minBatchSize) { - if (batch.ModificationCommands.Count == 1 - || batch.ModificationCommands.Count >= _minBatchSize) + if (batch.ModificationCommands.Count > 1) { - if (batch.ModificationCommands.Count > 1) - { - Dependencies.UpdateLogger.BatchReadyForExecution( - batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count); - } + Dependencies.UpdateLogger.BatchReadyForExecution( + batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count); + } - batch.Complete(moreBatchesExpected: true); + batch.Complete(moreBatchesExpected: true); - yield return batch; - } - else - { - Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize( - batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize); + yield return batch; + } + else + { + Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize( + batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize); - foreach (var command in batch.ModificationCommands) - { - batch = StartNewBatch(parameterNameGenerator, command); - batch.Complete(moreBatchesExpected: true); + foreach (var command in batch.ModificationCommands) + { + batch = StartNewBatch(parameterNameGenerator, command); + batch.Complete(moreBatchesExpected: true); - yield return batch; - } + yield return batch; } - - batch = StartNewBatch(parameterNameGenerator, modificationCommand); } - } - var hasMoreCommandSets = commandSetIndex < commandSets.Count - 1; + batch = StartNewBatch(parameterNameGenerator, modificationCommand); + } + } - if (batch.ModificationCommands.Count == 1 - || batch.ModificationCommands.Count >= _minBatchSize) + if (batch.ModificationCommands.Count == 1 + || batch.ModificationCommands.Count >= _minBatchSize) + { + if (batch.ModificationCommands.Count > 1) { - if (batch.ModificationCommands.Count > 1) - { - Dependencies.UpdateLogger.BatchReadyForExecution( - batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count); - } + Dependencies.UpdateLogger.BatchReadyForExecution( + batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count); + } - batch.Complete(moreBatchesExpected: hasMoreCommandSets); + batch.Complete(moreBatchesExpected: moreCommandSets); - yield return batch; - } - else - { - Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize( - batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize); + yield return batch; + } + else + { + Dependencies.UpdateLogger.BatchSmallerThanMinBatchSize( + batch.ModificationCommands.SelectMany(c => c.Entries), batch.ModificationCommands.Count, _minBatchSize); - for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++) - { - var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]); - singleCommandBatch.Complete( - moreBatchesExpected: hasMoreCommandSets || commandIndex < batch.ModificationCommands.Count - 1); + for (var commandIndex = 0; commandIndex < batch.ModificationCommands.Count; commandIndex++) + { + var singleCommandBatch = StartNewBatch(parameterNameGenerator, batch.ModificationCommands[commandIndex]); + singleCommandBatch.Complete( + moreBatchesExpected: moreCommandSets || commandIndex < batch.ModificationCommands.Count - 1); - yield return singleCommandBatch; - } + yield return singleCommandBatch; } } - } - private ModificationCommandBatch StartNewBatch( - ParameterNameGenerator parameterNameGenerator, - IReadOnlyModificationCommand modificationCommand) - { - parameterNameGenerator.Reset(); - var batch = Dependencies.ModificationCommandBatchFactory.Create(); - batch.TryAddCommand(modificationCommand); - return batch; + ModificationCommandBatch StartNewBatch( + ParameterNameGenerator? parameterNameGenerator, + IReadOnlyModificationCommand modificationCommand) + { + parameterNameGenerator?.Reset(); + var batch = Dependencies.ModificationCommandBatchFactory.Create(); + batch.TryAddCommand(modificationCommand); + return batch; + } } /// diff --git a/src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs b/src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs index d8d9a828beb..a83d4e2e00a 100644 --- a/src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs +++ b/src/EFCore.SqlServer/Migrations/SqlServerMigrationsSqlGenerator.cs @@ -8,6 +8,7 @@ using Microsoft.EntityFrameworkCore.SqlServer.Internal; using Microsoft.EntityFrameworkCore.SqlServer.Metadata.Internal; using Microsoft.EntityFrameworkCore.SqlServer.Update.Internal; +using Microsoft.EntityFrameworkCore.Update.Internal; // ReSharper disable once CheckNamespace namespace Microsoft.EntityFrameworkCore.Migrations; @@ -33,17 +34,18 @@ public class SqlServerMigrationsSqlGenerator : MigrationsSqlGenerator private IReadOnlyList _operations = null!; private int _variableCounter; + private readonly ICommandBatchPreparer _commandBatchPreparer; + /// /// Creates a new instance. /// /// Parameter object containing dependencies for this service. - /// Provider-specific Migrations annotations to use. + /// The command batch preparer. public SqlServerMigrationsSqlGenerator( MigrationsSqlGeneratorDependencies dependencies, - IRelationalAnnotationProvider migrationsAnnotations) + ICommandBatchPreparer commandBatchPreparer) : base(dependencies) - { - } + => _commandBatchPreparer = commandBatchPreparer; /// /// Generates commands from a list of operations. @@ -1445,10 +1447,14 @@ protected override void Generate( GenerateIdentityInsert(builder, operation, on: true, model); var sqlBuilder = new StringBuilder(); - ((ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator).AppendBulkInsertOperation( - sqlBuilder, - GenerateModificationCommands(operation, model).ToList(), - 0); + + var modificationCommands = GenerateModificationCommands(operation, model).ToList(); + var updateSqlGenerator = (ISqlServerUpdateSqlGenerator)Dependencies.UpdateSqlGenerator; + + foreach (var batch in _commandBatchPreparer.CreateCommandBatches(modificationCommands, moreCommandSets: true)) + { + updateSqlGenerator.AppendBulkInsertOperation(sqlBuilder, batch.ModificationCommands, commandPosition: 0); + } if (Options.HasFlag(MigrationsSqlGenerationOptions.Idempotent)) { diff --git a/test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs b/test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs index 3102fc24af2..b54b3168c06 100644 --- a/test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs +++ b/test/EFCore.SqlServer.FunctionalTests/Migrations/SqlServerMigrationsSqlGeneratorTest.cs @@ -788,6 +788,85 @@ IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name' "); } + [ConditionalFact] + public virtual void InsertDataOperation_max_batch_size_is_respected() + { + // The SQL Server max batch size is 42 by default + var values = new object[50, 1]; + for (var i = 0; i < 50; i++) + { + values[i, 0] = "Foo" + i; + } + + Generate( + CreateGotModel, + new InsertDataOperation + { + Table = "People", + Columns = new[] { "First Name" }, + Values = values + }); + + AssertSql( + @"IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]')) + SET IDENTITY_INSERT [dbo].[People] ON; +INSERT INTO [dbo].[People] ([First Name]) +VALUES (N'Foo0'), +(N'Foo1'), +(N'Foo2'), +(N'Foo3'), +(N'Foo4'), +(N'Foo5'), +(N'Foo6'), +(N'Foo7'), +(N'Foo8'), +(N'Foo9'), +(N'Foo10'), +(N'Foo11'), +(N'Foo12'), +(N'Foo13'), +(N'Foo14'), +(N'Foo15'), +(N'Foo16'), +(N'Foo17'), +(N'Foo18'), +(N'Foo19'), +(N'Foo20'), +(N'Foo21'), +(N'Foo22'), +(N'Foo23'), +(N'Foo24'), +(N'Foo25'), +(N'Foo26'), +(N'Foo27'), +(N'Foo28'), +(N'Foo29'), +(N'Foo30'), +(N'Foo31'), +(N'Foo32'), +(N'Foo33'), +(N'Foo34'), +(N'Foo35'), +(N'Foo36'), +(N'Foo37'), +(N'Foo38'), +(N'Foo39'), +(N'Foo40'), +(N'Foo41'); +INSERT INTO [dbo].[People] ([First Name]) +VALUES (N'Foo42'), +(N'Foo43'), +(N'Foo44'), +(N'Foo45'), +(N'Foo46'), +(N'Foo47'), +(N'Foo48'), +(N'Foo49'); +IF EXISTS (SELECT * FROM [sys].[identity_columns] WHERE [name] IN (N'First Name') AND [object_id] = OBJECT_ID(N'[dbo].[People]')) + SET IDENTITY_INSERT [dbo].[People] OFF; +"); + } + public override void InsertDataOperation_throws_for_unsupported_column_types() => base.InsertDataOperation_throws_for_unsupported_column_types(); @@ -1045,6 +1124,19 @@ public virtual void CreateIndex_generates_exec_when_legacy_filter_and_idempotent "); } + private static void CreateGotModel(ModelBuilder b) + => b.HasDefaultSchema("dbo").Entity( + "Person", pb => + { + pb.ToTable("People"); + pb.Property("FirstName").HasColumnName("First Name"); + pb.Property("LastName").HasColumnName("Last Name"); + pb.Property("Birthplace").HasColumnName("Birthplace"); + pb.Property("Allegiance").HasColumnName("House Allegiance"); + pb.Property("Culture").HasColumnName("Culture"); + pb.HasKey("FirstName", "LastName"); + }); + public SqlServerMigrationsSqlGeneratorTest() : base( SqlServerTestHelpers.Instance,