From b233f1e47ab128841ae9ec37c9e4ba7a7875eeba Mon Sep 17 00:00:00 2001 From: Shay Rojansky Date: Mon, 15 Aug 2022 03:12:46 +0200 Subject: [PATCH] Fix bugs in SQL Server bulk insert management Fixes #27840 Fixes #28712 --- All.sln.DotSettings | 1 + .../SqlServerModificationCommandBatch.cs | 42 +++++++++++++++---- .../SqlServerModificationCommandBatchTest.cs | 25 ++++++++--- 3 files changed, 54 insertions(+), 14 deletions(-) diff --git a/All.sln.DotSettings b/All.sln.DotSettings index 4e7bf4fbf52..cb5862cb1a8 100644 --- a/All.sln.DotSettings +++ b/All.sln.DotSettings @@ -320,6 +320,7 @@ The .NET Foundation licenses this file to you under the MIT license. True True True + True True True True diff --git a/src/EFCore.SqlServer/Update/Internal/SqlServerModificationCommandBatch.cs b/src/EFCore.SqlServer/Update/Internal/SqlServerModificationCommandBatch.cs index e26533d030b..d1056037bb0 100644 --- a/src/EFCore.SqlServer/Update/Internal/SqlServerModificationCommandBatch.cs +++ b/src/EFCore.SqlServer/Update/Internal/SqlServerModificationCommandBatch.cs @@ -1,7 +1,6 @@ // Licensed to the .NET Foundation under one or more agreements. // The .NET Foundation licenses this file to you under the MIT license. -using System.Collections; using Microsoft.Data.SqlClient; using Microsoft.EntityFrameworkCore.SqlServer.Internal; @@ -15,9 +14,15 @@ namespace Microsoft.EntityFrameworkCore.SqlServer.Update.Internal; /// public class SqlServerModificationCommandBatch : AffectedCountModificationCommandBatch { + // https://docs.microsoft.com/sql/sql-server/maximum-capacity-specifications-for-sql-server private const int DefaultNetworkPacketSizeBytes = 4096; private const int MaxScriptLength = 65536 * DefaultNetworkPacketSizeBytes / 2; - private const int MaxParameterCount = 2100; + + /// + /// The SQL Server limit on parameters, including two extra parameters to sp_executesql (@stmt and @params). + /// + private const int MaxParameterCount = 2100 - 2; + private readonly List _pendingBulkInsertCommands = new(); /// @@ -60,7 +65,6 @@ protected override void RollbackLastCommand() if (_pendingBulkInsertCommands.Count > 0) { _pendingBulkInsertCommands.RemoveAt(_pendingBulkInsertCommands.Count - 1); - return; } base.RollbackLastCommand(); @@ -73,9 +77,30 @@ protected override void RollbackLastCommand() /// doing so can result in application failures when updating to a new Entity Framework Core release. /// protected override bool IsValid() - => SqlBuilder.Length < MaxScriptLength - // A single implicit parameter for the command text itself - && ParameterValues.Count + 1 < MaxParameterCount; + { + if (ParameterValues.Count > MaxParameterCount) + { + return false; + } + + var sqlLength = SqlBuilder.Length; + + if (_pendingBulkInsertCommands.Count > 0) + { + // Conservative heuristic for the length of the pending bulk insert commands. + // See EXEC sp_server_info. + var numColumns = _pendingBulkInsertCommands[0].ColumnModifications.Count; + + sqlLength += + numColumns * 128 // column name lengths + + 128 // schema name length + + 128 // table name length + + _pendingBulkInsertCommands.Count * numColumns * 6 // column parameter placeholders + + 300; // some extra fixed overhead + } + + return sqlLength < MaxScriptLength; + } private void ApplyPendingBulkInsertCommands() { @@ -93,10 +118,8 @@ private void ApplyPendingBulkInsertCommands() SetRequiresTransaction(!wasCachedCommandTextEmpty || requiresTransaction); - foreach (var pendingCommand in _pendingBulkInsertCommands) + for (var i = 0; i < _pendingBulkInsertCommands.Count; i++) { - AddParameters(pendingCommand); - CommandResultSet.Add(resultSetMapping); } @@ -126,6 +149,7 @@ protected override void AddCommand(IReadOnlyModificationCommand modificationComm } _pendingBulkInsertCommands.Add(modificationCommand); + AddParameters(modificationCommand); } else { diff --git a/test/EFCore.SqlServer.Tests/Update/SqlServerModificationCommandBatchTest.cs b/test/EFCore.SqlServer.Tests/Update/SqlServerModificationCommandBatchTest.cs index 6b9ee9b6b43..873a6e4b4f8 100644 --- a/test/EFCore.SqlServer.Tests/Update/SqlServerModificationCommandBatchTest.cs +++ b/test/EFCore.SqlServer.Tests/Update/SqlServerModificationCommandBatchTest.cs @@ -11,20 +11,33 @@ namespace Microsoft.EntityFrameworkCore.Update; public class SqlServerModificationCommandBatchTest { - [ConditionalFact] - public void AddCommand_returns_false_when_max_batch_size_is_reached() + [ConditionalTheory] + [InlineData(EntityState.Added)] + [InlineData(EntityState.Deleted)] + [InlineData(EntityState.Modified)] + public void AddCommand_returns_false_when_max_batch_size_is_reached(EntityState entityState) { var batch = CreateBatch(maxBatchSize: 1); var firstCommand = CreateModificationCommand("T1", null, false); + firstCommand.EntityState = entityState; + var secondCommand = CreateModificationCommand("T1", null, false); + secondCommand.EntityState = entityState; + Assert.True(batch.TryAddCommand(firstCommand)); - Assert.False(batch.TryAddCommand(CreateModificationCommand("T1", null, false))); + Assert.False(batch.TryAddCommand(secondCommand)); Assert.Same(firstCommand, Assert.Single(batch.ModificationCommands)); } - [ConditionalFact] - public void AddCommand_returns_false_when_max_parameters_are_reached() + [ConditionalTheory] + [InlineData(EntityState.Added, true)] + [InlineData(EntityState.Added, false)] + [InlineData(EntityState.Deleted, true)] + [InlineData(EntityState.Deleted, false)] + [InlineData(EntityState.Modified, true)] + [InlineData(EntityState.Modified, false)] + public void AddCommand_returns_false_when_max_parameters_are_reached(EntityState entityState, bool withSameTable) { var typeMapper = CreateTypeMappingSource(); var intMapping = typeMapper.FindMapping(typeof(int)); @@ -33,6 +46,7 @@ public void AddCommand_returns_false_when_max_parameters_are_reached() var batch = CreateBatch(); var command = CreateModificationCommand("T1", null, false); + command.EntityState = entityState; for (var i = 0; i < 2098; i++) { command.AddColumnModification(CreateModificationParameters("col" + i)); @@ -40,6 +54,7 @@ public void AddCommand_returns_false_when_max_parameters_are_reached() Assert.True(batch.TryAddCommand(command)); var secondCommand = CreateModificationCommand("T2", null, false); + secondCommand.EntityState = entityState; secondCommand.AddColumnModification(CreateModificationParameters("col")); Assert.False(batch.TryAddCommand(secondCommand)); Assert.Same(command, Assert.Single(batch.ModificationCommands));