Skip to content

Commit

Permalink
Take min/max batch size into account again for seeding
Browse files Browse the repository at this point in the history 
Fixes #28876
  • Loading branch information
@roji
roji committed Sep 1, 2022
1 parent 85ef284 commit 5e525f2
Show file tree
Hide file tree
Showing 4 changed files with 212 additions and 163 deletions.
199 changes: 100 additions & 99 deletions src/EFCore.Relational/Migrations/Internal/MigrationsModelDiffer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2190,124 +2190,125 @@ private IEnumerable<MigrationOperation> GetDataOperations(
yield break;
}

var commands = identityMaps.Values.SelectMany(m => m.Rows).Where(r =>
{
return r.EntityState switch
{
EntityState.Added => true,
EntityState.Modified => true,
EntityState.Unchanged => false,
EntityState.Deleted => diffContext.FindDrop(r.Table!) == null,
_ => throw new InvalidOperationException($"Unexpected entity state: {r.EntityState}")
};
});
var commands = identityMaps.Values
.SelectMany(m => m.Rows)
.Where(
r => r.EntityState is EntityState.Added or EntityState.Modified
|| r.EntityState is EntityState.Deleted && diffContext.FindDrop(r.Table!) == null);

var commandSets = new CommandBatchPreparer(CommandBatchPreparerDependencies)
.TopologicalSort(commands);
var commandBatchPreparer = new CommandBatchPreparer(CommandBatchPreparerDependencies);
var commandSets = commandBatchPreparer.TopologicalSort(commands);

foreach (var commandSet in commandSets)
for (var commandSetIndex = 0; commandSetIndex < commandSets.Count; commandSetIndex++)
{
InsertDataOperation? batchInsertOperation = null;
foreach (var command in commandSet)
var hasMoreCommandSets = commandSetIndex < commandSets.Count - 1;

foreach (var batch in commandBatchPreparer.CreateCommandBatches(commandSets[commandSetIndex], hasMoreCommandSets))
{
switch (command.EntityState)
InsertDataOperation? batchInsertOperation = null;

foreach (var command in batch.ModificationCommands)
{
case EntityState.Added:
if (batchInsertOperation != null)
{
if (batchInsertOperation.Table == command.TableName
&& batchInsertOperation.Schema == command.Schema
&& batchInsertOperation.Columns.SequenceEqual(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)))
switch (command.EntityState)
{
case EntityState.Added:
if (batchInsertOperation != null)
{
batchInsertOperation.Values =
AddToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList(),
batchInsertOperation.Values);
continue;
if (batchInsertOperation.Table == command.TableName
&& batchInsertOperation.Schema == command.Schema
&& batchInsertOperation.Columns.SequenceEqual(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)))
{
batchInsertOperation.Values =
AddToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value)
.ToList(),
batchInsertOperation.Values);
continue;
}

yield return batchInsertOperation;
}

yield return batchInsertOperation;
}
if (forSource)
{
Check.DebugFail("Insert using the source model");
break;
}

if (forSource)
{
Check.DebugFail("Insert using the source model");
batchInsertOperation = new InsertDataOperation
{
Schema = command.Schema,
Table = command.TableName,
Columns = command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)
.ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList())
};
break;
}
case EntityState.Modified:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}

batchInsertOperation = new InsertDataOperation
{
Schema = command.Schema,
Table = command.TableName,
Columns = command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.ColumnName)
.ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey || col.IsWrite).Select(col => col.Value).ToList())
};
break;
case EntityState.Modified:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}
if (forSource)
{
Check.DebugFail("Update using the source model");
break;
}

if (forSource)
{
Check.DebugFail("Update using the source model");
yield return new UpdateDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToList()),
Columns = command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.ColumnName).ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.Value).ToList()),
IsDestructiveChange = true
};
break;
}

yield return new UpdateDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToList()),
Columns = command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.ColumnName).ToArray(),
Values = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsWrite).Select(col => col.Value).ToList()),
IsDestructiveChange = true
};
break;
case EntityState.Deleted:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}
case EntityState.Deleted:
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
batchInsertOperation = null;
}

// There shouldn't be any deletes using the target model
Check.DebugAssert(forSource, "Delete using the target model");
// There shouldn't be any deletes using the target model
Check.DebugAssert(forSource, "Delete using the target model");

var keyColumns = command.ColumnModifications.Where(col => col.IsKey)
.Select(c => (IColumn)c.Column!);
var anyKeyColumnDropped = keyColumns.Any(c => diffContext.FindDrop(c) != null);
var keyColumns = command.ColumnModifications.Where(col => col.IsKey)
.Select(c => (IColumn)c.Column!);
var anyKeyColumnDropped = keyColumns.Any(c => diffContext.FindDrop(c) != null);

yield return new DeleteDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyColumnTypes = anyKeyColumnDropped
? keyColumns.Select(col => col.StoreType).ToArray()
: null,
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToArray()),
IsDestructiveChange = true
};
yield return new DeleteDataOperation
{
Schema = command.Schema,
Table = command.TableName,
KeyColumns = command.ColumnModifications.Where(col => col.IsKey).Select(col => col.ColumnName).ToArray(),
KeyColumnTypes = anyKeyColumnDropped
? keyColumns.Select(col => col.StoreType).ToArray()
: null,
KeyValues = ToMultidimensionalArray(
command.ColumnModifications.Where(col => col.IsKey).Select(col => col.Value).ToArray()),
IsDestructiveChange = true
};

break;
default:
throw new InvalidOperationException(command.EntityState.ToString());
break;
default:
throw new InvalidOperationException(command.EntityState.ToString());
}
}
}

if (batchInsertOperation != null)
{
yield return batchInsertOperation;
if (batchInsertOperation != null)
{
yield return batchInsertOperation;
}
}
}
}
Expand Down
13 changes: 13 additions & 0 deletions src/EFCore.Relational/Update/ICommandBatchPreparer.cs
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,4 +34,17 @@ public interface ICommandBatchPreparer
/// <param name="updateAdapter">The model data.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> BatchCommands(IList<IUpdateEntry> entries, IUpdateAdapter updateAdapter);

/// <summary>
/// Given a set of modification commands, returns one more ready-to-execute batches for those commands, taking into account e.g.
/// maximum batch sizes and other batching constraints.
/// </summary>
/// <param name="commandSet">The set of commands to be organized in batches.</param>
/// <param name="moreCommandSets">Whether more command sets are expected after this one within the same save operation.</param>
/// <param name="parameterNameGenerator">An optional parameter name generator.</param>
/// <returns>The list of batches to execute.</returns>
IEnumerable<ModificationCommandBatch> CreateCommandBatches(
List<IReadOnlyModificationCommand> commandSet,
bool moreCommandSets,
ParameterNameGenerator? parameterNameGenerator = null);
}
Loading

0 comments on commit 5e525f2

Please sign in to comment.