From e23156f7a5f7fb525f71f44a13d29d8c0142a36e Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Thu, 7 Aug 2025 10:09:12 +0200 Subject: [PATCH 1/7] Issue #31819 "DetectChanges slow in long-term contexts" is no longer necessary if all entity classes in the model are defined with a change-tracking strategy other than "ChangeTrackingStrategy.Snapshot." This is controlled in the ProcessModelFinalizing method of the ChangeTrackingStrategyConvention class. DetectChanges is therefore not called when saving because change tracking is performed using the EntityReferenceMap (_entityReferenceMap property), which essentially corresponds to the previous caching mechanism using EntityState dictionaries from EF4/5/6. However, querying this cache was not enabled; instead, users were forced to use ChangeTracker.Entries(), which returned all objects and led to performance problems with large long-term contexts. This change now enables fast access to changed objects using the new GetEntriesForState method in the ChangeTracker class. This, in turn, calls EntityReferenceMap.GetEntriesForState(), which already exists. For fast change tracking, implement INotifyProperty-Changed on your entity classes and activate this strategy in the Model Builder using ChangeTrackingStrategy.ChangedNotifications via HasChangeTrackingStrategy(). https://github.com/dotnet/efcore/issues/31819 --- src/EFCore/ChangeTracking/ChangeTracker.cs | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/EFCore/ChangeTracking/ChangeTracker.cs b/src/EFCore/ChangeTracking/ChangeTracker.cs index 4f0a8134c8a..d2a6064cf8c 100644 --- a/src/EFCore/ChangeTracking/ChangeTracker.cs +++ b/src/EFCore/ChangeTracking/ChangeTracker.cs @@ -215,6 +215,44 @@ public virtual IEnumerable> Entries() .Select(e => new EntityEntry(e)); } + /// + /// Returns tracked entities that are in a given state from a fast cache. + /// + /// Entities in EntityState.Added state + /// Entities in Modified.Added state + /// Entities in Modified.Deleted state + /// Entities in Modified.Unchanged state + /// An entry for each entity that matched the search criteria. + public IEnumerable GetEntriesForState( + bool added = false, + bool modified = false, + bool deleted = false, + bool unchanged = false) + { + return StateManager.GetEntriesForState(added, modified, deleted, unchanged) + .Select(e => new EntityEntry(e)); + } + + /// + /// Returns tracked entities that are in a given state from a fast cache. + /// + /// Entities in EntityState.Added state + /// Entities in Modified.Added state + /// Entities in Modified.Deleted state + /// Entities in Modified.Unchanged state + /// An entry for each entity that matched the search criteria. + public IEnumerable> GetEntriesForState( + bool added = false, + bool modified = false, + bool deleted = false, + bool unchanged = false) + where TEntity : class + { + return StateManager.GetEntriesForState(added, modified, deleted, unchanged) + .Where(e => e.Entity is TEntity) + .Select(e => new EntityEntry(e)); + } + private void TryDetectChanges() { if (AutoDetectChangesEnabled) From 82f4b663c72e804d05a33d912ccc1d893a8487a5 Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Sat, 9 Aug 2025 13:23:59 +0200 Subject: [PATCH 2/7] #16491: Implemented legacy Merge-Option Feature from EF/4/5/6 https://github.com/dotnet/efcore/issues/16491 --- .../Internal/InternalEntityEntry.cs | 24 ++++ .../EntityFrameworkQueryableExtensions.cs | 31 +++++ src/EFCore/MergeOption.cs | 27 +++++ ...yableMethodNormalizingExpressionVisitor.cs | 8 ++ src/EFCore/Query/QueryCompilationContext.cs | 5 + .../ShapedQueryCompilingExpressionVisitor.cs | 111 +++++++++++++++++- 6 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/EFCore/MergeOption.cs diff --git a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs index ac9758001ed..1d6bc853152 100644 --- a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs +++ b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs @@ -1180,6 +1180,30 @@ public void SetOriginalValue( } } + /// + /// Refreshes the property value with the value from the database + /// + /// Property + /// New value from database + /// MergeOption + /// Sets the EntityState to Unchanged if MergeOption.OverwriteChanges else calls ChangeDetector to determine changes + public void ReloadValue(IPropertyBase propertyBase, object? value, MergeOption mergeOption, bool updateEntityState) + { + var property = (IProperty)propertyBase; + EnsureOriginalValues(); + bool isModified = IsModified(property); + _originalValues.SetValue(property, value, -1); + if (mergeOption == MergeOption.OverwriteChanges || !isModified) + SetProperty(propertyBase, value, isMaterialization: true, setModified: false); + if (updateEntityState) + { + if (mergeOption == MergeOption.OverwriteChanges) + SetEntityState(EntityState.Unchanged); + else + ((StateManager as StateManager)?.ChangeDetector as ChangeDetector)?.DetectChanges(this); + } + } + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in diff --git a/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs b/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs index 088c5193dcf..9c7dff1f9e3 100644 --- a/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs +++ b/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs @@ -2848,6 +2848,37 @@ public static IQueryable AsTracking( #endregion + #region Refreshing + + internal static readonly MethodInfo RefreshMethodInfo + = typeof(EntityFrameworkQueryableExtensions).GetMethod( + nameof(Refresh), [typeof(IQueryable<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), typeof(MergeOption)])!; + + + /// + /// Specifies that the current Entity Framework LINQ query should refresh already loaded objects with the specified merge option. + /// + /// The type of entity being queried. + /// The source query. + /// The MergeOption + /// A new query annotated with the given tag. + public static IQueryable Refresh( + this IQueryable source, + [NotParameterized] MergeOption mergeOption) + { + return + source.Provider is EntityQueryProvider + ? source.Provider.CreateQuery( + Expression.Call( + instance: null, + method: RefreshMethodInfo.MakeGenericMethod(typeof(T)), + arg0: source.Expression, + arg1: Expression.Constant(mergeOption))) + : source; + } + + #endregion + #region Tagging internal static readonly MethodInfo TagWithMethodInfo diff --git a/src/EFCore/MergeOption.cs b/src/EFCore/MergeOption.cs new file mode 100644 index 00000000000..a8eecb143af --- /dev/null +++ b/src/EFCore/MergeOption.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore; + +/// +/// The different ways that new objects loaded from the database can be merged with existing objects already in memory. +/// +public enum MergeOption +{ + /// + /// Will only append new (top level-unique) rows. This is the default behavior. + /// + AppendOnly = 0, + + /// + /// The incoming values for this row will be written to both the current value and + /// the original value versions of the data for each column. + /// + OverwriteChanges = 1, + + /// + /// The incoming values for this row will be written to the original value version + /// of each column. The current version of the data in each column will not be changed. + /// + PreserveChanges = 2 +} diff --git a/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs b/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs index 7a43eab9d98..80cf18888bf 100644 --- a/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs +++ b/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs @@ -339,6 +339,14 @@ private static void VerifyReturnType(Expression expression, ParameterExpression return visitedExpression; } + if (genericMethodDefinition == EntityFrameworkQueryableExtensions.RefreshMethodInfo) + { + var visitedExpression = Visit(methodCallExpression.Arguments[0]); + _queryCompilationContext.RefreshMergeOption = methodCallExpression.Arguments[1].GetConstantValue(); + + return visitedExpression; + } + return null; } diff --git a/src/EFCore/Query/QueryCompilationContext.cs b/src/EFCore/Query/QueryCompilationContext.cs index 651cefa1341..2159500978d 100644 --- a/src/EFCore/Query/QueryCompilationContext.cs +++ b/src/EFCore/Query/QueryCompilationContext.cs @@ -177,6 +177,11 @@ public QueryCompilationContext( /// public virtual bool IgnoreAutoIncludes { get; internal set; } + /// + /// A value indicating how already loaded objects should be merged and refreshed with the results of this query. + /// + public virtual MergeOption RefreshMergeOption { get; internal set; } + /// /// The set of tags applied to this query. /// diff --git a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs index fbc596366bf..ba73a1f66d9 100644 --- a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs +++ b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs @@ -58,7 +58,8 @@ protected ShapedQueryCompilingExpressionVisitor( dependencies.EntityMaterializerSource, dependencies.LiftableConstantFactory, queryCompilationContext.QueryTrackingBehavior, - queryCompilationContext.SupportsPrecompiledQuery); + queryCompilationContext.SupportsPrecompiledQuery, + queryCompilationContext.RefreshMergeOption); _constantVerifyingExpressionVisitor = new ConstantVerifyingExpressionVisitor(dependencies.TypeMappingSource); _materializationConditionConstantLifter = new MaterializationConditionConstantLifter(dependencies.LiftableConstantFactory); @@ -358,7 +359,8 @@ private sealed class EntityMaterializerInjectingExpressionVisitor( IEntityMaterializerSource entityMaterializerSource, ILiftableConstantFactory liftableConstantFactory, QueryTrackingBehavior queryTrackingBehavior, - bool supportsPrecompiledQuery) + bool supportsPrecompiledQuery, + MergeOption mergeOption) : ExpressionVisitor { private static readonly ConstructorInfo MaterializationContextConstructor @@ -391,6 +393,8 @@ private static readonly MethodInfo CreateNullKeyValueInNoTrackingQueryMethod private readonly bool _queryStateManager = queryTrackingBehavior is QueryTrackingBehavior.TrackAll or QueryTrackingBehavior.NoTrackingWithIdentityResolution; + private readonly MergeOption _MergeOption = mergeOption; + private readonly ISet _visitedEntityTypes = new HashSet(); private readonly MaterializationConditionConstantLifter _materializationConditionConstantLifter = new(liftableConstantFactory); private int _currentEntityIndex; @@ -505,7 +509,15 @@ private Expression ProcessEntityShaper(StructuralTypeShaperExpression shaper) Assign( instanceVariable, Convert( MakeMemberAccess(entryVariable, EntityMemberInfo), - clrType))), + clrType)), + // Update the existing entity with new property values from the database + // if the merge option is not AppendOnly + _MergeOption != MergeOption.AppendOnly + ? UpdateExistingEntityWithDatabaseValues( + entryVariable, + concreteEntityTypeVariable, + materializationContextVariable, + shaper) : Empty()), MaterializeEntity( shaper, materializationContextVariable, concreteEntityTypeVariable, instanceVariable, entryVariable)))); @@ -741,5 +753,98 @@ private BlockExpression CreateFullMaterializeExpression( return Block(blockExpressions); } + + /// + /// Creates an expression to update an existing tracked entity with values from the database, + /// similar to the EntityEntry.Reload() method. + /// + /// The variable representing the existing InternalEntityEntry. + /// The variable representing the concrete entity type. + /// The materialization context variable. + /// The structural type shaper expression. + /// An expression that updates the existing entity with database values. + private Expression UpdateExistingEntityWithDatabaseValues( + ParameterExpression entryVariable, + ParameterExpression concreteEntityTypeVariable, + ParameterExpression materializationContextVariable, + StructuralTypeShaperExpression shaper) + { + var updateExpressions = new List(); + var typeBase = shaper.StructuralType; + + if (typeBase is not IEntityType entityType) + { + // For complex types, we don't update existing instances + return Empty(); + } + + var valueBufferExpression = Call(materializationContextVariable, MaterializationContext.GetValueBufferMethod); + + // Get all properties to update (exclude key properties which should not change) + var propertiesToUpdate = entityType.GetProperties() + .Where(p => !p.IsPrimaryKey()) + .ToList(); + + var setReloadValueMethod = typeof(InternalEntityEntry) + .GetMethod(nameof(InternalEntityEntry.ReloadValue), new[] { typeof(IPropertyBase), typeof(object), typeof(MergeOption), typeof(bool) })!; + + // Update original values similar to EntityEntry.Reload() + // This ensures that the original values snapshot reflects the database state + var dbProperties = propertiesToUpdate.Where(p => !p.IsShadowProperty()); + int count = dbProperties.Count(); + int i = 0; + foreach (var property in dbProperties) + { + i++; + var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( + property.ClrType, + property.GetIndex(), + property); + + var setOriginalValueExpression = Call( + entryVariable, + setReloadValueMethod, + Constant(property), + property.ClrType.IsValueType && property.IsNullable + ? (Expression)Convert(newValue, typeof(object)) + : Convert(newValue, typeof(object)), + Constant(_MergeOption), + Constant(i == count)); + + updateExpressions.Add(setOriginalValueExpression); + } + + //foreach (var property in propertiesToUpdate) + //{ + // // Create expression to read the new value from the database + // var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( + // property.ClrType, + // property.GetIndex(), + // property); + + // var setSetPropertyMethod = typeof(InternalEntityEntry) + // .GetMethod(nameof(InternalEntityEntry.SetProperty), new[] { typeof(IPropertyBase), typeof(object), typeof(bool), typeof(bool), typeof(bool) })!; + + // // Create expression to set the property on the existing entity + // // This mimics what EntityEntry.Reload() does: entry[property] = newValue + // var setPropertyExpression = Call( + // entryVariable, + // setSetPropertyMethod, + // Constant(property), + // property.ClrType.IsValueType && property.IsNullable + // ? (Expression)Convert(newValue, typeof(object)) + // : Convert(newValue, typeof(object)), + // Constant(false), // isMaterialization + // Constant(false), // setModified + // Constant(false)); // isCascadeDelete + + // updateExpressions.Add(setPropertyExpression); + //} + + + return updateExpressions.Count > 0 + ? (Expression)Block(updateExpressions) + : Empty(); + } } } From eb1d8580b844214c9cca4ebcda1939da8f16f060 Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Tue, 12 Aug 2025 12:21:36 +0200 Subject: [PATCH 3/7] * #16491: Implemented Merge-Option Feature from EF/4/5/6 tested with iPlus https://github.com/dotnet/efcore/issues/16491 --- .../ShapedQueryCompilingExpressionVisitor.cs | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs index ba73a1f66d9..0aae053c1bf 100644 --- a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs +++ b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs @@ -814,34 +814,6 @@ private Expression UpdateExistingEntityWithDatabaseValues( updateExpressions.Add(setOriginalValueExpression); } - //foreach (var property in propertiesToUpdate) - //{ - // // Create expression to read the new value from the database - // var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( - // property.ClrType, - // property.GetIndex(), - // property); - - // var setSetPropertyMethod = typeof(InternalEntityEntry) - // .GetMethod(nameof(InternalEntityEntry.SetProperty), new[] { typeof(IPropertyBase), typeof(object), typeof(bool), typeof(bool), typeof(bool) })!; - - // // Create expression to set the property on the existing entity - // // This mimics what EntityEntry.Reload() does: entry[property] = newValue - // var setPropertyExpression = Call( - // entryVariable, - // setSetPropertyMethod, - // Constant(property), - // property.ClrType.IsValueType && property.IsNullable - // ? (Expression)Convert(newValue, typeof(object)) - // : Convert(newValue, typeof(object)), - // Constant(false), // isMaterialization - // Constant(false), // setModified - // Constant(false)); // isCascadeDelete - - // updateExpressions.Add(setPropertyExpression); - //} - - return updateExpressions.Count > 0 ? (Expression)Block(updateExpressions) : Empty(); From 5d5d30dcb13202e25b096bc98f2aa115323f6746 Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Thu, 7 Aug 2025 10:09:12 +0200 Subject: [PATCH 4/7] Issue #31819 "DetectChanges slow in long-term contexts" is no longer necessary if all entity classes in the model are defined with a change-tracking strategy other than "ChangeTrackingStrategy.Snapshot." This is controlled in the ProcessModelFinalizing method of the ChangeTrackingStrategyConvention class. DetectChanges is therefore not called when saving because change tracking is performed using the EntityReferenceMap (_entityReferenceMap property), which essentially corresponds to the previous caching mechanism using EntityState dictionaries from EF4/5/6. However, querying this cache was not enabled; instead, users were forced to use ChangeTracker.Entries(), which returned all objects and led to performance problems with large long-term contexts. This change now enables fast access to changed objects using the new GetEntriesForState method in the ChangeTracker class. This, in turn, calls EntityReferenceMap.GetEntriesForState(), which already exists. For fast change tracking, implement INotifyProperty-Changed on your entity classes and activate this strategy in the Model Builder using ChangeTrackingStrategy.ChangedNotifications via HasChangeTrackingStrategy(). https://github.com/dotnet/efcore/issues/31819 --- src/EFCore/ChangeTracking/ChangeTracker.cs | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/src/EFCore/ChangeTracking/ChangeTracker.cs b/src/EFCore/ChangeTracking/ChangeTracker.cs index 76f5400f052..6dfd833d7f4 100644 --- a/src/EFCore/ChangeTracking/ChangeTracker.cs +++ b/src/EFCore/ChangeTracking/ChangeTracker.cs @@ -215,6 +215,44 @@ public virtual IEnumerable> Entries() .Select(e => new EntityEntry(e)); } + /// + /// Returns tracked entities that are in a given state from a fast cache. + /// + /// Entities in EntityState.Added state + /// Entities in Modified.Added state + /// Entities in Modified.Deleted state + /// Entities in Modified.Unchanged state + /// An entry for each entity that matched the search criteria. + public IEnumerable GetEntriesForState( + bool added = false, + bool modified = false, + bool deleted = false, + bool unchanged = false) + { + return StateManager.GetEntriesForState(added, modified, deleted, unchanged) + .Select(e => new EntityEntry(e)); + } + + /// + /// Returns tracked entities that are in a given state from a fast cache. + /// + /// Entities in EntityState.Added state + /// Entities in Modified.Added state + /// Entities in Modified.Deleted state + /// Entities in Modified.Unchanged state + /// An entry for each entity that matched the search criteria. + public IEnumerable> GetEntriesForState( + bool added = false, + bool modified = false, + bool deleted = false, + bool unchanged = false) + where TEntity : class + { + return StateManager.GetEntriesForState(added, modified, deleted, unchanged) + .Where(e => e.Entity is TEntity) + .Select(e => new EntityEntry(e)); + } + private void TryDetectChanges() { if (AutoDetectChangesEnabled) From ba5329b5b52971a428446c85621985d102229361 Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Sat, 9 Aug 2025 13:23:59 +0200 Subject: [PATCH 5/7] #16491: Implemented legacy Merge-Option Feature from EF/4/5/6 https://github.com/dotnet/efcore/issues/16491 --- .../Internal/InternalEntityEntry.cs | 24 ++++ .../EntityFrameworkQueryableExtensions.cs | 31 +++++ src/EFCore/MergeOption.cs | 27 +++++ ...yableMethodNormalizingExpressionVisitor.cs | 8 ++ src/EFCore/Query/QueryCompilationContext.cs | 5 + .../ShapedQueryCompilingExpressionVisitor.cs | 111 +++++++++++++++++- 6 files changed, 203 insertions(+), 3 deletions(-) create mode 100644 src/EFCore/MergeOption.cs diff --git a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs index 411e875575a..9dfa822ece6 100644 --- a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs +++ b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs @@ -526,6 +526,30 @@ public TProperty GetRelationshipSnapshotValue(IPropertyBase propertyB => _relationshipsSnapshot.GetValue(this, propertyBase); + /// + /// Refreshes the property value with the value from the database + /// + /// Property + /// New value from database + /// MergeOption + /// Sets the EntityState to Unchanged if MergeOption.OverwriteChanges else calls ChangeDetector to determine changes + public void ReloadValue(IPropertyBase propertyBase, object? value, MergeOption mergeOption, bool updateEntityState) + { + var property = (IProperty)propertyBase; + EnsureOriginalValues(); + bool isModified = IsModified(property); + _originalValues.SetValue(property, value, -1); + if (mergeOption == MergeOption.OverwriteChanges || !isModified) + SetProperty(propertyBase, value, isMaterialization: true, setModified: false); + if (updateEntityState) + { + if (mergeOption == MergeOption.OverwriteChanges) + SetEntityState(EntityState.Unchanged); + else + ((StateManager as StateManager)?.ChangeDetector as ChangeDetector)?.DetectChanges(this); + } + } + /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in diff --git a/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs b/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs index 1fcda68fe55..dd40dc9839b 100644 --- a/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs +++ b/src/EFCore/Extensions/EntityFrameworkQueryableExtensions.cs @@ -2883,6 +2883,37 @@ public static IQueryable AsTracking( #endregion + #region Refreshing + + internal static readonly MethodInfo RefreshMethodInfo + = typeof(EntityFrameworkQueryableExtensions).GetMethod( + nameof(Refresh), [typeof(IQueryable<>).MakeGenericType(Type.MakeGenericMethodParameter(0)), typeof(MergeOption)])!; + + + /// + /// Specifies that the current Entity Framework LINQ query should refresh already loaded objects with the specified merge option. + /// + /// The type of entity being queried. + /// The source query. + /// The MergeOption + /// A new query annotated with the given tag. + public static IQueryable Refresh( + this IQueryable source, + [NotParameterized] MergeOption mergeOption) + { + return + source.Provider is EntityQueryProvider + ? source.Provider.CreateQuery( + Expression.Call( + instance: null, + method: RefreshMethodInfo.MakeGenericMethod(typeof(T)), + arg0: source.Expression, + arg1: Expression.Constant(mergeOption))) + : source; + } + + #endregion + #region Tagging internal static readonly MethodInfo TagWithMethodInfo diff --git a/src/EFCore/MergeOption.cs b/src/EFCore/MergeOption.cs new file mode 100644 index 00000000000..a8eecb143af --- /dev/null +++ b/src/EFCore/MergeOption.cs @@ -0,0 +1,27 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +namespace Microsoft.EntityFrameworkCore; + +/// +/// The different ways that new objects loaded from the database can be merged with existing objects already in memory. +/// +public enum MergeOption +{ + /// + /// Will only append new (top level-unique) rows. This is the default behavior. + /// + AppendOnly = 0, + + /// + /// The incoming values for this row will be written to both the current value and + /// the original value versions of the data for each column. + /// + OverwriteChanges = 1, + + /// + /// The incoming values for this row will be written to the original value version + /// of each column. The current version of the data in each column will not be changed. + /// + PreserveChanges = 2 +} diff --git a/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs b/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs index 80dbb37f040..16affea1898 100644 --- a/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs +++ b/src/EFCore/Query/Internal/QueryableMethodNormalizingExpressionVisitor.cs @@ -365,6 +365,14 @@ private static void VerifyReturnType(Expression expression, ParameterExpression return visitedExpression; } + if (genericMethodDefinition == EntityFrameworkQueryableExtensions.RefreshMethodInfo) + { + var visitedExpression = Visit(methodCallExpression.Arguments[0]); + _queryCompilationContext.RefreshMergeOption = methodCallExpression.Arguments[1].GetConstantValue(); + + return visitedExpression; + } + return null; } diff --git a/src/EFCore/Query/QueryCompilationContext.cs b/src/EFCore/Query/QueryCompilationContext.cs index 88545f5ee40..2fe1010d613 100644 --- a/src/EFCore/Query/QueryCompilationContext.cs +++ b/src/EFCore/Query/QueryCompilationContext.cs @@ -138,6 +138,11 @@ public QueryCompilationContext(QueryCompilationContextDependencies dependencies, /// public virtual bool IgnoreAutoIncludes { get; internal set; } + /// + /// A value indicating how already loaded objects should be merged and refreshed with the results of this query. + /// + public virtual MergeOption RefreshMergeOption { get; internal set; } + /// /// The set of tags applied to this query. /// diff --git a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs index 3dba903bf1f..ca0969de88e 100644 --- a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs +++ b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs @@ -59,7 +59,8 @@ protected ShapedQueryCompilingExpressionVisitor( dependencies.EntityMaterializerSource, dependencies.LiftableConstantFactory, queryCompilationContext.QueryTrackingBehavior, - queryCompilationContext.SupportsPrecompiledQuery); + queryCompilationContext.SupportsPrecompiledQuery, + queryCompilationContext.RefreshMergeOption); _constantVerifyingExpressionVisitor = new ConstantVerifyingExpressionVisitor(dependencies.TypeMappingSource); _materializationConditionConstantLifter = new MaterializationConditionConstantLifter(dependencies.LiftableConstantFactory); @@ -378,7 +379,8 @@ private sealed class StructuralTypeMaterializerInjector( IStructuralTypeMaterializerSource materializerSource, ILiftableConstantFactory liftableConstantFactory, QueryTrackingBehavior queryTrackingBehavior, - bool supportsPrecompiledQuery) + bool supportsPrecompiledQuery, + MergeOption mergeOption) : ExpressionVisitor { private static readonly ConstructorInfo MaterializationContextConstructor @@ -411,6 +413,8 @@ private static readonly MethodInfo CreateNullKeyValueInNoTrackingQueryMethod private readonly bool _queryStateManager = queryTrackingBehavior is QueryTrackingBehavior.TrackAll or QueryTrackingBehavior.NoTrackingWithIdentityResolution; + private readonly MergeOption _MergeOption = mergeOption; + private readonly ISet _visitedEntityTypes = new HashSet(); private readonly MaterializationConditionConstantLifter _materializationConditionConstantLifter = new(liftableConstantFactory); private int _currentEntityIndex; @@ -525,7 +529,15 @@ private Expression ProcessStructuralTypeShaper(StructuralTypeShaperExpression sh Assign( instanceVariable, Convert( MakeMemberAccess(entryVariable, EntityMemberInfo), - clrType))), + clrType)), + // Update the existing entity with new property values from the database + // if the merge option is not AppendOnly + _MergeOption != MergeOption.AppendOnly + ? UpdateExistingEntityWithDatabaseValues( + entryVariable, + concreteEntityTypeVariable, + materializationContextVariable, + shaper) : Empty()), MaterializeEntity( shaper, materializationContextVariable, concreteEntityTypeVariable, instanceVariable, entryVariable)))); @@ -764,5 +776,98 @@ private BlockExpression CreateFullMaterializeExpression( return Block(blockExpressions); } + + /// + /// Creates an expression to update an existing tracked entity with values from the database, + /// similar to the EntityEntry.Reload() method. + /// + /// The variable representing the existing InternalEntityEntry. + /// The variable representing the concrete entity type. + /// The materialization context variable. + /// The structural type shaper expression. + /// An expression that updates the existing entity with database values. + private Expression UpdateExistingEntityWithDatabaseValues( + ParameterExpression entryVariable, + ParameterExpression concreteEntityTypeVariable, + ParameterExpression materializationContextVariable, + StructuralTypeShaperExpression shaper) + { + var updateExpressions = new List(); + var typeBase = shaper.StructuralType; + + if (typeBase is not IEntityType entityType) + { + // For complex types, we don't update existing instances + return Empty(); + } + + var valueBufferExpression = Call(materializationContextVariable, MaterializationContext.GetValueBufferMethod); + + // Get all properties to update (exclude key properties which should not change) + var propertiesToUpdate = entityType.GetProperties() + .Where(p => !p.IsPrimaryKey()) + .ToList(); + + var setReloadValueMethod = typeof(InternalEntityEntry) + .GetMethod(nameof(InternalEntityEntry.ReloadValue), new[] { typeof(IPropertyBase), typeof(object), typeof(MergeOption), typeof(bool) })!; + + // Update original values similar to EntityEntry.Reload() + // This ensures that the original values snapshot reflects the database state + var dbProperties = propertiesToUpdate.Where(p => !p.IsShadowProperty()); + int count = dbProperties.Count(); + int i = 0; + foreach (var property in dbProperties) + { + i++; + var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( + property.ClrType, + property.GetIndex(), + property); + + var setOriginalValueExpression = Call( + entryVariable, + setReloadValueMethod, + Constant(property), + property.ClrType.IsValueType && property.IsNullable + ? (Expression)Convert(newValue, typeof(object)) + : Convert(newValue, typeof(object)), + Constant(_MergeOption), + Constant(i == count)); + + updateExpressions.Add(setOriginalValueExpression); + } + + //foreach (var property in propertiesToUpdate) + //{ + // // Create expression to read the new value from the database + // var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( + // property.ClrType, + // property.GetIndex(), + // property); + + // var setSetPropertyMethod = typeof(InternalEntityEntry) + // .GetMethod(nameof(InternalEntityEntry.SetProperty), new[] { typeof(IPropertyBase), typeof(object), typeof(bool), typeof(bool), typeof(bool) })!; + + // // Create expression to set the property on the existing entity + // // This mimics what EntityEntry.Reload() does: entry[property] = newValue + // var setPropertyExpression = Call( + // entryVariable, + // setSetPropertyMethod, + // Constant(property), + // property.ClrType.IsValueType && property.IsNullable + // ? (Expression)Convert(newValue, typeof(object)) + // : Convert(newValue, typeof(object)), + // Constant(false), // isMaterialization + // Constant(false), // setModified + // Constant(false)); // isCascadeDelete + + // updateExpressions.Add(setPropertyExpression); + //} + + + return updateExpressions.Count > 0 + ? (Expression)Block(updateExpressions) + : Empty(); + } } } From 81450c472fb35c43f7295f53ff4dc798520cf735 Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Tue, 12 Aug 2025 12:21:36 +0200 Subject: [PATCH 6/7] * #16491: Implemented Merge-Option Feature from EF/4/5/6 tested with iPlus https://github.com/dotnet/efcore/issues/16491 --- .../ShapedQueryCompilingExpressionVisitor.cs | 28 ------------------- 1 file changed, 28 deletions(-) diff --git a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs index ca0969de88e..020d8f3adfd 100644 --- a/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs +++ b/src/EFCore/Query/ShapedQueryCompilingExpressionVisitor.cs @@ -837,34 +837,6 @@ private Expression UpdateExistingEntityWithDatabaseValues( updateExpressions.Add(setOriginalValueExpression); } - //foreach (var property in propertiesToUpdate) - //{ - // // Create expression to read the new value from the database - // var newValue = valueBufferExpression.CreateValueBufferReadValueExpression( - // property.ClrType, - // property.GetIndex(), - // property); - - // var setSetPropertyMethod = typeof(InternalEntityEntry) - // .GetMethod(nameof(InternalEntityEntry.SetProperty), new[] { typeof(IPropertyBase), typeof(object), typeof(bool), typeof(bool), typeof(bool) })!; - - // // Create expression to set the property on the existing entity - // // This mimics what EntityEntry.Reload() does: entry[property] = newValue - // var setPropertyExpression = Call( - // entryVariable, - // setSetPropertyMethod, - // Constant(property), - // property.ClrType.IsValueType && property.IsNullable - // ? (Expression)Convert(newValue, typeof(object)) - // : Convert(newValue, typeof(object)), - // Constant(false), // isMaterialization - // Constant(false), // setModified - // Constant(false)); // isCascadeDelete - - // updateExpressions.Add(setPropertyExpression); - //} - - return updateExpressions.Count > 0 ? (Expression)Block(updateExpressions) : Empty(); From 7e131d90bfae968ddca4c1a00712575e5444c0fe Mon Sep 17 00:00:00 2001 From: Damir Lisak Date: Fri, 15 Aug 2025 13:40:32 +0200 Subject: [PATCH 7/7] Incompatibility fixed after rebasing to 10.0 regarding MergeOption #36556 --- .../Internal/InternalEntityEntry.cs | 49 ------------------- .../Internal/InternalEntryBase.cs | 24 +++++++++ 2 files changed, 24 insertions(+), 49 deletions(-) diff --git a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs index 27bd8f3cbc1..c326399fce5 100644 --- a/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs +++ b/src/EFCore/ChangeTracking/Internal/InternalEntityEntry.cs @@ -525,55 +525,6 @@ public TProperty GetRelationshipSnapshotValue(IPropertyBase propertyB public object? GetRelationshipSnapshotValue(IPropertyBase propertyBase) => _relationshipsSnapshot.GetValue(this, propertyBase); - - /// - /// Refreshes the property value with the value from the database - /// - /// Property - /// New value from database - /// MergeOption - /// Sets the EntityState to Unchanged if MergeOption.OverwriteChanges else calls ChangeDetector to determine changes - public void ReloadValue(IPropertyBase propertyBase, object? value, MergeOption mergeOption, bool updateEntityState) - { - var property = (IProperty)propertyBase; - EnsureOriginalValues(); - bool isModified = IsModified(property); - _originalValues.SetValue(property, value, -1); - if (mergeOption == MergeOption.OverwriteChanges || !isModified) - SetProperty(propertyBase, value, isMaterialization: true, setModified: false); - if (updateEntityState) - { - if (mergeOption == MergeOption.OverwriteChanges) - SetEntityState(EntityState.Unchanged); - else - ((StateManager as StateManager)?.ChangeDetector as ChangeDetector)?.DetectChanges(this); - } - } - - /// - /// Refreshes the property value with the value from the database - /// - /// Property - /// New value from database - /// MergeOption - /// Sets the EntityState to Unchanged if MergeOption.OverwriteChanges else calls ChangeDetector to determine changes - public void ReloadValue(IPropertyBase propertyBase, object? value, MergeOption mergeOption, bool updateEntityState) - { - var property = (IProperty)propertyBase; - EnsureOriginalValues(); - bool isModified = IsModified(property); - _originalValues.SetValue(property, value, -1); - if (mergeOption == MergeOption.OverwriteChanges || !isModified) - SetProperty(propertyBase, value, isMaterialization: true, setModified: false); - if (updateEntityState) - { - if (mergeOption == MergeOption.OverwriteChanges) - SetEntityState(EntityState.Unchanged); - else - ((StateManager as StateManager)?.ChangeDetector as ChangeDetector)?.DetectChanges(this); - } - } - /// /// This is an internal API that supports the Entity Framework Core infrastructure and not subject to /// the same compatibility standards as public APIs. It may be changed or removed without notice in diff --git a/src/EFCore/ChangeTracking/Internal/InternalEntryBase.cs b/src/EFCore/ChangeTracking/Internal/InternalEntryBase.cs index d81686ad029..cb32e90c866 100644 --- a/src/EFCore/ChangeTracking/Internal/InternalEntryBase.cs +++ b/src/EFCore/ChangeTracking/Internal/InternalEntryBase.cs @@ -974,6 +974,30 @@ public void SetOriginalValue( } } + /// + /// Refreshes the property value with the value from the database + /// + /// Property + /// New value from database + /// MergeOption + /// Sets the EntityState to Unchanged if MergeOption.OverwriteChanges else calls ChangeDetector to determine changes + public void ReloadValue(IPropertyBase propertyBase, object? value, MergeOption mergeOption, bool updateEntityState) + { + var property = (IProperty)propertyBase; + EnsureOriginalValues(); + bool isModified = IsModified(property); + _originalValues.SetValue(property, value, -1); + if (mergeOption == MergeOption.OverwriteChanges || !isModified) + SetProperty(propertyBase, value, isMaterialization: true, setModified: false); + if (updateEntityState) + { + if (mergeOption == MergeOption.OverwriteChanges) + SetEntityState(EntityState.Unchanged); + else + ((StateManager as StateManager)?.ChangeDetector as ChangeDetector)?.DetectValueChange(this, property); + } + } + private void ReorderOriginalComplexCollectionEntries(IComplexProperty complexProperty, IList? newOriginalCollection) { Check.DebugAssert(HasOriginalValuesSnapshot, "This should only be called when original values are present");