Allow configuration of the sentinel value that indicates a property h…

…as not been set

Fixes #701

Part of #13224, #15070, #13613

This PR contains the underlying model and change tracking changes needed to support sentinel values. Future PRs will add model building API surface and setting the sentinel automatically based on the database default.

There is a breaking change here: explicitly setting the value of a property with a temporary value no longer automatically makes the value non temporary.

src/EFCore/ChangeTracking/Internal/KeyPropagator.cs

@@ -149,9 +149,8 @@ private static void SetValue(InternalEntityEntry entry, IProperty property, Valu
 
                         if (principalProperty != property)
                         {
-                            var principalValue = principalEntry[principalProperty];
                             if (generationProperty == null
-                                || !principalProperty.ClrType.IsDefaultValue(principalValue))
+                                || principalEntry.HasExplicitValue(principalProperty))
                             {
                                 entry.PropagateValue(principalEntry, principalProperty, property);
 

src/EFCore/ChangeTracking/Internal/SidecarValues.cs

@@ -26,8 +26,11 @@ public bool TryGetValue(int index, out object? value)
             return true;
         }
 
+        public object? GetValue(int index)
+            => _values[index];
+
         public T GetValue<T>(int index)
-            => IsEmpty ? default! : _values.GetValue<T>(index);
+            => _values.GetValue<T>(index);
 
         public void SetValue(IProperty property, object? value, int index)
         {

src/EFCore/ChangeTracking/Internal/SimpleFullyNullableDependentKeyValueFactory.cs

@@ -61,7 +61,7 @@ public virtual bool TryCreateFromBuffer(in ValueBuffer valueBuffer, [NotNullWhen
     /// </summary>
     public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.CurrentValueGetter)(entry);
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.CurrentValueGetter)((InternalEntityEntry)entry);
         return key != null;
     }
 
@@ -73,7 +73,7 @@ public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen
     /// </summary>
     public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)(entry);
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)((InternalEntityEntry)entry);
         return key != null;
     }
 
@@ -85,7 +85,7 @@ public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry ent
     /// </summary>
     public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.OriginalValueGetter!)(entry);
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.OriginalValueGetter!)((InternalEntityEntry)entry);
         return key != null;
     }
 
@@ -97,7 +97,7 @@ public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhe
     /// </summary>
     public virtual bool TryCreateFromRelationshipSnapshot(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)(entry);
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)((InternalEntityEntry)entry);
         return key != null;
     }
 }

src/EFCore/ChangeTracking/Internal/SimpleNonNullableDependentKeyValueFactory.cs

@@ -68,7 +68,7 @@ public virtual bool TryCreateFromBuffer(in ValueBuffer valueBuffer, [NotNullWhen
     /// </summary>
     public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.CurrentValueGetter)(entry)!;
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.CurrentValueGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -80,7 +80,7 @@ public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen
     /// </summary>
     public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)(entry)!;
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -92,7 +92,7 @@ public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry ent
     /// </summary>
     public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.OriginalValueGetter!)(entry)!;
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.OriginalValueGetter!)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -104,7 +104,7 @@ public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhe
     /// </summary>
     public virtual bool TryCreateFromRelationshipSnapshot(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = ((Func<IUpdateEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)(entry)!;
+        key = ((Func<InternalEntityEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 }

src/EFCore/ChangeTracking/Internal/SimpleNullableDependentKeyValueFactory.cs

@@ -66,7 +66,7 @@ public virtual bool TryCreateFromBuffer(in ValueBuffer valueBuffer, out TKey key
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public override bool TryCreateFromCurrentValues(IUpdateEntry entry, out TKey key)
-        => HandleNullableValue(((Func<IUpdateEntry, TKey?>)_propertyAccessors.CurrentValueGetter)(entry), out key);
+        => HandleNullableValue(((Func<InternalEntityEntry, TKey?>)_propertyAccessors.CurrentValueGetter)((InternalEntityEntry)entry), out key);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -76,7 +76,7 @@ public override bool TryCreateFromCurrentValues(IUpdateEntry entry, out TKey key
     /// </summary>
     public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry entry, out TKey key)
         => HandleNullableValue(
-            ((Func<IUpdateEntry, TKey?>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)(entry), out key);
+            ((Func<InternalEntityEntry, TKey?>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)((InternalEntityEntry)entry), out key);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -85,7 +85,7 @@ public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry ent
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public override bool TryCreateFromOriginalValues(IUpdateEntry entry, out TKey key)
-        => HandleNullableValue(((Func<IUpdateEntry, TKey?>)_propertyAccessors.OriginalValueGetter!)(entry), out key);
+        => HandleNullableValue(((Func<InternalEntityEntry, TKey?>)_propertyAccessors.OriginalValueGetter!)((InternalEntityEntry)entry), out key);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -94,7 +94,7 @@ public override bool TryCreateFromOriginalValues(IUpdateEntry entry, out TKey ke
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public virtual bool TryCreateFromRelationshipSnapshot(IUpdateEntry entry, out TKey key)
-        => HandleNullableValue(((Func<IUpdateEntry, TKey?>)_propertyAccessors.RelationshipSnapshotGetter)(entry), out key);
+        => HandleNullableValue(((Func<InternalEntityEntry, TKey?>)_propertyAccessors.RelationshipSnapshotGetter)((InternalEntityEntry)entry), out key);
 
     private static bool HandleNullableValue(TKey? value, out TKey key)
     {

src/EFCore/ChangeTracking/Internal/SimpleNullablePrincipalDependentKeyValueFactory.cs

@@ -72,7 +72,7 @@ public virtual bool TryCreateFromBuffer(in ValueBuffer valueBuffer, [NotNullWhen
     /// </summary>
     public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = (TKey)(object)((Func<IUpdateEntry, TNonNullableKey>)_propertyAccessors.CurrentValueGetter)(entry)!;
+        key = (TKey)(object)((Func<InternalEntityEntry, TNonNullableKey>)_propertyAccessors.CurrentValueGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -84,7 +84,7 @@ public override bool TryCreateFromCurrentValues(IUpdateEntry entry, [NotNullWhen
     /// </summary>
     public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = (TKey)(object)((Func<IUpdateEntry, TNonNullableKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)(entry)!;
+        key = (TKey)(object)((Func<InternalEntityEntry, TNonNullableKey>)_propertyAccessors.PreStoreGeneratedCurrentValueGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -96,7 +96,7 @@ public virtual bool TryCreateFromPreStoreGeneratedCurrentValues(IUpdateEntry ent
     /// </summary>
     public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = (TKey)(object)((Func<IUpdateEntry, TNonNullableKey>)_propertyAccessors.OriginalValueGetter!)(entry)!;
+        key = (TKey)(object)((Func<InternalEntityEntry, TNonNullableKey>)_propertyAccessors.OriginalValueGetter!)((InternalEntityEntry)entry)!;
         return true;
     }
 
@@ -108,7 +108,7 @@ public override bool TryCreateFromOriginalValues(IUpdateEntry entry, [NotNullWhe
     /// </summary>
     public virtual bool TryCreateFromRelationshipSnapshot(IUpdateEntry entry, [NotNullWhen(true)] out TKey? key)
     {
-        key = (TKey)(object)((Func<IUpdateEntry, TNonNullableKey>)_propertyAccessors.RelationshipSnapshotGetter)(entry)!;
+        key = (TKey)(object)((Func<InternalEntityEntry, TNonNullableKey>)_propertyAccessors.RelationshipSnapshotGetter)((InternalEntityEntry)entry)!;
         return true;
     }
 }

src/EFCore/ChangeTracking/Internal/SimplePrincipalKeyValueFactory.cs

@@ -68,7 +68,7 @@ public virtual IProperty FindNullPropertyInKeyValues(IReadOnlyList<object?> keyV
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public virtual TKey CreateFromCurrentValues(IUpdateEntry entry)
-        => ((Func<IUpdateEntry, TKey>)_propertyAccessors.CurrentValueGetter)(entry);
+        => ((Func<InternalEntityEntry, TKey>)_propertyAccessors.CurrentValueGetter)((InternalEntityEntry)entry);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -86,7 +86,7 @@ public virtual IProperty FindNullPropertyInCurrentValues(IUpdateEntry entry)
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public virtual TKey CreateFromOriginalValues(IUpdateEntry entry)
-        => ((Func<IUpdateEntry, TKey>)_propertyAccessors.OriginalValueGetter!)(entry);
+        => ((Func<InternalEntityEntry, TKey>)_propertyAccessors.OriginalValueGetter!)((InternalEntityEntry)entry);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -95,7 +95,7 @@ public virtual TKey CreateFromOriginalValues(IUpdateEntry entry)
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
     public virtual TKey CreateFromRelationshipSnapshot(IUpdateEntry entry)
-        => ((Func<IUpdateEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)(entry);
+        => ((Func<InternalEntityEntry, TKey>)_propertyAccessors.RelationshipSnapshotGetter)((InternalEntityEntry)entry);
 
     /// <summary>
     ///     This is an internal API that supports the Entity Framework Core infrastructure and not subject to
@@ -119,18 +119,6 @@ public virtual object CreateEquatableKey(IUpdateEntry entry, bool fromOriginalVa
                 : CreateFromCurrentValues(entry),
             EqualityComparer);
 
-    private sealed class NoNullsStructuralEqualityComparer : IEqualityComparer<TKey>
-    {
-        private readonly IEqualityComparer _comparer
-            = StructuralComparisons.StructuralEqualityComparer;
-
-        public bool Equals(TKey? x, TKey? y)
-            => _comparer.Equals(x, y);
-
-        public int GetHashCode([DisallowNull] TKey obj)
-            => _comparer.GetHashCode(obj);
-    }
-
     private sealed class NoNullsCustomEqualityComparer : IEqualityComparer<TKey>
     {
         private readonly Func<TKey?, TKey?, bool> _equals;

src/EFCore/ChangeTracking/Internal/SnapshotFactoryFactory.cs

@@ -130,7 +130,7 @@ protected virtual Expression CreateSnapshotExpression(
 
             if (memberAccess.Type != propertyBase.ClrType)
             {
-                var hasDefaultValueExpression = memberAccess.MakeHasDefaultValue(propertyBase);
+                var hasDefaultValueExpression = memberAccess.MakeHasSentinelValue(propertyBase);
 
                 memberAccess = Expression.Condition(
                     hasDefaultValueExpression,

src/EFCore/ChangeTracking/Internal/StateData.cs

@@ -11,7 +11,8 @@ internal enum PropertyFlag
         Null = 1,
         Unknown = 2,
         IsLoaded = 3,
-        IsTemporary = 4
+        IsTemporary = 4,
+        IsStoreGenerated = 5
     }
 
     internal readonly struct StateData

src/EFCore/ChangeTracking/Internal/StateManager.cs

@@ -281,13 +281,13 @@ public virtual InternalEntityEntry CreateEntry(IDictionary<string, object?> valu
         {
             valuesArray[i++] = values.TryGetValue(property.Name, out var value)
                 ? value
-                : property.ClrType.GetDefaultValue();
+                : property.Sentinel;
 
             if (property.IsShadowProperty())
             {
                 shadowPropertyValuesArray[property.GetShadowIndex()] = values.TryGetValue(property.Name, out var shadowValue)
                     ? shadowValue
-                    : property.ClrType.GetDefaultValue();
+                    : property.Sentinel;
             }
         }
 

src/EFCore/ChangeTracking/Internal/ValueGenerationManager.cs

@@ -45,7 +45,8 @@ public ValueGenerationManager(
         InternalEntityEntry? chosenPrincipal = null;
         foreach (var property in entry.EntityType.GetForeignKeyProperties())
         {
-            if (!entry.HasDefaultValue(property))
+            if (!entry.IsUnknown(property)
+                && entry.HasExplicitValue(property))
             {
                 continue;
             }
@@ -68,7 +69,7 @@ public ValueGenerationManager(
         InternalEntityEntry? chosenPrincipal = null;
         foreach (var property in entry.EntityType.GetForeignKeyProperties())
         {
-            if (!entry.HasDefaultValue(property))
+            if (entry.HasExplicitValue(property))
             {
                 continue;
             }
@@ -94,7 +95,7 @@ public virtual bool Generate(InternalEntityEntry entry, bool includePrimaryKey =
 
         foreach (var property in entry.EntityType.GetValueGeneratingProperties())
         {
-            if (!entry.HasDefaultValue(property)
+            if (entry.HasExplicitValue(property)
                 || (!includePrimaryKey
                     && property.IsPrimaryKey()))
             {
@@ -153,7 +154,7 @@ public virtual async Task<bool> GenerateAsync(
         var hasNonStableValues = false;
         foreach (var property in entry.EntityType.GetValueGeneratingProperties())
         {
-            if (!entry.HasDefaultValue(property)
+            if (entry.HasExplicitValue(property)
                 || (!includePrimaryKey
                     && property.IsPrimaryKey()))
             {

src/EFCore/Extensions/Internal/ExpressionExtensions.cs

@@ -20,37 +20,55 @@ public static class ExpressionExtensions
     ///     any release. You should only use it directly in your code with extreme caution and knowing that
     ///     doing so can result in application failures when updating to a new Entity Framework Core release.
     /// </summary>
-    public static Expression MakeHasDefaultValue(
+    public static Expression MakeHasSentinelValue(
         this Expression currentValueExpression,
         IReadOnlyPropertyBase? propertyBase)
     {
-        if (!currentValueExpression.Type.IsValueType)
-        {
-            return Expression.ReferenceEqual(
-                currentValueExpression,
-                Expression.Constant(null, currentValueExpression.Type));
-        }
+        var sentinel = propertyBase?.Sentinel;
 
-        if (currentValueExpression.Type.IsGenericType
-            && currentValueExpression.Type.GetGenericTypeDefinition() == typeof(Nullable<>))
+        var isReferenceType = !currentValueExpression.Type.IsValueType;
+        var isNullableValueType = currentValueExpression.Type.IsGenericType
+            && currentValueExpression.Type.GetGenericTypeDefinition() == typeof(Nullable<>);
+
+        if (sentinel == null)
         {
-            return Expression.Not(
-                Expression.Call(
+            return isReferenceType
+                ? Expression.ReferenceEqual(
                     currentValueExpression,
-                    Check.NotNull(
-                        currentValueExpression.Type.GetMethod("get_HasValue"), $"get_HasValue on {currentValueExpression.Type.Name}")));
+                    Expression.Constant(null, currentValueExpression.Type))
+                : isNullableValueType
+                    ? Expression.Not(
+                        Expression.Call(
+                            currentValueExpression,
+                            currentValueExpression.Type.GetMethod("get_HasValue")!))
+                    : Expression.Constant(false);
         }
 
-        var property = propertyBase as IReadOnlyProperty;
-        var comparer = property?.GetValueComparer()
+        var comparer = (propertyBase as IProperty)?.GetValueComparer()
             ?? ValueComparer.CreateDefault(
                 propertyBase?.ClrType ?? currentValueExpression.Type, favorStructuralComparisons: false);
 
-        return comparer.ExtractEqualsBody(
+        var equalsExpression = comparer.ExtractEqualsBody(
             comparer.Type != currentValueExpression.Type
                 ? Expression.Convert(currentValueExpression, comparer.Type)
                 : currentValueExpression,
-            Expression.Default(comparer.Type));
+            Expression.Constant(sentinel, comparer.Type));
+
+        if (isReferenceType || isNullableValueType)
+        {
+            return Expression.AndAlso(
+                isReferenceType
+                    ? Expression.Not(
+                        Expression.ReferenceEqual(
+                            currentValueExpression,
+                            Expression.Constant(null, currentValueExpression.Type)))
+                    : Expression.Call(
+                        currentValueExpression,
+                        currentValueExpression.Type.GetMethod("get_HasValue")!),
+                equalsExpression);
+        }
+
+        return equalsExpression;
     }
 
     /// <summary>

src/EFCore/Metadata/Conventions/RuntimeModelConvention.cs

@@ -337,6 +337,7 @@ private static RuntimeProperty Create(IProperty property, RuntimeEntityType runt
         => runtimeEntityType.AddProperty(
             property.Name,
             property.ClrType,
+            property.Sentinel,
             property.PropertyInfo,
             property.FieldInfo,
             property.GetPropertyAccessMode(),

src/EFCore/Metadata/IClrPropertyGetter.cs

@@ -31,5 +31,5 @@ public interface IClrPropertyGetter
     /// </summary>
     /// <param name="entity">The entity instance.</param>
     /// <returns><see langword="true" /> if the property value is the CLR default; <see langword="false" /> it is any other value.</returns>
-    bool HasDefaultValue(object entity);
+    bool HasSentinelValue(object entity);
 }