Skip to content

Commit cf27c6d

Browse files
maumarmaumar
committed
Fix to #31100 - Switch to storing enums as ints in JSON instead of strings
Problem was that in 7.0 we stored enums inside JSON as strings by default (by applying EnumToStringConverter by convention), but in 8.0 we are changing this to int. This is a breaking change and it's extra problematic for databases that used EF JSON functionality in 7.0. This can easily create a scenario where there is a mix of string and int representation for an enum value within the same document. (some data was added in 7.0, and then some in 8.0 before customer realized that the breaking change has been made). To mitigate this we are adding a fallback mechanism when reading enum data that is part of JSON entity. We try to read as int and if that fails we try to read again as string. This way should minimize the disruption, moreover any data saved back to the database will be saved in the new format, so over time everything should normalize. We will still throw when projecting individual enum properties of a JSON entity (as opposed to the entire entity), because materialization for this goes through different path, indistinguishable from normal enum value read from column in relational table. Fixes #31100
1 parent c6b5eac commit cf27c6d

File tree

6 files changed

+411
-100
lines changed

6 files changed

+411
-100
lines changed

‎src/EFCore.Relational/Metadata/Conventions/RelationalMapToJsonConvention.cs‎

Lines changed: 0 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -50,13 +50,5 @@ public virtual void ProcessModelFinalizing(
5050
IConventionModelBuilder modelBuilder,
5151
IConventionContext<IConventionModelBuilder> context)
5252
{
53-
foreach (var jsonEntityType in modelBuilder.Metadata.GetEntityTypes().Where(e => e.IsMappedToJson()))
54-
{
55-
foreach (var enumProperty in jsonEntityType.GetDeclaredProperties().Where(p => p.ClrType.UnwrapNullableType().IsEnum))
56-
{
57-
// by default store enums as strings - values should be human-readable
58-
enumProperty.Builder.HasConversion(typeof(string));
59-
}
60-
}
6153
}
6254
}

‎src/EFCore.Relational/Query/RelationalShapedQueryCompilingExpressionVisitor.ShaperProcessingExpressionVisitor.cs‎

Lines changed: 47 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,12 @@ private static readonly MethodInfo Utf8JsonReaderTrySkipMethod
6464
private static readonly PropertyInfo Utf8JsonReaderTokenTypeProperty
6565
= typeof(Utf8JsonReader).GetProperty(nameof(Utf8JsonReader.TokenType))!;
6666

67+
private static readonly MethodInfo Utf8JsonReaderGetStringMethod
68+
= typeof(Utf8JsonReader).GetMethod(nameof(Utf8JsonReader.GetString), Array.Empty<Type>())!;
69+
70+
private static readonly MethodInfo EnumParseMethodInfo
71+
= typeof(Enum).GetMethod(nameof(Enum.Parse), new[] { typeof(Type), typeof(string) })!;
72+
6773
private readonly RelationalShapedQueryCompilingExpressionVisitor _parentVisitor;
6874
private readonly ISet<string>? _tags;
6975
private readonly bool _isTracking;
@@ -2339,7 +2345,6 @@ private Expression CreateReadJsonPropertyValueExpression(
23392345
{
23402346
// in case of null value we can't just use the JsonReader method, but rather check the current token type
23412347
// if it's JsonTokenType.Null means value is null, only if it's not we are safe to read the value
2342-
23432348
if (resultExpression.Type != property.ClrType)
23442349
{
23452350
resultExpression = Convert(resultExpression, property.ClrType);
@@ -2357,6 +2362,47 @@ private Expression CreateReadJsonPropertyValueExpression(
23572362
resultExpression);
23582363
}
23592364

2365+
// see issue #31100
2366+
// in 7.0 we stored enums as strings by default (by applying EnumToStringConverter by convention on enums inside JSON entities)
2367+
// in 8.0 we are reverting this decision and store enums as int (their natural representation in JSON as well as in database using EF)
2368+
// however it is a breaking change and it makes it very hard for people who used 7.0 version to work with legacy representation
2369+
// users can easily land in a situation where they have a mix of int and string representations
2370+
// update to 8.0, start adding more entities to the database, new entities are stored as ints, whereas old ones remain string
2371+
// to mitigate this we provide fallback mechanism here - if we can't read enum value using new default (EnumToNumberConverter)
2372+
// we assume it's the legacy scenario and attempt to parse from string
2373+
if (property.ClrType.UnwrapNullableType().IsEnum
2374+
&& property.GetRelationalTypeMapping().Converter is ValueConverter converter
2375+
&& converter.GetType() is Type { IsGenericType: true } converterType
2376+
&& converterType.GetGenericTypeDefinition() == typeof(EnumToNumberConverter<,>)
2377+
&& converterType.GetGenericArguments() is Type[] genericArgs
2378+
&& genericArgs[0] == converter.ModelClrType
2379+
&& genericArgs[1] == converter.ProviderClrType)
2380+
{
2381+
// try
2382+
// {
2383+
// JsonConvertedValueReaderWriter<MyEnumType, underlying_type>.FromJsonTyped(...));
2384+
// }
2385+
// catch(InvalidOperationException e)
2386+
// {
2387+
// (MyEnumType)Enum.Parse(typeof(MyEnumType), jsonReaderManager.CurrentReader.GetString());
2388+
// }
2389+
var exceptionParameter = Parameter(typeof(InvalidOperationException), name: "e");
2390+
var catchBlock = Catch(
2391+
exceptionParameter,
2392+
Convert(
2393+
Call(
2394+
EnumParseMethodInfo,
2395+
Constant(converter.ModelClrType),
2396+
Call(
2397+
Field(
2398+
jsonReaderManagerParameter,
2399+
Utf8JsonReaderManagerCurrentReaderField),
2400+
Utf8JsonReaderGetStringMethod)),
2401+
property.ClrType));
2402+
2403+
resultExpression = TryCatch(resultExpression, catchBlock);
2404+
}
2405+
23602406
if (_detailedErrorsEnabled)
23612407
{
23622408
var exceptionParameter = Parameter(typeof(Exception), name: "e");

‎test/EFCore.Relational.Specification.Tests/Query/JsonQueryAdHocTestBase.cs‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -707,6 +707,123 @@ public MyJsonEntityShadowPropertiesWithCtor(string name)
707707

708708
#endregion
709709

710+
//#region EnumLegacyValues
711+
712+
//[ConditionalTheory]
713+
//[MemberData(nameof(IsAsyncData))]
714+
//public virtual async Task Read_enum_property_with_legacy_values(bool async)
715+
//{
716+
// var contextFactory = await InitializeAsync<MyContextEnumLegacyValues>(
717+
// seed: SeedEnumLegacyValues);
718+
719+
// using (var context = contextFactory.CreateContext())
720+
// {
721+
// var query = context.Entities.Select(x => new
722+
// {
723+
// x.Reference.IntEnum,
724+
// x.Reference.ByteEnum,
725+
// x.Reference.LongEnum,
726+
// x.Reference.NullableEnum
727+
// });
728+
729+
// var result = async
730+
// ? await query.ToListAsync()
731+
// : query.ToList();
732+
733+
// Assert.Equal(1, result.Count);
734+
// //Assert.Equal("Foo", result[0].ShadowString);
735+
// //Assert.Equal(143, result[0].ShadowInt);
736+
// }
737+
//}
738+
739+
//[ConditionalTheory]
740+
//[MemberData(nameof(IsAsyncData))]
741+
//public virtual async Task Read_json_entity_with_enum_properties_with_legacy_values(bool async)
742+
//{
743+
// var contextFactory = await InitializeAsync<MyContextEnumLegacyValues>(
744+
// seed: SeedEnumLegacyValues);
745+
746+
// using (var context = contextFactory.CreateContext())
747+
// {
748+
// var query = context.Entities.Select(x => x.Reference).AsNoTracking();
749+
750+
// var result = async
751+
// ? await query.ToListAsync()
752+
// : query.ToList();
753+
754+
// Assert.Equal(1, result.Count);
755+
// //Assert.Equal("Foo", result[0].ShadowString);
756+
// //Assert.Equal(143, result[0].ShadowInt);
757+
// }
758+
//}
759+
760+
//protected abstract void SeedEnumLegacyValues(MyContextEnumLegacyValues ctx);
761+
762+
//protected class MyContextEnumLegacyValues : DbContext
763+
//{
764+
// public MyContextEnumLegacyValues(DbContextOptions options)
765+
// : base(options)
766+
// {
767+
// }
768+
769+
// public DbSet<MyEntityEnumLegacyValues> Entities { get; set; }
770+
771+
// protected override void OnModelCreating(ModelBuilder modelBuilder)
772+
// {
773+
// modelBuilder.Entity<MyEntityEnumLegacyValues>().Property(x => x.Id).ValueGeneratedNever();
774+
// modelBuilder.Entity<MyEntityEnumLegacyValues>().OwnsOne(x => x.Reference, b =>
775+
// {
776+
// b.ToJson();
777+
// });
778+
// modelBuilder.Entity<MyEntityEnumLegacyValues>().OwnsMany(x => x.Collection, b =>
779+
// {
780+
// b.ToJson();
781+
// });
782+
// }
783+
//}
784+
785+
//public class MyEntityEnumLegacyValues
786+
//{
787+
// public int Id { get; set; }
788+
// public string Name { get; set; }
789+
790+
// public MyJsonEntityEnumLegacyValues Reference { get; set; }
791+
// public List<MyJsonEntityEnumLegacyValues> Collection { get; set; }
792+
//}
793+
794+
//public class MyJsonEntityEnumLegacyValues
795+
//{
796+
// public string Name { get; set; }
797+
798+
// public IntEnumLegacyValues IntEnum { get; set; }
799+
// public ByteEnumLegacyValues ByteEnum { get; set; }
800+
// public LongEnumLegacyValues LongEnum { get; set; }
801+
// public IntEnumLegacyValues? NullableEnum { get; set; }
802+
//}
803+
804+
//public enum IntEnumLegacyValues
805+
//{
806+
// Foo,
807+
// Bar,
808+
// Baz,
809+
//}
810+
811+
//public enum ByteEnumLegacyValues : byte
812+
//{
813+
// Seattle,
814+
// Redmond,
815+
// Bellevue,
816+
//}
817+
818+
//public enum LongEnumLegacyValues : long
819+
//{
820+
// One,
821+
// Two,
822+
// Three,
823+
//}
824+
825+
//#endregion
826+
710827
protected TestSqlLoggerFactory TestSqlLoggerFactory
711828
=> (TestSqlLoggerFactory)ListLoggerFactory;
712829
}

‎test/EFCore.SqlServer.FunctionalTests/Query/JsonQueryAdHocSqlServerTest.cs‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,8 @@
11
// Licensed to the .NET Foundation under one or more agreements.
22
// The .NET Foundation licenses this file to you under the MIT license.
33

4+
using Microsoft.Data.SqlClient;
5+
46
namespace Microsoft.EntityFrameworkCore.Query;
57

68
public class JsonQueryAdHocSqlServerTest : JsonQueryAdHocTestBase
@@ -140,4 +142,158 @@ protected override void SeedShadowProperties(MyContextShadowProperties ctx)
140142
1,
141143
N'e1')");
142144
}
145+
146+
#region EnumLegacyValues
147+
148+
[ConditionalTheory]
149+
[MemberData(nameof(IsAsyncData))]
150+
public virtual async Task Read_enum_property_with_legacy_values(bool async)
151+
{
152+
var contextFactory = await InitializeAsync<MyContextEnumLegacyValues>(
153+
seed: SeedEnumLegacyValues);
154+
155+
using (var context = contextFactory.CreateContext())
156+
{
157+
var query = context.Entities.Select(x => new
158+
{
159+
x.Reference.IntEnum,
160+
x.Reference.ByteEnum,
161+
x.Reference.LongEnum,
162+
x.Reference.NullableEnum
163+
});
164+
165+
var exception = async
166+
? await (Assert.ThrowsAsync<SqlException>(() => query.ToListAsync()))
167+
: Assert.Throws<SqlException>(() => query.ToList());
168+
169+
// Conversion failed when converting the nvarchar value '...' to data type int
170+
Assert.Equal(245, exception.Number);
171+
}
172+
}
173+
174+
[ConditionalTheory]
175+
[MemberData(nameof(IsAsyncData))]
176+
public virtual async Task Read_json_entity_with_enum_properties_with_legacy_values(bool async)
177+
{
178+
var contextFactory = await InitializeAsync<MyContextEnumLegacyValues>(
179+
seed: SeedEnumLegacyValues);
180+
181+
using (var context = contextFactory.CreateContext())
182+
{
183+
var query = context.Entities.Select(x => x.Reference).AsNoTracking();
184+
185+
var result = async
186+
? await query.ToListAsync()
187+
: query.ToList();
188+
189+
Assert.Equal(1, result.Count);
190+
Assert.Equal(ByteEnumLegacyValues.Redmond, result[0].ByteEnum);
191+
Assert.Equal(IntEnumLegacyValues.Foo, result[0].IntEnum);
192+
Assert.Equal(LongEnumLegacyValues.Three, result[0].LongEnum);
193+
Assert.Equal(IntEnumLegacyValues.Bar, result[0].NullableEnum);
194+
}
195+
}
196+
197+
[ConditionalTheory]
198+
[MemberData(nameof(IsAsyncData))]
199+
public virtual async Task Read_json_entity_collection_with_enum_properties_with_legacy_values(bool async)
200+
{
201+
var contextFactory = await InitializeAsync<MyContextEnumLegacyValues>(
202+
seed: SeedEnumLegacyValues);
203+
204+
using (var context = contextFactory.CreateContext())
205+
{
206+
var query = context.Entities.Select(x => x.Collection).AsNoTracking();
207+
208+
var result = async
209+
? await query.ToListAsync()
210+
: query.ToList();
211+
212+
Assert.Equal(1, result.Count);
213+
Assert.Equal(2, result[0].Count);
214+
Assert.Equal(ByteEnumLegacyValues.Bellevue, result[0][0].ByteEnum);
215+
Assert.Equal(IntEnumLegacyValues.Foo, result[0][0].IntEnum);
216+
Assert.Equal(LongEnumLegacyValues.One, result[0][0].LongEnum);
217+
Assert.Equal(IntEnumLegacyValues.Bar, result[0][0].NullableEnum);
218+
Assert.Equal(ByteEnumLegacyValues.Seattle, result[0][1].ByteEnum);
219+
Assert.Equal(IntEnumLegacyValues.Baz, result[0][1].IntEnum);
220+
Assert.Equal(LongEnumLegacyValues.Two, result[0][1].LongEnum);
221+
Assert.Null(result[0][1].NullableEnum);
222+
}
223+
}
224+
225+
private void SeedEnumLegacyValues(MyContextEnumLegacyValues ctx)
226+
{
227+
ctx.Database.ExecuteSqlRaw(@"INSERT INTO [Entities] ([Collection], [Reference], [Id], [Name])
228+
VALUES(
229+
N'[{{""ByteEnum"":""Bellevue"",""IntEnum"":""Foo"",""LongEnum"":""One"",""Name"":""e1_c1"",""NullableEnum"":""Bar""}},{{""ByteEnum"":""Seattle"",""IntEnum"":""Baz"",""LongEnum"":""Two"",""Name"":""e1_c2"",""NullableEnum"":null}}]',
230+
N'{{""ByteEnum"":""Redmond"",""IntEnum"":""Foo"",""LongEnum"":""Three"",""Name"":""e1_r"",""NullableEnum"":""Bar""}}',
231+
1,
232+
N'e1')");
233+
}
234+
235+
private class MyContextEnumLegacyValues : DbContext
236+
{
237+
public MyContextEnumLegacyValues(DbContextOptions options)
238+
: base(options)
239+
{
240+
}
241+
242+
public DbSet<MyEntityEnumLegacyValues> Entities { get; set; }
243+
244+
protected override void OnModelCreating(ModelBuilder modelBuilder)
245+
{
246+
modelBuilder.Entity<MyEntityEnumLegacyValues>().Property(x => x.Id).ValueGeneratedNever();
247+
modelBuilder.Entity<MyEntityEnumLegacyValues>().OwnsOne(x => x.Reference, b =>
248+
{
249+
b.ToJson();
250+
});
251+
modelBuilder.Entity<MyEntityEnumLegacyValues>().OwnsMany(x => x.Collection, b =>
252+
{
253+
b.ToJson();
254+
});
255+
}
256+
}
257+
258+
private class MyEntityEnumLegacyValues
259+
{
260+
public int Id { get; set; }
261+
public string Name { get; set; }
262+
263+
public MyJsonEntityEnumLegacyValues Reference { get; set; }
264+
public List<MyJsonEntityEnumLegacyValues> Collection { get; set; }
265+
}
266+
267+
private class MyJsonEntityEnumLegacyValues
268+
{
269+
public string Name { get; set; }
270+
271+
public IntEnumLegacyValues IntEnum { get; set; }
272+
public ByteEnumLegacyValues ByteEnum { get; set; }
273+
public LongEnumLegacyValues LongEnum { get; set; }
274+
public IntEnumLegacyValues? NullableEnum { get; set; }
275+
}
276+
277+
private enum IntEnumLegacyValues
278+
{
279+
Foo,
280+
Bar,
281+
Baz,
282+
}
283+
284+
private enum ByteEnumLegacyValues : byte
285+
{
286+
Seattle,
287+
Redmond,
288+
Bellevue,
289+
}
290+
291+
private enum LongEnumLegacyValues : long
292+
{
293+
One,
294+
Two,
295+
Three,
296+
}
297+
298+
#endregion
143299
}

0 commit comments

Comments
 (0)