Improve persistence by annotating supported types with OData Edm

Previously, we didn't ever annotate properties with OData Edm types, meaning the following supported native column types in Azure Tables would all be converted (and used) as plain strings: byte[], DateTimeOffset, Guid and long.

By following more closely the documentation at https://learn.microsoft.com/en-us/rest/api/storageservices/payload-format-for-table-service-operations#property-types-in-a-json-feed, we effectively make our API more optimal and compatible with the Azure SDK for Table Storage.

In order to avoid too many breaking changes, we only support Edm.DateTime annotation for DateTimeOffset properties, since that's the behavior of the SDK when deserializing such annotated values (not to DateTime). This effectively means that DateTime is more of a legacy type that might best be avoided.

This also now allows using a GUID-typed property for both partition and row keys.

Closes #235.

readme.md

@@ -139,6 +139,8 @@ your "real" data, but for reference data, it could come in handy.
 Stored entities using `TableRepository` and `TablePartition` use individual columns for 
 properties, which makes it easy to browse the data (and query, as shown above!). 
 
+> NOTE: partition and row keys can also be typed as `Guid`
+
 But document-based storage is also available via `DocumentRepository` and `DocumentPartition` if 
 you don't need the individual columns.
 

src/TableStorage/DocumentRepository`1.cs

@@ -180,6 +180,7 @@ async Task<T> PutBinaryAsync(T entity, CancellationToken cancellation = default)
 
             var row = ToTable(entity);
             row["Document"] = binarySerializer!.Serialize(entity);
+            row["Document@odata.type"] = "Edm.Binary";
 
             // We use Replace because all the existing entity data is in a single 
             // column, no point in merging since it can't be done at that level anyway.
@@ -266,7 +267,7 @@ TableEntity ToTable(T entity)
 
             foreach (var prop in EntityPropertiesMapper.Default.ToProperties(entity))
             {
-                // Never allow properties to overwrite out built-in queryable columns
+                // Never allow properties to overwrite our built-in queryable columns
                 if (!te.ContainsKey(prop.Key))
                     te[prop.Key] = prop.Value;
             }

src/TableStorage/EntityPropertiesMapper.cs

@@ -6,12 +6,15 @@
 using System.ComponentModel;
 using System.Linq;
 using System.Reflection;
+using Azure.Data.Tables;
 
 namespace Devlooped
 {
     class EntityPropertiesMapper
     {
         static readonly ConcurrentDictionary<Type, PropertyInfo[]> propertiesMap = new();
+        static readonly ConcurrentDictionary<Type, IDictionary<string, string>> edmAnnotations = new();
+        static readonly KeyValuePair<string, string> edmNone = new("", "");
 
         private EntityPropertiesMapper() { }
 
@@ -27,6 +30,37 @@ public IDictionary<string, object> ToProperties<T>(T entity, string? partitionKe
                     prop.Name != rowKeyProperty)
                 .ToArray());
 
+            var annotations = edmAnnotations.GetOrAdd(entity.GetType(), _ =>
+            {
+                var dictionary = properties
+                    .Where(prop =>
+                        prop.GetCustomAttribute<BrowsableAttribute>()?.Browsable != false &&
+                        prop.Name != partitionKeyProperty &&
+                        prop.Name != rowKeyProperty)
+                    .Select(prop => prop.PropertyType switch
+                    {
+                        var type when type == typeof(byte[]) => new KeyValuePair<string, string>(prop.Name + "@odata.type", "Edm.Binary"),
+                        //var type when type == typeof(DateTime) => new KeyValuePair<string, string>(prop.Name + "@odata.type", "Edm.DateTime"),
+                        var type when type == typeof(DateTimeOffset) => new KeyValuePair<string, string>(prop.Name + "@odata.type", "Edm.DateTime"),
+                        var type when type == typeof(Guid) => new KeyValuePair<string, string>(prop.Name + "@odata.type", "Edm.Guid"),
+                        var type when type == typeof(long) => new KeyValuePair<string, string>(prop.Name + "@odata.type", "Edm.Int64"),
+                        _ => edmNone,
+                    })
+                    // This is an unnecessary annotation, so skip it.
+                    .Where(x => x.Key != edmNone.Key)
+                    .ToDictionary(x => x.Key, x => x.Value);
+
+                // Make sure the Timestamp property is always annotated as DateTime, since it can also
+                // be read by the client using a string or a DateTime, according to our documentation.
+                // It will typically never be written back, since it's a server-managed value, but this 
+                // makes it more obvious and consistent with the Azure SDK behavior which always includes 
+                // this annotation when using odata:minimalmetadata or fullmetadata.
+                if (properties.Any(x => x.Name == nameof(ITableEntity.Timestamp)) && !dictionary.ContainsKey(nameof(ITableEntity.Timestamp)))
+                    dictionary.Add(nameof(ITableEntity.Timestamp), "Edm.DateTime");
+
+                return dictionary;
+            });
+
             var values = properties
                 .Select(prop => new { prop.Name, Value = prop.GetValue(entity) })
                 .Where(pair => pair.Value != null)
@@ -35,13 +69,22 @@ public IDictionary<string, object> ToProperties<T>(T entity, string? partitionKe
                     pair =>
 #if NET6_0_OR_GREATER
                     pair.Value is DateOnly date ? 
-                        date.ToString("O") : 
-                        pair.Value.GetType().IsEnum ? 
-                        pair.Value.ToString() : pair.Value!
-#else
-                    pair.Value.GetType().IsEnum ? pair.Value!.ToString() : pair.Value!
+                    date.ToString("O") : 
 #endif
-                    );
+                    // EDM annotation for date time requires persisting as UTC only, which is best practice too.
+                    //pair.Value is DateTime dateTime ?
+                    //dateTime.ToUniversalTime().ToString("O") :
+                    pair.Value is DateTimeOffset dateOffset ? 
+                    dateOffset.ToUniversalTime().ToString("O") :
+                    pair.Value.GetType().IsEnum ? pair.Value!.ToString() : 
+                    pair.Value!
+                );
+
+            // Add any required metadata annotations to improve persistence.
+            foreach (var annotation in annotations)
+            {
+                values.Add(annotation.Key, annotation.Value);
+            }
 
             return (IDictionary<string, object>)values;
         }

src/TableStorage/TableEntityPartition.cs

@@ -10,7 +10,10 @@
 
 namespace Devlooped
 {
-    /// <inheritdoc />
+    /// <summary>
+    /// A specific partition within a <see cref="TableEntityRepository"/>, which 
+    /// allows querying by the entity properties, since they are stored in individual columns.
+    /// </summary>
     partial class TableEntityPartition : ITablePartition<TableEntity>
     {
         readonly TableEntityRepository repository;

src/TableStorage/TableEntityRepository.cs

@@ -12,7 +12,10 @@
 
 namespace Devlooped
 {
-    /// <inheritdoc />
+    /// <summary>
+    /// Default implementation of <see cref="ITableRepository{TableEntity}"/> which allows querying 
+    /// the repository by the entity properties, since they are stored in individual columns.
+    /// </summary>
     partial class TableEntityRepository : ITableRepository<TableEntity>
     {
         TableConnection tableConnection;

src/TableStorage/TableRepositoryQuery`1.cs

@@ -254,7 +254,8 @@ static Dictionary<string, object> JsonElementToDictionary(JsonElement element)
                                     dictionary.Add(property.Name, Convert.FromBase64String(value));
                                     continue;
                                 case "Edm.DateTime":
-                                    dictionary.Add(property.Name, DateTime.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind));
+                                    // Reading from Azure SDK will always return DateTimeOffset for dates, instead of date time.
+                                    dictionary.Add(property.Name, DateTimeOffset.Parse(value, CultureInfo.InvariantCulture, DateTimeStyles.RoundtripKind));
                                     continue;
                                 case "Edm.Guid":
                                     dictionary.Add(property.Name, Guid.Parse(value));

src/TableStorage/TableStorageAttribute.cs

@@ -77,8 +77,8 @@ static Expression<Func<TEntity, string>> CreateGetterCore<TEntity, TAttribute>()
                 throw new ArgumentException($"Expected entity type '{typeof(TEntity).Name}' to have one property annotated with [{attributeName}]");
             }
 
-            if (keyProp.PropertyType != typeof(string))
-                throw new ArgumentException($"Property '{typeof(TEntity).Name}.{keyProp.Name}' annotated with [{attributeName}] must be of type string.");
+            if (keyProp.PropertyType != typeof(string) && keyProp.PropertyType != typeof(Guid))
+                throw new ArgumentException($"Property '{typeof(TEntity).Name}.{keyProp.Name}' annotated with [{attributeName}] must be of type string or Guid.");
 
             var param = Expression.Parameter(typeof(TEntity), "entity");
 
@@ -94,7 +94,9 @@ static Expression<Func<TEntity, string>> CreateGetterCore<TEntity, TAttribute>()
                         typeof(TableStorageAttribute).GetMethod(nameof(EnsureValid), BindingFlags.NonPublic | BindingFlags.Static)!,
                         Expression.Constant(attributeName),
                         Expression.Constant(keyProp.Name, typeof(string)),
-                        Expression.Property(param, keyProp))),
+                        keyProp.PropertyType == typeof(string)
+                            ? Expression.Property(param, keyProp)
+                            : Expression.Call(Expression.Property(param, keyProp), typeof(Guid).GetMethod(nameof(Guid.ToString), Type.EmptyTypes)!))),
                 param);
         }
 

src/Tests/DocumentRepositoryTests.cs

@@ -62,6 +62,15 @@ public async Task DocumentEndToEnd(IDocumentSerializer serializer)
 
                 Assert.Single(entities);
 
+                // Verify that the entity is not serialized as a string for non-string serializer
+                if (serializer is not IStringDocumentSerializer)
+                {
+                    var generic = TableRepository.Create(CloudStorageAccount.DevelopmentStorageAccount, table.Name);
+                    var row = await generic.GetAsync(partitionKey, rowKey);
+                    Assert.NotNull(row);
+                    Assert.IsType<byte[]>(row["Document"]);
+                }
+
                 await repo.DeleteAsync(saved);
 
                 Assert.Null(await repo.GetAsync(partitionKey, rowKey));

src/Tests/RepositoryTests.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Text;
 using System.Threading.Tasks;
 using Azure;
 using Azure.Data.Tables;
@@ -485,6 +486,31 @@ public async Task CanGetFromEntity()
             Assert.Equal(entity.Kind, generic.PartitionKey);
         }
 
+        [Fact]
+        public async Task CanGetFromEdmEntityTypes()
+        {
+            var repo = TableRepository.Create<EdmAnnotatedEntity>(CloudStorageAccount.DevelopmentStorageAccount);
+            var entity = new EdmAnnotatedEntity(Guid.NewGuid(), Guid.NewGuid(), DateTimeOffset.Now, Encoding.UTF8.GetBytes("Hello World"), Random.Shared.NextInt64(int.MaxValue, long.MaxValue));
+
+            var saved = await repo.PutAsync(entity);
+
+            Assert.NotNull(saved);
+
+            Assert.Equal(entity.ID, saved.ID);
+            Assert.Equal(entity.Date, saved.Date);
+            Assert.Equal("Hello World", Encoding.UTF8.GetString(saved.Data));
+            Assert.Equal(entity.Count, saved.Count);
+
+            var generic = await new TableEntityRepository(CloudStorageAccount.DevelopmentStorageAccount, TableRepository.GetDefaultTableName<EdmAnnotatedEntity>())
+                .GetAsync(new TableEntity(entity.Partition.ToString(), entity.ID.ToString()));
+
+            Assert.NotNull(generic);
+            Assert.Equal(entity.ID.ToString(), generic.RowKey);
+            Assert.IsType<byte[]>(generic["Data"]);
+            Assert.IsType<DateTimeOffset>(generic["Date"]);
+            Assert.IsType<long>(generic["Count"]);
+        }
+
         class MyEntity
         {
             public MyEntity(string id) => Id = id;
@@ -540,6 +566,9 @@ record TimestampedDateTimeEntity(string ID)
             public DateTime? Timestamp { get; set; }
         }
 
+        [Table(nameof(EdmAnnotatedEntity))]
+        record EdmAnnotatedEntity([PartitionKey] Guid Partition, [RowKey] Guid ID, DateTimeOffset Date, byte[] Data, long Count);
+
         [PartitionKey("MyPartition")]
         record CustomPartition(string Id);
     }