diff --git a/Directory.Packages.props b/Directory.Packages.props index 5db9340..50460ed 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -9,7 +9,7 @@ - + diff --git a/src/SIL.Harmony.Core/EntityNotFoundException.cs b/src/SIL.Harmony.Core/EntityNotFoundException.cs new file mode 100644 index 0000000..6bcbb0a --- /dev/null +++ b/src/SIL.Harmony.Core/EntityNotFoundException.cs @@ -0,0 +1,3 @@ +namespace SIL.Harmony.Core; + +public class EntityNotFoundException(string message) : Exception(message); diff --git a/src/SIL.Harmony.Core/IRemoteResourceService.cs b/src/SIL.Harmony.Core/IRemoteResourceService.cs new file mode 100644 index 0000000..22ebb06 --- /dev/null +++ b/src/SIL.Harmony.Core/IRemoteResourceService.cs @@ -0,0 +1,27 @@ +namespace SIL.Harmony.Core; + +/// +/// interface to facilitate downloading of resources, typically implemented in application code +/// the remote Id is opaque to the CRDT lib and could be a URL or some other identifier provided by the backend +/// the local path returned for the application code to use as required, it could be a URL if needed also. +/// +public interface IRemoteResourceService +{ + /// + /// instructs application code to download a resource from the remote server + /// the service is responsible for downloading the resource and returning the local path + /// + /// ID used to identify the remote resource, could be a URL + /// path defined by the CRDT config where the resource should be stored + /// download result containing the path to the downloaded file, this is stored in the local db and not synced + Task DownloadResource(string remoteId, string localResourceCachePath); + /// + /// upload a resource to the remote server + /// + /// full path to the resource on the local machine + /// an upload result with the remote id, the id will be stored and transmitted to other clients so they can also download the resource + Task UploadResource(string localPath); +} + +public record DownloadResult(string LocalPath); +public record UploadResult(string RemoteId); diff --git a/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs b/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs new file mode 100644 index 0000000..2e208d1 --- /dev/null +++ b/src/SIL.Harmony.Sample/Changes/AddWordImageChange.cs @@ -0,0 +1,15 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Sample.Changes; + +public class AddWordImageChange(Guid entityId, Guid imageId) : EditChange(entityId), ISelfNamedType +{ + public Guid ImageId { get; } = imageId; + + public override async ValueTask ApplyChange(Word entity, ChangeContext context) + { + if (!await context.IsObjectDeleted(ImageId)) entity.ImageResourceId = ImageId; + } +} \ No newline at end of file diff --git a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs index 5bf8061..410eb93 100644 --- a/src/SIL.Harmony.Sample/CrdtSampleKernel.cs +++ b/src/SIL.Harmony.Sample/CrdtSampleKernel.cs @@ -36,6 +36,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi services.AddCrdtData(config => { config.EnableProjectedTables = true; + config.AddRemoteResourceEntity(); config.ChangeTypeListBuilder .Add() .Add() @@ -44,6 +45,7 @@ public static IServiceCollection AddCrdtDataSample(this IServiceCollection servi .Add() .Add() .Add() + .Add() .Add>() .Add() .Add>() diff --git a/src/SIL.Harmony.Sample/Models/Word.cs b/src/SIL.Harmony.Sample/Models/Word.cs index 143a425..21c2b05 100644 --- a/src/SIL.Harmony.Sample/Models/Word.cs +++ b/src/SIL.Harmony.Sample/Models/Word.cs @@ -10,10 +10,17 @@ public class Word : IObjectBase public Guid Id { get; init; } public DateTimeOffset? DeletedAt { get; set; } public Guid? AntonymId { get; set; } + public Guid? ImageResourceId { get; set; } public Guid[] GetReferences() { - return AntonymId is null ? [] : [AntonymId.Value]; + return Refs().ToArray(); + + IEnumerable Refs() + { + if (AntonymId.HasValue) yield return AntonymId.Value; + if (ImageResourceId.HasValue) yield return ImageResourceId.Value; + } } public void RemoveReference(Guid id, Commit commit) diff --git a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt index 219be98..3bf3a9e 100644 --- a/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt +++ b/src/SIL.Harmony.Tests/DbContextTests.VerifyModel.verified.txt @@ -81,6 +81,40 @@ Relational:TableName: Snapshots Relational:ViewName: Relational:ViewSchema: + EntityType: LocalResource + Properties: + Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd + LocalPath (string) Required + Keys: + Id PK + Annotations: + DiscriminatorProperty: + Relational:FunctionName: + Relational:Schema: + Relational:SqlQuery: + Relational:TableName: LocalResource + Relational:ViewName: + Relational:ViewSchema: + EntityType: RemoteResource + Properties: + Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd + DeletedAt (DateTimeOffset?) + RemoteId (string) + SnapshotId (no field, Guid?) Shadow FK Index + Keys: + Id PK + Foreign keys: + RemoteResource {'SnapshotId'} -> ObjectSnapshot {'Id'} Unique SetNull + Indexes: + SnapshotId Unique + Annotations: + DiscriminatorProperty: + Relational:FunctionName: + Relational:Schema: + Relational:SqlQuery: + Relational:TableName: RemoteResource + Relational:ViewName: + Relational:ViewSchema: EntityType: Definition Properties: Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd @@ -133,6 +167,7 @@ Id (Guid) Required PK AfterSave:Throw ValueGenerated.OnAdd AntonymId (Guid?) DeletedAt (DateTimeOffset?) + ImageResourceId (Guid?) Note (string) SnapshotId (no field, Guid?) Shadow FK Index Text (string) Required diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs new file mode 100644 index 0000000..f9d6576 --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteResourcesTests.cs @@ -0,0 +1,210 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Resource; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class RemoteResourcesTests : DataModelTestBase +{ + private RemoteServiceMock _remoteServiceMock = new(); + private ResourceService _resourceService => _services.GetRequiredService(); + + public RemoteResourcesTests() + { + } + + private string CreateFile(string contents, [CallerMemberName] string fileName = "") + { + var filePath = Path.GetFullPath(fileName + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + private async Task<(Guid resourceId, string remoteId)> SetupRemoteResource(string fileContents) + { + var remoteId = _remoteServiceMock.CreateRemoteResource(fileContents); + var resourceId = Guid.NewGuid(); + await DataModel.AddChange(_localClientId, new CreateRemoteResourceChange(resourceId, remoteId)); + return (resourceId, remoteId); + } + + private async Task<(Guid resourceId, string localPath)> SetupLocalFile(string contents, [CallerMemberName] string fileName = "") + { + var file = CreateFile(contents, fileName); + //because resource service is null the file is not uploaded + var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); + return (crdtResource.Id, file); + } + + [Fact] + public async Task CreatingAResourceResultsInPendingLocalResources() + { + var (_, file) = await SetupLocalFile("contents"); + + //act + var pending = await _resourceService.ListResourcesPendingUpload(); + + + pending.Should().ContainSingle().Which.LocalPath.Should().Be(file); + } + + [Fact] + public async Task ResourcesNotLocalShouldShowUpAsNotDownloaded() + { + var (resourceId, remoteId) = await SetupRemoteResource("test"); + + //act + var pending = await _resourceService.ListResourcesPendingDownload(); + + + var remoteResource = pending.Should().ContainSingle().Subject; + remoteResource.RemoteId.Should().Be(remoteId); + remoteResource.Id.Should().Be(resourceId); + } + + [Fact] + public async Task CanUploadFileToRemote() + { + var fileContents = "resource"; + var localFile = CreateFile(fileContents); + + //act + var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); + + + var resource = await DataModel.GetLatest(crdtResource.Id); + ArgumentNullException.ThrowIfNull(resource); + ArgumentNullException.ThrowIfNull(resource.RemoteId); + _remoteServiceMock.ReadFile(resource.RemoteId).Should().Be(fileContents); + var pendingUpload = await _resourceService.ListResourcesPendingUpload(); + pendingUpload.Should().BeEmpty(); + } + + [Fact] + public async Task IfUploadingFailsTheResourceIsStillAddedAsPendingUpload() + { + var fileContents = "resource"; + var localFile = CreateFile(fileContents); + + //todo setup a mock that throws an exception when uploading + _remoteServiceMock.ThrowOnUpload(localFile); + + //act + var crdtResource = await _resourceService.AddLocalResource(localFile, _localClientId, resourceService: _remoteServiceMock); + + var harmonyResource = await _resourceService.GetResource(crdtResource.Id); + harmonyResource.Should().NotBeNull(); + harmonyResource.Id.Should().Be(crdtResource.Id); + harmonyResource.RemoteId.Should().BeNull(); + harmonyResource.LocalPath.Should().Be(localFile); + var pendingUpload = await _resourceService.ListResourcesPendingUpload(); + pendingUpload.Should().ContainSingle().Which.Id.Should().Be(crdtResource.Id); + } + + [Fact] + public async Task WillUploadMultiplePendingLocalFilesAtOnce() + { + await SetupLocalFile("file1", "file1"); + await SetupLocalFile("file2", "file2"); + + //act + await _resourceService.UploadPendingResources(_localClientId, _remoteServiceMock); + + + _remoteServiceMock.ListRemoteFiles() + .Select(Path.GetFileName) + .Should() + .Contain(["file1.txt", "file2.txt"]); + } + + [Fact] + public async Task CanDownloadFileFromRemote() + { + var fileContents = "resource"; + var (resourceId, _) = await SetupRemoteResource(fileContents); + + //act + var localResource = await _resourceService.DownloadResource(resourceId, _remoteServiceMock); + + + localResource.Id.Should().Be(resourceId); + var actualFileContents = await File.ReadAllTextAsync(localResource.LocalPath); + actualFileContents.Should().Be(fileContents); + var pendingDownloads = await _resourceService.ListResourcesPendingDownload(); + pendingDownloads.Should().BeEmpty(); + } + + [Fact] + public async Task CanGetALocalResourceGivenAnId() + { + var file = CreateFile("resource"); + //because resource service is null the file is not uploaded + var crdtResource = await _resourceService.AddLocalResource(file, _localClientId, resourceService: null); + + //act + var localResource = await _resourceService.GetLocalResource(crdtResource.Id); + + + localResource.Should().NotBeNull(); + localResource!.LocalPath.Should().Be(file); + } + + [Fact] + public async Task LocalResourceIsNullIfNotDownloaded() + { + var (resourceId, _) = await SetupRemoteResource("test"); + var localResource = await _resourceService.GetLocalResource(resourceId); + localResource.Should().BeNull(); + } + + [Fact] + public async Task CanListAllResources() + { + var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt"); + var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly"); + var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), _localClientId, resourceService: _remoteServiceMock); + + var crdtResources = await _resourceService.AllResources(); + crdtResources.Should().BeEquivalentTo( + [ + new HarmonyResource + { + Id = localResourceId, + LocalPath = localResourcePath, + RemoteId = null + }, + new HarmonyResource + { + Id = remoteResourceId, + LocalPath = null, + RemoteId = remoteId + }, + localAndRemoteResource + ]); + } + + [Fact] + public async Task CanGetAResourceGivenAnId() + { + var (localResourceId, localResourcePath) = await SetupLocalFile("localOnly", "localOnly.txt"); + var (remoteResourceId, remoteId) = await SetupRemoteResource("remoteOnly"); + var localAndRemoteResource = await _resourceService.AddLocalResource(CreateFile("localAndRemove"), + _localClientId, + resourceService: _remoteServiceMock); + + (await _resourceService.GetResource(localResourceId)).Should().BeEquivalentTo(new HarmonyResource + { + Id = localResourceId, + LocalPath = localResourcePath, + RemoteId = null + }); + (await _resourceService.GetResource(remoteResourceId)).Should().BeEquivalentTo(new HarmonyResource + { + Id = remoteResourceId, + LocalPath = null, + RemoteId = remoteId + }); + (await _resourceService.GetResource(localAndRemoteResource.Id)).Should().BeEquivalentTo(localAndRemoteResource); + (await _resourceService.GetResource(Guid.NewGuid())).Should().BeNull(); + } +} diff --git a/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs new file mode 100644 index 0000000..6266ce4 --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/RemoteServiceMock.cs @@ -0,0 +1,61 @@ +using SIL.Harmony.Core; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class RemoteServiceMock : IRemoteResourceService +{ + public static readonly string RemotePath = Directory.CreateTempSubdirectory("RemoteServiceMock").FullName; + + /// + /// directly creates a remote resource + /// + /// the remote id + public string CreateRemoteResource(string contents) + { + var filePath = Path.Combine(RemotePath, Guid.NewGuid().ToString("N") + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + public Task DownloadResource(string remoteId, string localResourceCachePath) + { + var fileName = Path.GetFileName(remoteId); + var localPath = Path.Combine(localResourceCachePath, fileName); + Directory.CreateDirectory(localResourceCachePath); + File.Copy(remoteId, localPath); + return Task.FromResult(new DownloadResult(localPath)); + } + + private readonly Queue _throwOnUpload = new(); + + public async Task UploadResource(string localPath) + { + await Task.Yield();//yield back to the scheduler to emulate how exceptions are thrown + if (_throwOnUpload.TryPeek(out var throwOnUpload)) + { + if (throwOnUpload == localPath) + { + _throwOnUpload.Dequeue(); + throw new Exception($"Simulated upload failure for {localPath}"); + } + } + var remoteId = Path.Combine(RemotePath, Path.GetFileName(localPath)); + File.Copy(localPath, remoteId); + return new UploadResult(remoteId); + } + + public void ThrowOnUpload(string localPath) + { + _throwOnUpload.Enqueue(localPath); + } + + public string ReadFile(string remoteId) + { + return File.ReadAllText(remoteId); + } + + public IEnumerable ListRemoteFiles() + { + return Directory.GetFiles(RemotePath); + } +} diff --git a/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs new file mode 100644 index 0000000..e7fc702 --- /dev/null +++ b/src/SIL.Harmony.Tests/ResourceTests/WordResourceTests.cs @@ -0,0 +1,41 @@ +using System.Runtime.CompilerServices; +using Microsoft.Extensions.DependencyInjection; +using SIL.Harmony.Sample.Changes; +using SIL.Harmony.Sample.Models; + +namespace SIL.Harmony.Tests.ResourceTests; + +public class WordResourceTests: DataModelTestBase +{ + private RemoteServiceMock _remoteServiceMock = new(); + private ResourceService _resourceService => _services.GetRequiredService(); + private readonly Guid _entity1Id = Guid.NewGuid(); + + private string CreateFile(string contents, [CallerMemberName] string fileName = "") + { + var filePath = Path.GetFullPath(fileName + ".txt"); + File.WriteAllText(filePath, contents); + return filePath; + } + + [Fact] + public async Task CanReferenceAResourceFromAWord() + { + await WriteNextChange(SetWord(_entity1Id, "test-value")); + var imageFile = CreateFile("not image data"); + //set commit date for add local resource + MockTimeProvider.SetNextDateTime(NextDate()); + var resource = await _resourceService.AddLocalResource(imageFile, Guid.NewGuid(), resourceService: _remoteServiceMock); + await WriteNextChange(new AddWordImageChange(_entity1Id, resource.Id)); + + var word = await DataModel.GetLatest(_entity1Id); + word.Should().NotBeNull(); + word!.ImageResourceId.Should().Be(resource.Id); + + + var localResource = await _resourceService.GetLocalResource(word.ImageResourceId!.Value); + localResource.Should().NotBeNull(); + localResource!.LocalPath.Should().Be(imageFile); + (await File.ReadAllTextAsync(localResource.LocalPath)).Should().Be("not image data"); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/Changes/EditChange.cs b/src/SIL.Harmony/Changes/EditChange.cs index 5d3d70c..cdece9b 100644 --- a/src/SIL.Harmony/Changes/EditChange.cs +++ b/src/SIL.Harmony/Changes/EditChange.cs @@ -1,4 +1,4 @@ -namespace SIL.Harmony.Changes; +namespace SIL.Harmony.Changes; public abstract class EditChange(Guid entityId) : Change(entityId) where T : class diff --git a/src/SIL.Harmony/CrdtConfig.cs b/src/SIL.Harmony/CrdtConfig.cs index 6c16337..76ef9ea 100644 --- a/src/SIL.Harmony/CrdtConfig.cs +++ b/src/SIL.Harmony/CrdtConfig.cs @@ -1,4 +1,4 @@ -using System.Text.Json; +using System.Text.Json; using System.Text.Json.Serialization.Metadata; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.Metadata.Builders; @@ -6,6 +6,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Entities; +using SIL.Harmony.Resource; namespace SIL.Harmony; public delegate ValueTask BeforeSaveObjectDelegate(object obj, ObjectSnapshot snapshot); @@ -68,6 +69,25 @@ private void JsonTypeModifier(JsonTypeInfo typeInfo) } } } + + public bool RemoteResourcesEnabled { get; private set; } + public string LocalResourceCachePath { get; set; } = Path.GetFullPath("./localResourceCache"); + public void AddRemoteResourceEntity(string? cachePath = null) + { + RemoteResourcesEnabled = true; + LocalResourceCachePath = cachePath ?? LocalResourceCachePath; + ObjectTypeListBuilder.DefaultAdapter().Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add(); + ChangeTypeListBuilder.Add>(); + ObjectTypeListBuilder.ModelConfigurations.Add((builder, config) => + { + var entity = builder.Entity(); + entity.HasKey(lr => lr.Id); + entity.Property(lr => lr.LocalPath); + }); + } } public class ChangeTypeListBuilder diff --git a/src/SIL.Harmony/CrdtKernel.cs b/src/SIL.Harmony/CrdtKernel.cs index 67671be..724ae99 100644 --- a/src/SIL.Harmony/CrdtKernel.cs +++ b/src/SIL.Harmony/CrdtKernel.cs @@ -1,6 +1,7 @@ using System.Text.Json; using SIL.Harmony.Core; using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; using SIL.Harmony.Db; @@ -11,6 +12,7 @@ public static class CrdtKernel public static IServiceCollection AddCrdtData(this IServiceCollection services, Action configureCrdt) where TContext: ICrdtDbContext { + services.AddLogging(); services.AddOptions().Configure(configureCrdt).PostConfigure(crdtConfig => crdtConfig.ObjectTypeListBuilder.Freeze()); services.AddSingleton(sp => sp.GetRequiredService>().Value.JsonSerializerOptions); services.AddSingleton(TimeProvider.System); @@ -26,6 +28,13 @@ public static IServiceCollection AddCrdtData(this IServiceCollection s provider.GetRequiredService(), provider.GetRequiredService>() )); + //must use factory method because ResourceService constructor is internal + services.AddScoped(provider => new ResourceService( + provider.GetRequiredService(), + provider.GetRequiredService>(), + provider.GetRequiredService(), + provider.GetRequiredService>() + )); return services; } diff --git a/src/SIL.Harmony/DataModel.cs b/src/SIL.Harmony/DataModel.cs index 7f34d8a..a8beded 100644 --- a/src/SIL.Harmony/DataModel.cs +++ b/src/SIL.Harmony/DataModel.cs @@ -5,6 +5,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Db; using SIL.Harmony.Entities; +using SIL.Harmony.Resource; namespace SIL.Harmony; @@ -87,8 +88,7 @@ public async Task AddChanges( private async Task Add(Commit commit, bool deferSnapshotUpdates) { if (await _crdtRepository.HasCommit(commit.Id)) return; - - await using var transaction = await _crdtRepository.BeginTransactionAsync(); + await using var transaction = _crdtRepository.IsInTransaction ? null : await _crdtRepository.BeginTransactionAsync(); await _crdtRepository.AddCommit(commit); if (!deferSnapshotUpdates) { @@ -101,8 +101,7 @@ private async Task Add(Commit commit, bool deferSnapshotUpdates) { _deferredCommits.Add(commit); } - - await transaction.CommitAsync(); + if (transaction is not null) await transaction.CommitAsync(); } public async ValueTask DisposeAsync() diff --git a/src/SIL.Harmony/Db/CrdtRepository.cs b/src/SIL.Harmony/Db/CrdtRepository.cs index fb86c6d..39de888 100644 --- a/src/SIL.Harmony/Db/CrdtRepository.cs +++ b/src/SIL.Harmony/Db/CrdtRepository.cs @@ -1,4 +1,4 @@ -using SIL.Harmony.Core; +using SIL.Harmony.Core; using Microsoft.EntityFrameworkCore; using Microsoft.EntityFrameworkCore.ChangeTracking; using Microsoft.EntityFrameworkCore.Infrastructure; @@ -7,6 +7,7 @@ using SIL.Harmony.Changes; using SIL.Harmony.Entities; using SIL.Harmony.Helpers; +using SIL.Harmony.Resource; namespace SIL.Harmony.Db; @@ -35,6 +36,9 @@ public Task BeginTransactionAsync() return _dbContext.Database.BeginTransactionAsync(); } + public bool IsInTransaction => _dbContext.Database.CurrentTransaction is not null; + + public async Task HasCommit(Guid commitId) { return await Commits.AnyAsync(c => c.Id == commitId); @@ -294,6 +298,36 @@ public async Task AddCommits(IEnumerable commits, bool save = true) .Select(c => c.HybridDateTime) .FirstOrDefault(); } + + + public async Task AddLocalResource(LocalResource localResource) + { + _dbContext.Set().Add(localResource); + await _dbContext.SaveChangesAsync(); + } + + public IAsyncEnumerable LocalResourcesByIds(IEnumerable resourceIds) + { + return _dbContext.Set().Where(r => resourceIds.Contains(r.Id)).AsAsyncEnumerable(); + } + public IAsyncEnumerable LocalResources() + { + return _dbContext.Set().AsAsyncEnumerable(); + } + + /// + /// primarily for filtering other queries + /// + public IQueryable LocalResourceIds() + { + return _dbContext.Set().Select(r => r.Id); + } + + public async Task GetLocalResource(Guid resourceId) + { + return await _dbContext.Set().FindAsync(resourceId); + } + } internal class ScopedDbContext(ICrdtDbContext inner, Commit ignoreChangesAfter) : ICrdtDbContext diff --git a/src/SIL.Harmony/Helpers/LinqHelpers.cs b/src/SIL.Harmony/Helpers/LinqHelpers.cs new file mode 100644 index 0000000..c643928 --- /dev/null +++ b/src/SIL.Harmony/Helpers/LinqHelpers.cs @@ -0,0 +1,31 @@ +using System.Diagnostics.CodeAnalysis; + +namespace SIL.Harmony.Helpers; + +public static class LinqHelpers +{ + internal static IEnumerable FullOuterJoin( + this IEnumerable a, + IEnumerable b, + Func selectKeyA, + Func selectKeyB, + Func projection, + TA? defaultA = default(TA), + TB? defaultB = default(TB), + IEqualityComparer? cmp = null) + { + cmp ??= EqualityComparer.Default; + var alookup = a.ToLookup(selectKeyA, cmp); + var blookup = b.ToLookup(selectKeyB, cmp); + + var keys = new HashSet(alookup.Select(p => p.Key), cmp); + keys.UnionWith(blookup.Select(p => p.Key)); + + var join = from key in keys + from xa in alookup[key].DefaultIfEmpty(defaultA) + from xb in blookup[key].DefaultIfEmpty(defaultB) + select projection(xa, xb, key); + + return join; + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs new file mode 100644 index 0000000..aa4f821 --- /dev/null +++ b/src/SIL.Harmony/Resource/CreateRemoteResourceChange.cs @@ -0,0 +1,20 @@ +using SIL.Harmony.Core; +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class CreateRemoteResourceChange(Guid resourceId, string remoteId) : CreateChange(resourceId), IPolyType +{ + public string RemoteId { get; set; } = remoteId; + public override ValueTask NewEntity(Commit commit, ChangeContext context) + { + return ValueTask.FromResult(new RemoteResource + { + Id = EntityId, + RemoteId = RemoteId + }); + } + + public static string TypeName => "create:remote-resource"; +} diff --git a/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs new file mode 100644 index 0000000..4859f8d --- /dev/null +++ b/src/SIL.Harmony/Resource/CreateRemoteResourcePendingUpload.cs @@ -0,0 +1,21 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +public class CreateRemoteResourcePendingUploadChange: CreateChange, IPolyType +{ + public CreateRemoteResourcePendingUploadChange(Guid resourceId) : base(resourceId) + { + } + + public override ValueTask NewEntity(Commit commit, ChangeContext context) + { + return ValueTask.FromResult(new RemoteResource + { + Id = EntityId + }); + } + + public static string TypeName => "create:pendingUpload"; +} diff --git a/src/SIL.Harmony/Resource/HarmonyResource.cs b/src/SIL.Harmony/Resource/HarmonyResource.cs new file mode 100644 index 0000000..072f29d --- /dev/null +++ b/src/SIL.Harmony/Resource/HarmonyResource.cs @@ -0,0 +1,10 @@ +namespace SIL.Harmony.Resource; + +public class HarmonyResource +{ + public required Guid Id { get; init; } + public string? RemoteId { get; init; } + public string? LocalPath { get; init; } + public bool Local => !string.IsNullOrEmpty(LocalPath); + public bool Remote => !string.IsNullOrEmpty(RemoteId); +} \ No newline at end of file diff --git a/src/SIL.Harmony/Resource/LocalResource.cs b/src/SIL.Harmony/Resource/LocalResource.cs new file mode 100644 index 0000000..0beaf4c --- /dev/null +++ b/src/SIL.Harmony/Resource/LocalResource.cs @@ -0,0 +1,16 @@ +namespace SIL.Harmony.Resource; + +/// +/// a non CRDT object that tracks local resource files +/// +public class LocalResource +{ + public required Guid Id { get; set; } + //could probably be a URL if working in an electron context, not sure what would be best here. It depends on the app + public required string LocalPath { get; set; } + + public bool FileExists() + { + return File.Exists(LocalPath); + } +} diff --git a/src/SIL.Harmony/Resource/RemoteResource.cs b/src/SIL.Harmony/Resource/RemoteResource.cs new file mode 100644 index 0000000..fd13bdd --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResource.cs @@ -0,0 +1,34 @@ +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +/// +/// represents a remote binary resource (e.g. image, video, audio, etc.) +/// +public class RemoteResource: IObjectBase +{ + public Guid Id { get; init; } + public DateTimeOffset? DeletedAt { get; set; } + /// + /// will be null when the resource has not been uploaded yet + /// + public string? RemoteId { get; set; } + public Guid[] GetReferences() + { + return []; + } + + public void RemoveReference(Guid id, Commit commit) + { + } + + public IObjectBase Copy() + { + return new RemoteResource + { + Id = Id, + RemoteId = RemoteId, + DeletedAt = DeletedAt + }; + } +} diff --git a/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs new file mode 100644 index 0000000..aaa37a0 --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResourceNotEnabledException.cs @@ -0,0 +1,4 @@ +namespace SIL.Harmony.Resource; + +public class RemoteResourceNotEnabledException() + : Exception("remote resources were not enabled, to enable them call CrdtConfig.AddRemoteResourceEntity when adding the CRDT library"); diff --git a/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs new file mode 100644 index 0000000..bdff98b --- /dev/null +++ b/src/SIL.Harmony/Resource/RemoteResourceUploadedChange.cs @@ -0,0 +1,21 @@ +using SIL.Harmony.Changes; +using SIL.Harmony.Entities; + +namespace SIL.Harmony.Resource; + +/// +/// used when a resource is uploaded to the remote server, stores the remote url in the resource entity +/// +/// +/// +public class RemoteResourceUploadedChange(Guid entityId, string remoteId) : EditChange(entityId), IPolyType +{ + public string RemoteId { get; set; } = remoteId; + public static string TypeName => "uploaded:RemoteResource"; + + public override ValueTask ApplyChange(RemoteResource entity, ChangeContext context) + { + entity.RemoteId = RemoteId; + return ValueTask.CompletedTask; + } +} diff --git a/src/SIL.Harmony/ResourceService.cs b/src/SIL.Harmony/ResourceService.cs new file mode 100644 index 0000000..f21d38d --- /dev/null +++ b/src/SIL.Harmony/ResourceService.cs @@ -0,0 +1,185 @@ +using Microsoft.EntityFrameworkCore; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Options; +using SIL.Harmony.Changes; +using SIL.Harmony.Core; +using SIL.Harmony.Db; +using SIL.Harmony.Helpers; +using SIL.Harmony.Resource; + +namespace SIL.Harmony; + +public class ResourceService +{ + private readonly CrdtRepository _crdtRepository; + private readonly IOptions _crdtConfig; + private readonly DataModel _dataModel; + private readonly ILogger _logger; + + internal ResourceService(CrdtRepository crdtRepository, IOptions crdtConfig, DataModel dataModel, ILogger logger) + { + _crdtRepository = crdtRepository; + _crdtConfig = crdtConfig; + _dataModel = dataModel; + _logger = logger; + } + + private void ValidateResourcesSetup() + { + if (!_crdtConfig.Value.RemoteResourcesEnabled) throw new RemoteResourceNotEnabledException(); + } + + public async Task AddLocalResource(string resourcePath, + Guid clientId, + Guid id = default, + IRemoteResourceService? resourceService = null) + { + ValidateResourcesSetup(); + var localResource = new LocalResource + { + Id = id == default ? Guid.NewGuid() : id, + LocalPath = Path.GetFullPath(resourcePath) + }; + if (!localResource.FileExists()) throw new FileNotFoundException(localResource.LocalPath); + await using var transaction = await _crdtRepository.BeginTransactionAsync(); + await _crdtRepository.AddLocalResource(localResource); + UploadResult? uploadResult = null; + if (resourceService is not null) + { + try + { + + uploadResult = await resourceService.UploadResource(localResource.LocalPath); + } + catch (Exception e) + { + _logger.LogError(e, "Error uploading resource {resourcePath}, resource will be marked as pending upload", localResource.LocalPath); + } + } + + if (uploadResult is not null) + { + await _dataModel.AddChange(clientId, new CreateRemoteResourceChange(localResource.Id, uploadResult.RemoteId)); + } + else + { + await _dataModel.AddChange(clientId, new CreateRemoteResourcePendingUploadChange(localResource.Id)); + } + + await transaction.CommitAsync(); + return new HarmonyResource + { + Id = localResource.Id, + RemoteId = uploadResult?.RemoteId, + LocalPath = localResource.LocalPath + }; + } + + public async Task ListResourcesPendingUpload() + { + ValidateResourcesSetup(); + var remoteResources = await _dataModel.QueryLatest().Where(r => r.RemoteId == null).ToArrayAsync(); + var localResource = _crdtRepository.LocalResourcesByIds(remoteResources.Select(r => r.Id)); + return await localResource.ToArrayAsync(); + } + + public async Task UploadPendingResources(Guid clientId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + var pendingUploads = await ListResourcesPendingUpload(); + var changes = new List(pendingUploads.Length); + try + { + foreach (var localResource in pendingUploads) + { + var uploadResult = await remoteResourceService.UploadResource(localResource.LocalPath); + changes.Add(new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + } + finally + { + //if upload throws at any point we will at least save the changes that did get made. + await _dataModel.AddChanges(clientId, changes); + } + } + + public async Task UploadPendingResource(Guid resourceId, Guid clientId, IRemoteResourceService remoteResourceService) + { + var localResource = await _crdtRepository.GetLocalResource(resourceId) ?? + throw new ArgumentException($"unable to find local resource with id {resourceId}"); + ValidateResourcesSetup(); + await UploadPendingResource(localResource, clientId, remoteResourceService); + } + + public async Task UploadPendingResource(LocalResource localResource, Guid clientId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + var uploadResult = await remoteResourceService.UploadResource(localResource.LocalPath); + await _dataModel.AddChange(clientId, new RemoteResourceUploadedChange(localResource.Id, uploadResult.RemoteId)); + } + + public async Task ListResourcesPendingDownload() + { + ValidateResourcesSetup(); + var localResourceIds = _crdtRepository.LocalResourceIds(); + var remoteResources = await _dataModel.QueryLatest() + .Where(r => r.RemoteId != null && !localResourceIds.Contains(r.Id)) + .ToArrayAsync(); + return remoteResources; + } + + public async Task DownloadResource(Guid resourceId, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + return await DownloadResource( + await _dataModel.GetLatest(resourceId) ?? + throw new EntityNotFoundException("Unable to find remote resource"), + remoteResourceService + ); + } + + public async Task DownloadResource(RemoteResource remoteResource, IRemoteResourceService remoteResourceService) + { + ValidateResourcesSetup(); + ArgumentNullException.ThrowIfNull(remoteResource.RemoteId); + var downloadResult = await remoteResourceService.DownloadResource(remoteResource.RemoteId, _crdtConfig.Value.LocalResourceCachePath); + var localResource = new LocalResource + { + Id = remoteResource.Id, + LocalPath = downloadResult.LocalPath + }; + await _crdtRepository.AddLocalResource(localResource); + return localResource; + } + + public async Task GetLocalResource(Guid resourceId) + { + return await _crdtRepository.GetLocalResource(resourceId); + } + + public async Task AllResources() + { + return (await AllResourcesInternal()).ToArray(); + } + + private async Task> AllResourcesInternal() + { + var remoteResources = await _dataModel.QueryLatest().ToArrayAsync(); + var localResources = await _crdtRepository.LocalResources().ToArrayAsync(); + return remoteResources.FullOuterJoin(localResources, + r => r.Id, + l => l.Id, + (r, l, id) => new HarmonyResource + { + Id = id, + RemoteId = r?.RemoteId, + LocalPath = l?.LocalPath + }); + } + + public async Task GetResource(Guid resourceId) + { + var resources = await AllResourcesInternal(); + return resources.FirstOrDefault(r => r.Id == resourceId); + } +} \ No newline at end of file diff --git a/src/SIL.Harmony/SyncHelper.cs b/src/SIL.Harmony/SyncHelper.cs index 4e108fe..22fec59 100644 --- a/src/SIL.Harmony/SyncHelper.cs +++ b/src/SIL.Harmony/SyncHelper.cs @@ -5,6 +5,16 @@ namespace SIL.Harmony; internal static class SyncHelper { + public static async Task SyncWithResourceUpload(this DataModel localModel, + ISyncable remoteModel, + ResourceService resourceService, + IRemoteResourceService remoteResourceService, + Guid localClientId) + { + await resourceService.UploadPendingResources(localClientId, remoteResourceService); + return await localModel.SyncWith(remoteModel); + } + /// /// simple sync example, each ISyncable could be over the wire or in memory /// prefer that remote is over the wire for the best performance, however they could both be remote