diff --git a/src/Echo/src/EchoEngine/Usings.cs b/src/Echo/src/EchoEngine/Usings.cs index 9c6b61ba..98613a11 100644 --- a/src/Echo/src/EchoEngine/Usings.cs +++ b/src/Echo/src/EchoEngine/Usings.cs @@ -4,4 +4,4 @@ global using Google.Protobuf.WellKnownTypes; global using Grpc.Core; global using Microsoft.Extensions.Diagnostics.HealthChecks; -global using SIL.ServiceToolkit.Utils; +global using SIL.ServiceToolkit.Services; diff --git a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs index 8601e5e7..90b9be97 100644 --- a/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs +++ b/src/Machine/src/Serval.Machine.Shared/Configuration/IMachineBuilderExtensions.cs @@ -297,6 +297,7 @@ public static IMachineBuilder AddServalTranslationPlatformService(this IMachineB x.AddConsumer(); x.AddConsumer(); x.AddConsumer(); + x.AddConsumer(); }); builder diff --git a/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateParallelCorpusAnalysisConsumer.cs b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateParallelCorpusAnalysisConsumer.cs new file mode 100644 index 00000000..cbedaf71 --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Consumers/TranslationUpdateParallelCorpusAnalysisConsumer.cs @@ -0,0 +1,10 @@ +using Serval.Translation.V1; + +namespace Serval.Machine.Shared.Consumers; + +public class TranslationUpdateParallelCorpusAnalysisConsumer(TranslationPlatformApi.TranslationPlatformApiClient client) + : ServalPlatformConsumerBase( + ServalTranslationPlatformOutboxConstants.OutboxId, + ServalTranslationPlatformOutboxConstants.UpdateParallelCorpusAnalysis, + client.UpdateParallelCorpusAnalysisAsync + ) { } diff --git a/src/Machine/src/Serval.Machine.Shared/Models/ParallelCorpusAnalysis.cs b/src/Machine/src/Serval.Machine.Shared/Models/ParallelCorpusAnalysis.cs new file mode 100644 index 00000000..681e866f --- /dev/null +++ b/src/Machine/src/Serval.Machine.Shared/Models/ParallelCorpusAnalysis.cs @@ -0,0 +1,8 @@ +namespace Serval.Machine.Shared.Models; + +public record ParallelCorpusAnalysis +{ + public required string ParallelCorpusRef { get; init; } + public required string SourceQuoteConvention { get; init; } + public required string TargetQuoteConvention { get; init; } +} diff --git a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs index 3a296d12..df55c522 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/IPlatformService.cs @@ -36,4 +36,11 @@ Task UpdateBuildExecutionDataAsync( IReadOnlyDictionary executionData, CancellationToken cancellationToken = default ); + + Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyCollection parallelCorpusAnalysis, + CancellationToken cancellationToken = default + ); } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs index 2698932a..baa1eaf4 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/NmtPreprocessBuildJob.cs @@ -26,4 +26,42 @@ protected override bool ResolveLanguageCodeForBaseModel(string languageCode, out { return _languageTagService.ConvertToFlores200Code(languageCode, out resolvedCode); } + + protected override async Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + List parallelCorpusAnalysis = []; + foreach (ParallelCorpus parallelCorpus in corpora) + { + (QuoteConventionAnalysis? sourceQuotationConvention, QuoteConventionAnalysis? targetQuotationConvention) = + ParallelCorpusPreprocessingService.AnalyzeParallelCorpus(parallelCorpus); + string sourceQuotationConventionName = sourceQuotationConvention?.BestQuoteConvention.Name ?? string.Empty; + string targetQuotationConventionName = targetQuotationConvention?.BestQuoteConvention.Name ?? string.Empty; + if ( + !string.IsNullOrWhiteSpace(sourceQuotationConventionName) + || !string.IsNullOrWhiteSpace(sourceQuotationConventionName) + ) + { + parallelCorpusAnalysis.Add( + new ParallelCorpusAnalysis + { + ParallelCorpusRef = parallelCorpus.Id, + SourceQuoteConvention = sourceQuotationConventionName, + TargetQuoteConvention = targetQuotationConventionName, + } + ); + } + } + + await PlatformService.UpdateParallelCorpusAnalysisAsync( + engineId, + buildId, + parallelCorpusAnalysis, + cancellationToken + ); + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs index eb7212ae..0c200b75 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/PreprocessBuildJob.cs @@ -58,6 +58,8 @@ await UpdateBuildExecutionData( cancellationToken ); + await UpdateParallelCorpusAnalysisAsync(engineId, buildId, data, cancellationToken); + if (trainCount == 0 && (!sourceTagInBaseModel || !targetTagInBaseModel)) { throw new InvalidOperationException( @@ -90,6 +92,13 @@ protected abstract Task UpdateBuildExecutionData( CancellationToken cancellationToken ); + protected virtual Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) => Task.CompletedTask; + protected abstract Task<(int TrainCount, int InferenceCount)> WriteDataFilesAsync( string buildId, IReadOnlyList corpora, diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs index 863fb358..ccda89b0 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformOutboxConstants.cs @@ -12,4 +12,5 @@ public static class ServalTranslationPlatformOutboxConstants public const string InsertPretranslations = "InsertPretranslations"; public const string IncrementEngineCorpusSize = "IncrementTrainEngineCorpusSize"; public const string UpdateBuildExecutionData = "UpdateBuildExecutionData"; + public const string UpdateParallelCorpusAnalysis = "UpdateParallelCorpusAnalysis"; } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs index 73023b81..1d6f75ba 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalTranslationPlatformService.cs @@ -163,4 +163,33 @@ await _outboxService.EnqueueMessageAsync( cancellationToken: cancellationToken ); } + + public async Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyCollection parallelCorpusAnalysis, + CancellationToken cancellationToken = default + ) + { + var content = new UpdateParallelCorpusAnalysisRequest { EngineId = engineId, BuildId = buildId }; + foreach (ParallelCorpusAnalysis analysis in parallelCorpusAnalysis) + { + content.ParallelCorpusAnalysis.Add( + new ParallelCorpusAnalysisResult + { + ParallelCorpusId = analysis.ParallelCorpusRef, + SourceQuoteConvention = analysis.SourceQuoteConvention, + TargetQuoteConvention = analysis.TargetQuoteConvention, + } + ); + } + + await _outboxService.EnqueueMessageAsync( + outboxId: ServalTranslationPlatformOutboxConstants.OutboxId, + method: ServalTranslationPlatformOutboxConstants.UpdateParallelCorpusAnalysis, + groupId: engineId, + content, + cancellationToken: cancellationToken + ); + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs index a6a0840a..bc0ed075 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/ServalWordAlignmentPlatformService.cs @@ -163,4 +163,15 @@ await _outboxService.EnqueueMessageAsync( cancellationToken: cancellationToken ); } + + public Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyCollection parallelCorpusAnalysis, + CancellationToken cancellationToken = default + ) + { + // Word alignment does not support parallel corpus analysis + return Task.CompletedTask; + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs index 0515be75..7d774f8d 100644 --- a/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs +++ b/src/Machine/src/Serval.Machine.Shared/Services/WordAlignmentPreprocessBuildJob.cs @@ -112,4 +112,15 @@ CancellationToken cancellationToken }; await PlatformService.UpdateBuildExecutionDataAsync(engineId, buildId, executionData, cancellationToken); } + + protected override Task UpdateParallelCorpusAnalysisAsync( + string engineId, + string buildId, + IReadOnlyList corpora, + CancellationToken cancellationToken + ) + { + // Word alignment does not support parallel corpus analysis + return Task.CompletedTask; + } } diff --git a/src/Machine/src/Serval.Machine.Shared/Usings.cs b/src/Machine/src/Serval.Machine.Shared/Usings.cs index ff8aa0c5..191f648a 100644 --- a/src/Machine/src/Serval.Machine.Shared/Usings.cs +++ b/src/Machine/src/Serval.Machine.Shared/Usings.cs @@ -52,6 +52,7 @@ global using SIL.DataAccess; global using SIL.Machine.Corpora; global using SIL.Machine.Morphology.HermitCrab; +global using SIL.Machine.PunctuationAnalysis; global using SIL.Machine.Tokenization; global using SIL.Machine.Translation; global using SIL.Machine.Translation.Thot; diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs index 2cd9ed9b..35e863f3 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/PreprocessBuildJobTests.cs @@ -297,13 +297,17 @@ public async Task RunAsync_RemoveFreestandingEllipses() string sourceExtract = await env.GetSourceExtractAsync(); Assert.That( sourceExtract, - Is.EqualTo("Source one, chapter two, verse one.\nSource one, chapter two, verse two.\n\n"), + Is.EqualTo( + "Source one, chapter two, verse one.\nSource one, chapter two, verse two. \u201ca quotation\u201d\n\n" + ), sourceExtract ); string targetExtract = await env.GetTargetExtractAsync(); Assert.That( targetExtract, - Is.EqualTo("Target one, chapter two, verse one.\n\nTarget one, chapter two, verse three.\n"), + Is.EqualTo( + "Target one, chapter two, verse one.\n\nTarget one, chapter two, verse three. \"a quotation\"\n" + ), targetExtract ); JsonArray? pretranslations = await env.GetPretranslationsAsync(); diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM index bca7c590..b5ba9651 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-source1/41MATTe1.SFM @@ -14,6 +14,6 @@ \c 2 \p \v 1 Source one, chapter two, verse one. -\v 2 Source one, chapter two, verse two. +\v 2 Source one, chapter two, verse two. “a quotation” \v 3 ... \v 4 ... diff --git a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM index 202c3ae1..7efc5539 100644 --- a/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM +++ b/src/Machine/test/Serval.Machine.Shared.Tests/Services/data/pt-target1/41MATTe2.SFM @@ -15,4 +15,4 @@ \p \v 1 Target one, chapter two, verse one. \v 2 ... -\v 3 Target one, chapter two, verse three. +\v 3 Target one, chapter two, verse three. "a quotation" diff --git a/src/Serval/src/Serval.Client/Client.g.cs b/src/Serval/src/Serval.Client/Client.g.cs index 0d93a4ca..da2c01b4 100644 --- a/src/Serval/src/Serval.Client/Client.g.cs +++ b/src/Serval/src/Serval.Client/Client.g.cs @@ -10407,6 +10407,9 @@ public partial class TranslationBuild [Newtonsoft.Json.JsonProperty("phases", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] public System.Collections.Generic.IList? Phases { get; set; } = default!; + [Newtonsoft.Json.JsonProperty("analysis", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)] + public System.Collections.Generic.IList? Analysis { get; set; } = default!; + } [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] @@ -10522,6 +10525,23 @@ public enum PhaseStage } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] + public partial class ParallelCorpusAnalysis + { + [Newtonsoft.Json.JsonProperty("parallelCorpusRef", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string ParallelCorpusRef { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("sourceQuoteConvention", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string SourceQuoteConvention { get; set; } = default!; + + [Newtonsoft.Json.JsonProperty("targetQuoteConvention", Required = Newtonsoft.Json.Required.Always)] + [System.ComponentModel.DataAnnotations.Required(AllowEmptyStrings = true)] + public string TargetQuoteConvention { get; set; } = default!; + + } + [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")] public partial class TranslationBuildConfig { diff --git a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto index 8c6b19bb..18ce5c1a 100644 --- a/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto +++ b/src/Serval/src/Serval.Grpc/Protos/serval/translation/v1/platform.proto @@ -17,6 +17,7 @@ service TranslationPlatformApi { rpc IncrementEngineCorpusSize(IncrementEngineCorpusSizeRequest) returns (google.protobuf.Empty); rpc InsertPretranslations(stream InsertPretranslationsRequest) returns (google.protobuf.Empty); rpc UpdateBuildExecutionData(UpdateBuildExecutionDataRequest) returns (google.protobuf.Empty); + rpc UpdateParallelCorpusAnalysis(UpdateParallelCorpusAnalysisRequest) returns (google.protobuf.Empty); } message UpdateBuildStatusRequest { @@ -73,6 +74,18 @@ message UpdateBuildExecutionDataRequest { map execution_data = 3; } +message UpdateParallelCorpusAnalysisRequest { + string engine_id = 1; + string build_id = 2; + repeated ParallelCorpusAnalysisResult parallel_corpus_analysis = 3; +} + +message ParallelCorpusAnalysisResult { + string parallel_corpus_id = 1; + string source_quote_convention = 2; + string target_quote_convention = 3; +} + message Phase { PhaseStage stage = 1; optional int32 step = 2; diff --git a/src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs b/src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs new file mode 100644 index 00000000..b64bad60 --- /dev/null +++ b/src/Serval/src/Serval.Shared/Models/ParallelCorpusAnalysis.cs @@ -0,0 +1,8 @@ +namespace Serval.Shared.Models; + +public record ParallelCorpusAnalysis +{ + public required string ParallelCorpusRef { get; init; } + public required string SourceQuoteConvention { get; init; } + public required string TargetQuoteConvention { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs b/src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs new file mode 100644 index 00000000..0acf2ce4 --- /dev/null +++ b/src/Serval/src/Serval.Translation/Contracts/ParallelCorpusAnalysisDto.cs @@ -0,0 +1,8 @@ +namespace Serval.Translation.Contracts; + +public record ParallelCorpusAnalysisDto +{ + public required string ParallelCorpusRef { get; init; } + public required string SourceQuoteConvention { get; init; } + public required string TargetQuoteConvention { get; init; } +} diff --git a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs index 51bfa374..0a74efa5 100644 --- a/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs +++ b/src/Serval/src/Serval.Translation/Contracts/TranslationBuildDto.cs @@ -33,4 +33,5 @@ public record TranslationBuildDto public string? DeploymentVersion { get; init; } public IReadOnlyDictionary? ExecutionData { get; init; } public IReadOnlyList? Phases { get; init; } + public IReadOnlyList? Analysis { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs index 536e1a8a..3ca47f0e 100644 --- a/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs +++ b/src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs @@ -1662,7 +1662,8 @@ private TranslationBuildDto Map(Build source) Options = source.Options, DeploymentVersion = source.DeploymentVersion, ExecutionData = source.ExecutionData, - Phases = source.Phases?.Select(Map).ToList() + Phases = source.Phases?.Select(Map).ToList(), + Analysis = source.Analysis?.Select(Map).ToList(), }; } @@ -1890,6 +1891,16 @@ private static PhaseDto Map(BuildPhase source) StepCount = source.StepCount }; } + + private static ParallelCorpusAnalysisDto Map(ParallelCorpusAnalysis source) + { + return new ParallelCorpusAnalysisDto + { + ParallelCorpusRef = source.ParallelCorpusRef, + SourceQuoteConvention = source.SourceQuoteConvention, + TargetQuoteConvention = source.TargetQuoteConvention, + }; + } } #pragma warning restore CS0612 // Type or member is obsolete diff --git a/src/Serval/src/Serval.Translation/Models/Build.cs b/src/Serval/src/Serval.Translation/Models/Build.cs index f87ee755..e3ee1c63 100644 --- a/src/Serval/src/Serval.Translation/Models/Build.cs +++ b/src/Serval/src/Serval.Translation/Models/Build.cs @@ -20,4 +20,5 @@ public record Build : IInitializableEntity public bool? IsInitialized { get; set; } public DateTime? DateCreated { get; set; } public IReadOnlyList? Phases { get; init; } + public IReadOnlyCollection? Analysis { get; init; } } diff --git a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs index 71ce0c29..191605e3 100644 --- a/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs +++ b/src/Serval/src/Serval.Translation/Services/TranslationPlatformServiceV1.cs @@ -293,6 +293,36 @@ await _builds.UpdateAsync( return new Empty(); } + public override async Task UpdateParallelCorpusAnalysis( + UpdateParallelCorpusAnalysisRequest request, + ServerCallContext context + ) + { + // Ensure only parallel corpus IDs are present + Engine? engine = await _engines.GetAsync(request.EngineId, context.CancellationToken); + if (engine == null) + return Empty; + var analysis = request + .ParallelCorpusAnalysis.Where(p => engine.ParallelCorpora.Select(pc => pc.Id).Contains(p.ParallelCorpusId)) + .Select(a => new ParallelCorpusAnalysis + { + ParallelCorpusRef = a.ParallelCorpusId, + SourceQuoteConvention = a.SourceQuoteConvention, + TargetQuoteConvention = a.TargetQuoteConvention, + }) + .ToList(); + if (analysis.Count > 0) + { + await _builds.UpdateAsync( + b => b.Id == request.BuildId && b.EngineRef == request.EngineId, + u => u.Set(b => b.Analysis, analysis), + cancellationToken: context.CancellationToken + ); + } + + return Empty; + } + public override async Task IncrementEngineCorpusSize( IncrementEngineCorpusSizeRequest request, ServerCallContext context diff --git a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs index 31aa71da..89582908 100644 --- a/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs +++ b/src/Serval/test/Serval.Translation.Tests/Services/PlatformServiceTests.cs @@ -1,5 +1,6 @@ using System.Globalization; using Serval.Translation.V1; +using ParallelCorpus = Serval.Shared.Models.ParallelCorpus; using PhaseStage = Serval.Translation.V1.PhaseStage; namespace Serval.Translation.Services; @@ -165,6 +166,103 @@ public async Task UpdateBuildExecutionData() Assert.That(staticCount, Is.EqualTo(0)); } + [Test] + public async Task UpdateParallelCorpusAnalysisAsync() + { + var env = new TestEnvironment(); + + var engine = new Engine + { + Id = "e0", + Owner = "owner1", + Type = "nmt", + SourceLanguage = "en", + TargetLanguage = "es", + ParallelCorpora = + [ + new ParallelCorpus + { + Id = "parallelCorpus01", + SourceCorpora = [], + TargetCorpora = [], + }, + ], + }; + await env.Engines.InsertAsync(engine); + + var build = new Build { Id = "123", EngineRef = "e0" }; + await env.Builds.InsertAsync(build); + + List expected = + [ + new ParallelCorpusAnalysis + { + ParallelCorpusRef = "parallelCorpus01", + SourceQuoteConvention = "standard_english", + TargetQuoteConvention = "typewriter_english", + }, + ]; + + var updateRequest = new UpdateParallelCorpusAnalysisRequest { BuildId = "123", EngineId = engine.Id }; + updateRequest.ParallelCorpusAnalysis.Add( + new ParallelCorpusAnalysisResult + { + ParallelCorpusId = "parallelCorpus01", + SourceQuoteConvention = "standard_english", + TargetQuoteConvention = "typewriter_english", + } + ); + + await env.PlatformService.UpdateParallelCorpusAnalysis(updateRequest, env.ServerCallContext); + + build = await env.Builds.GetAsync(c => c.Id == build.Id); + + Assert.That(build?.Analysis, Is.EqualTo(expected)); + } + + [Test] + public async Task UpdateParallelCorpusAnalysisAsync_NoEngine() + { + var env = new TestEnvironment(); + + var build = new Build { Id = "123", EngineRef = "e0" }; + await env.Builds.InsertAsync(build); + + var updateRequest = new UpdateParallelCorpusAnalysisRequest { BuildId = "123", EngineId = "e0" }; + await env.PlatformService.UpdateParallelCorpusAnalysis(updateRequest, env.ServerCallContext); + + build = await env.Builds.GetAsync(c => c.Id == build.Id); + + Assert.That(build?.Analysis, Is.Null); + } + + [Test] + public async Task UpdateParallelCorpusAnalysisAsync_NoParallelCorpora() + { + var env = new TestEnvironment(); + + var engine = new Engine + { + Id = "e0", + Owner = "owner1", + Type = "nmt", + SourceLanguage = "en", + TargetLanguage = "es", + ParallelCorpora = [], + }; + await env.Engines.InsertAsync(engine); + + var build = new Build { Id = "123", EngineRef = "e0" }; + await env.Builds.InsertAsync(build); + + var updateRequest = new UpdateParallelCorpusAnalysisRequest { BuildId = "123", EngineId = engine.Id }; + await env.PlatformService.UpdateParallelCorpusAnalysis(updateRequest, env.ServerCallContext); + + build = await env.Builds.GetAsync(c => c.Id == build.Id); + + Assert.That(build?.Analysis, Is.Null); + } + [Test] public async Task IncrementCorpusSizeAsync() { diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs index 7dd8e16f..9483fe5c 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/IParallelCorpusPreprocessingService.cs @@ -1,7 +1,8 @@ -namespace SIL.ServiceToolkit.Utils; +namespace SIL.ServiceToolkit.Services; public interface IParallelCorpusPreprocessingService { + (QuoteConventionAnalysis?, QuoteConventionAnalysis?) AnalyzeParallelCorpus(ParallelCorpus corpus); Task PreprocessAsync( IReadOnlyList corpora, Func train, diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs index 61dac31c..bd300ad4 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Services/ParallelCorpusPreprocessingService.cs @@ -5,6 +5,33 @@ public class ParallelCorpusPreprocessingService(ICorpusService corpusService) : private readonly ICorpusService _corpusService = corpusService; private const int Seed = 1234; + public (QuoteConventionAnalysis?, QuoteConventionAnalysis?) AnalyzeParallelCorpus(ParallelCorpus parallelCorpus) + { + var sourceHandler = new QuoteConventionDetector(); + foreach (MonolingualCorpus sourceMonolingualCorpus in parallelCorpus.SourceCorpora) + { + foreach (CorpusFile file in sourceMonolingualCorpus.Files.Where(f => f.Format == FileFormat.Paratext)) + { + using ZipArchive zipArchive = ZipFile.OpenRead(file.Location); + var quoteConventionDetector = new ZipParatextProjectQuoteConventionDetector(zipArchive); + quoteConventionDetector.GetQuoteConventionAnalysis(sourceHandler); + } + } + + var targetHandler = new QuoteConventionDetector(); + foreach (MonolingualCorpus targetMonolingualCorpus in parallelCorpus.TargetCorpora) + { + foreach (CorpusFile file in targetMonolingualCorpus.Files.Where(f => f.Format == FileFormat.Paratext)) + { + using ZipArchive zipArchive = ZipFile.OpenRead(file.Location); + var quoteConventionDetector = new ZipParatextProjectQuoteConventionDetector(zipArchive); + quoteConventionDetector.GetQuoteConventionAnalysis(targetHandler); + } + } + + return (sourceHandler.DetectQuotationConvention(), targetHandler.DetectQuotationConvention()); + } + public async Task PreprocessAsync( IReadOnlyList corpora, Func train, @@ -108,6 +135,7 @@ ParallelTextRow row in parallelKeyTermsCorpus.DistinctBy(row => await inference(row, isInTrainingData, corpus); } } + if (useKeyTerms && parallelTrainingDataPresent) { foreach (Row row in keyTermTrainingData) diff --git a/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs b/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs index 0e53c458..c489e381 100644 --- a/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs +++ b/src/ServiceToolkit/src/SIL.ServiceToolkit/Usings.cs @@ -1,5 +1,6 @@ global using System.Diagnostics; global using System.Diagnostics.CodeAnalysis; +global using System.IO.Compression; global using System.Net; global using System.Runtime.CompilerServices; global using System.Text; @@ -25,6 +26,7 @@ global using MongoDB.Bson.Serialization.Serializers; global using SIL.DataAccess; global using SIL.Machine.Corpora; +global using SIL.Machine.PunctuationAnalysis; global using SIL.ServiceToolkit.Configuration; global using SIL.ServiceToolkit.Models; global using SIL.ServiceToolkit.Services; diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs index ebe1fa78..b4d914a5 100644 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/ParallelCorpusProcessingServiceTests.cs @@ -3,100 +3,199 @@ namespace SIL.ServiceToolkit.Services; [TestFixture] public class ParallelCorpusPreprocessingServiceTests { - private static readonly string TestDataPath = Path.Combine( - AppContext.BaseDirectory, - "..", - "..", - "..", - "Services", - "data" - ); + [Test] + public void TestParallelCorpusAnalysis_FileFormatParatext() + { + using var env = new TestEnvironment(); + ParallelCorpus parallelCorpus = env.GetCorpus(paratextProject: true); + const string ExpectedSourceName = "standard_english"; + const string ExpectedTargetName = "typewriter_english"; + + (QuoteConventionAnalysis? sourceQuotationConvention, QuoteConventionAnalysis? targetQuotationConvention) = + env.Processor.AnalyzeParallelCorpus(parallelCorpus); + + Assert.Multiple(() => + { + Assert.That(sourceQuotationConvention, Is.Not.Null); + Assert.That(sourceQuotationConvention!.BestQuoteConvention.Name, Is.EqualTo(ExpectedSourceName)); + Assert.That(targetQuotationConvention, Is.Not.Null); + Assert.That(targetQuotationConvention!.BestQuoteConvention.Name, Is.EqualTo(ExpectedTargetName)); + }); + } + + [Test] + public void TestParallelCorpusAnalysis_FileFormatText() + { + using var env = new TestEnvironment(); + ParallelCorpus parallelCorpus = env.GetCorpus(paratextProject: false); + + (QuoteConventionAnalysis? sourceQuotationConvention, QuoteConventionAnalysis? targetQuotationConvention) = + env.Processor.AnalyzeParallelCorpus(parallelCorpus); + + Assert.Multiple(() => + { + Assert.That(sourceQuotationConvention, Is.Null); + Assert.That(targetQuotationConvention, Is.Null); + }); + } [Test] public async Task TestParallelCorpusPreprocessor() { - ParallelCorpusPreprocessingService processor = new(new CorpusService()); - List corpora = - [ - new() + using var env = new TestEnvironment(); + IReadOnlyList corpora = [env.GetCorpus(paratextProject: false)]; + int trainCount = 0; + int inferenceCount = 0; + await env.Processor.PreprocessAsync( + corpora, + row => + { + if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) + trainCount++; + return Task.CompletedTask; + }, + (row, isInTrainingData, _) => + { + if (row.SourceSegment.Length > 0 && !isInTrainingData) + { + inferenceCount++; + } + + return Task.CompletedTask; + }, + false + ); + + Assert.Multiple(() => + { + Assert.That(trainCount, Is.EqualTo(2)); + Assert.That(inferenceCount, Is.EqualTo(3)); + }); + } + + private class TestEnvironment : DisposableBase + { + private static readonly string TestDataPath = Path.Combine( + AppContext.BaseDirectory, + "..", + "..", + "..", + "Services", + "data" + ); + private readonly TempDirectory _tempDir = new TempDirectory(name: "ParallelCorpusProcessingServiceTests"); + + public IParallelCorpusPreprocessingService Processor { get; } = + new ParallelCorpusPreprocessingService(new CorpusService()); + + public ParallelCorpus GetCorpus(bool paratextProject) + { + if (paratextProject) + { + return new ParallelCorpus + { + Id = "corpus1", + SourceCorpora = + [ + new MonolingualCorpus + { + Id = "pt-source1", + Language = "en", + Files = + [ + new CorpusFile + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-source1"), + }, + ], + }, + ], + TargetCorpora = + [ + new MonolingualCorpus + { + Id = "pt-target1", + Language = "en", + Files = + [ + new CorpusFile + { + TextId = "textId1", + Format = FileFormat.Paratext, + Location = ZipParatextProject("pt-target1"), + }, + ], + }, + ], + }; + } + + return new ParallelCorpus { Id = "corpus1", SourceCorpora = [ - new() + new MonolingualCorpus { Id = "source-corpus1", Language = "en", Files = [ - new() + new CorpusFile { TextId = "textId1", Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "source1.txt") - } - ] + Location = Path.Combine(TestDataPath, "source1.txt"), + }, + ], }, - new() + new MonolingualCorpus { Id = "source-corpus2", Language = "en", Files = [ - new() + new CorpusFile { TextId = "textId1", Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "source2.txt") - } - ] - } + Location = Path.Combine(TestDataPath, "source2.txt"), + }, + ], + }, ], TargetCorpora = [ - new() + new MonolingualCorpus { Id = "target-corpus1", Language = "en", Files = [ - new() + new CorpusFile { TextId = "textId1", Format = FileFormat.Text, - Location = Path.Combine(TestDataPath, "target1.txt") - } - ] - } - ] - } - ]; - int trainCount = 0; - int inferenceCount = 0; - await processor.PreprocessAsync( - corpora, - row => - { - if (row.SourceSegment.Length > 0 && row.TargetSegment.Length > 0) - trainCount++; - return Task.CompletedTask; - }, - (row, isInTrainingData, _) => - { - if (row.SourceSegment.Length > 0 && !isInTrainingData) - { - inferenceCount++; - } + Location = Path.Combine(TestDataPath, "target1.txt"), + }, + ], + }, + ], + }; + } - return Task.CompletedTask; - }, - false - ); + protected override void DisposeManagedResources() + { + _tempDir.Dispose(); + } - Assert.Multiple(() => + private string ZipParatextProject(string name) { - Assert.That(trainCount, Is.EqualTo(2)); - Assert.That(inferenceCount, Is.EqualTo(3)); - }); + string fileName = Path.Combine(_tempDir.Path, $"{name}.zip"); + ZipFile.CreateFromDirectory(Path.Combine(TestDataPath, name), fileName); + return fileName; + } } } diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM new file mode 100644 index 00000000..b8665290 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/04LEVTe1.SFM @@ -0,0 +1,8 @@ +\id LEV - Test +\h Leviticus +\mt Leviticus +\c 14 +\p +\v 55 Source one, chapter fourteen, verse fifty-five. +\v 55b Segment b. +\v 56 Source one, chapter fourteen, verse fifty-six. diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM new file mode 100644 index 00000000..4eb8b5fd --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/131CHTe1.SFM @@ -0,0 +1,12 @@ +\id 1CH - Test +\h 1 Chronicles +\mt 1 Chronicles +\c 12 +\p +\v 1 Source one, chapter twelve, verse one. +\v 2 Source one, chapter twelve, verse two. +\v 3-7 Source one, chapter twelve, verses three through seven. +\v 8 Source one, chapter twelve, verse eight. +\c 13 +\p +\v 1 Source one, chapter thirteen, verse one. diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM new file mode 100644 index 00000000..b5ba9651 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/41MATTe1.SFM @@ -0,0 +1,19 @@ +\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew +\c 1 +\p +\v 1 Source one, chapter one, verse one. +\v 2-3 Source one, chapter one, verse two and three. +\v 4 Source one, chapter one, verse four. +\v 5 Source one, chapter one, verse five. +\v 6 Source one, chapter one, verse six. +\v 7-9 Source one, chapter one, verse seven, eight, and nine. +\v 10 Source one, chapter one, verse ten. +\c 2 +\p +\v 1 Source one, chapter two, verse one. +\v 2 Source one, chapter two, verse two. “a quotation” +\v 3 ... +\v 4 ... diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM new file mode 100644 index 00000000..ff8aaf6e --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/42MRKTe1.SFM @@ -0,0 +1,4 @@ +\id MRK - Test +\h Mark +\mt Mark +\ip An introduction to Mark diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml new file mode 100644 index 00000000..c80caedf --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/Settings.xml @@ -0,0 +1,34 @@ + + usfm.sty + 4 + en::: + English + 8.0.100.76 + Test1 + 65001 + T + + NFC + Te1 + a7e0b3ce0200736062f9f810a444dbfbe64aca35 + Charis SIL + 12 + + + + 41MAT + + Tes.SFM + Major::BiblicalTerms.xml + F + F + F + Public + Standard:: + + 3 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 + + + \ No newline at end of file diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml new file mode 100644 index 00000000..b5c2bb97 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/TermRenderings.xml @@ -0,0 +1,16 @@ + + + Abraham + + + + + + + Zedekiah + + + + + + diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs new file mode 100644 index 00000000..9c1cd387 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-source1/custom.vrs @@ -0,0 +1,31 @@ +# custom.vrs + +LEV 14:56 +ROM 14:26 +REV 12:17 +TOB 5:22 +TOB 10:12 +SIR 23:28 +ESG 1:22 +ESG 3:15 +ESG 5:14 +ESG 8:17 +ESG 10:14 +SIR 33:33 +SIR 41:24 +BAR 1:22 +4MA 7:25 +4MA 12:20 + +# deliberately missing verses +-ROM 16:26 +-ROM 16:27 +-3JN 1:15 +-S3Y 1:49 +-ESG 4:6 +-ESG 9:5 +-ESG 9:30 + +LEV 14:55 = LEV 14:55 +LEV 14:55 = LEV 14:56 +LEV 14:56 = LEV 14:57 diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM new file mode 100644 index 00000000..7efc5539 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/41MATTe2.SFM @@ -0,0 +1,18 @@ +\id MAT - Test +\h Matthew +\mt Matthew +\ip An introduction to Matthew +\c 1 +\p +\v 1 Target one, chapter one, verse one. +\v 2 Target one, chapter one, verse two. +\v 3 Target one, chapter one, verse three. +\v 4 +\v 5-6 Target one, chapter one, verse five and six. +\v 7-8 Target one, chapter one, verse seven and eight. +\v 9-10 Target one, chapter one, verse nine and ten. +\c 2 +\p +\v 1 Target one, chapter two, verse one. +\v 2 ... +\v 3 Target one, chapter two, verse three. "a quotation" diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM new file mode 100644 index 00000000..46000963 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/42MRKTe2.SFM @@ -0,0 +1,4 @@ +\id MRK - Test +\h Mark +\mt Mark +\ip An introduction to Mark \ No newline at end of file diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/Settings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/Settings.xml new file mode 100644 index 00000000..37d2772a --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/Settings.xml @@ -0,0 +1,33 @@ + + usfm.sty + 4 + en::: + English + 8.0.100.76 + Test2 + 65001 + T + + NFC + Te2 + a7e0b3ce0200736062f9f810a444dbfbe64aca35 + Charis SIL + 12 + + + + 41MAT + + Ten.SFM + F + F + F + Public + Standard:: + + 3 + 000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000000 + 000000000000000000000000000000000000001100000000000000000000000000000000000000000000000000000000000000000000000000000000000 + + + \ No newline at end of file diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml new file mode 100644 index 00000000..b5c2bb97 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/TermRenderings.xml @@ -0,0 +1,16 @@ + + + Abraham + + + + + + + Zedekiah + + + + + + diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs new file mode 100644 index 00000000..9c1cd387 --- /dev/null +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Services/data/pt-target1/custom.vrs @@ -0,0 +1,31 @@ +# custom.vrs + +LEV 14:56 +ROM 14:26 +REV 12:17 +TOB 5:22 +TOB 10:12 +SIR 23:28 +ESG 1:22 +ESG 3:15 +ESG 5:14 +ESG 8:17 +ESG 10:14 +SIR 33:33 +SIR 41:24 +BAR 1:22 +4MA 7:25 +4MA 12:20 + +# deliberately missing verses +-ROM 16:26 +-ROM 16:27 +-3JN 1:15 +-S3Y 1:49 +-ESG 4:6 +-ESG 9:5 +-ESG 9:30 + +LEV 14:55 = LEV 14:55 +LEV 14:55 = LEV 14:56 +LEV 14:56 = LEV 14:57 diff --git a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs index 95ceaa76..4fb4a8ae 100644 --- a/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs +++ b/src/ServiceToolkit/test/SIL.ServiceToolkit.Tests/Usings.cs @@ -1,3 +1,4 @@ +global using System.IO.Compression; global using System.Text; global using Grpc.Core; global using Microsoft.Extensions.Logging; @@ -6,5 +7,8 @@ global using NSubstitute.ExceptionExtensions; global using NUnit.Framework; global using SIL.DataAccess; +global using SIL.Machine.PunctuationAnalysis; +global using SIL.Machine.Utils; +global using SIL.ObjectModel; global using SIL.ServiceToolkit.Configuration; global using SIL.ServiceToolkit.Models;