Denormalize quotation marks

src/Machine/src/Serval.Machine.Shared/Serval.Machine.Shared.csproj

@@ -36,9 +36,9 @@
 		<PackageReference Include="Hangfire.Mongo" Version="1.11.6" />
 		<PackageReference Include="Microsoft.AspNetCore.Mvc.NewtonsoftJson" Version="8.0.8" />
 		<PackageReference Include="Microsoft.Extensions.Http.Polly" Version="8.0.8" />
-		<PackageReference Include="SIL.Machine" Version="3.7.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
-		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.7.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
-		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.7.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.7.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine.Morphology.HermitCrab" Version="3.7.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Morphology.HermitCrab\SIL.Machine.Morphology.HermitCrab.csproj')" />
+		<PackageReference Include="SIL.Machine.Translation.Thot" Version="3.7.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine.Translation.Thot\SIL.Machine.Translation.Thot.csproj')" />
 		<PackageReference Include="SIL.WritingSystems" Version="14.1.1" />
 		<PackageReference Include="System.Linq.Async" Version="6.0.1" />
 		<PackageReference Include="YamlDotNet" Version="11.2.1" />

src/Serval/src/Serval.Client/Client.g.cs

@@ -2471,6 +2471,10 @@ public partial interface ITranslationEnginesClient
         /// <br/>* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language.**.
         /// <br/>* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**.
         /// <br/>            
+        /// <br/>Quote normalization behavior is controlled by the `quoteNormalizationBehavior` parameter options:
+        /// <br/>* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data.
+        /// <br/>* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized.
+        /// <br/>            
         /// <br/>Only pretranslations for the most recent successful build of the engine are returned.
         /// <br/>The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation).
         /// </remarks>
@@ -2482,9 +2486,10 @@ public partial interface ITranslationEnginesClient
         /// <param name="paragraphMarkerBehavior">The behavior of paragraph markers.</param>
         /// <param name="embedBehavior">The behavior of embed markers.</param>
         /// <param name="styleMarkerBehavior">The behavior of style markers.</param>
+        /// <param name="quoteNormalizationBehavior">The normalization behavior of quotes.</param>
         /// <returns>The book in USFM format</returns>
         /// <exception cref="ServalApiException">A server side error occurred.</exception>
-        System.Threading.Tasks.Task<string> GetPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));
+        System.Threading.Tasks.Task<string> GetPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken));
 
         /// <param name="cancellationToken">A cancellation token that can be used by other objects or threads to receive notice of cancellation.</param>
         /// <summary>
@@ -5168,6 +5173,10 @@ public string BaseUrl
         /// <br/>* `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language.**.
         /// <br/>* `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**.
         /// <br/>            
+        /// <br/>Quote normalization behavior is controlled by the `quoteNormalizationBehavior` parameter options:
+        /// <br/>* `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data.
+        /// <br/>* `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized.
+        /// <br/>            
         /// <br/>Only pretranslations for the most recent successful build of the engine are returned.
         /// <br/>The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation).
         /// </remarks>
@@ -5179,9 +5188,10 @@ public string BaseUrl
         /// <param name="paragraphMarkerBehavior">The behavior of paragraph markers.</param>
         /// <param name="embedBehavior">The behavior of embed markers.</param>
         /// <param name="styleMarkerBehavior">The behavior of style markers.</param>
+        /// <param name="quoteNormalizationBehavior">The normalization behavior of quotes.</param>
         /// <returns>The book in USFM format</returns>
         /// <exception cref="ServalApiException">A server side error occurred.</exception>
-        public virtual async System.Threading.Tasks.Task<string> GetPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken))
+        public virtual async System.Threading.Tasks.Task<string> GetPretranslatedUsfmAsync(string id, string corpusId, string textId, PretranslationUsfmTextOrigin? textOrigin = null, PretranslationUsfmTemplate? template = null, PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior = null, PretranslationUsfmMarkerBehavior? embedBehavior = null, PretranslationUsfmMarkerBehavior? styleMarkerBehavior = null, PretranslationNormalizationBehavior? quoteNormalizationBehavior = null, System.Threading.CancellationToken cancellationToken = default(System.Threading.CancellationToken))
         {
             if (id == null)
                 throw new System.ArgumentNullException("id");
@@ -5232,6 +5242,10 @@ public string BaseUrl
                     {
                         urlBuilder_.Append(System.Uri.EscapeDataString("style-marker-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(styleMarkerBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&');
                     }
+                    if (quoteNormalizationBehavior != null)
+                    {
+                        urlBuilder_.Append(System.Uri.EscapeDataString("quotation-mark-behavior")).Append('=').Append(System.Uri.EscapeDataString(ConvertToString(quoteNormalizationBehavior, System.Globalization.CultureInfo.InvariantCulture))).Append('&');
+                    }
                     urlBuilder_.Length--;
 
                     PrepareRequest(client_, request_, urlBuilder_);
@@ -10341,6 +10355,18 @@ public enum PretranslationUsfmMarkerBehavior
 
     }
 
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
+    public enum PretranslationNormalizationBehavior
+    {
+
+        [System.Runtime.Serialization.EnumMember(Value = @"Normalized")]
+        Normalized = 0,
+
+        [System.Runtime.Serialization.EnumMember(Value = @"Denormalized")]
+        Denormalized = 1,
+
+    }
+
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "14.1.0.0 (NJsonSchema v11.0.2.0 (Newtonsoft.Json v13.0.0.0))")]
     public partial class TranslationBuild
     {

src/Serval/src/Serval.Shared/Serval.Shared.csproj

@@ -19,7 +19,7 @@
 		<PackageReference Include="Grpc.Core.Api" Version="2.65.0" />
 		<PackageReference Include="Grpc.HealthCheck" Version="2.65.0" />
 		<PackageReference Include="Grpc.Net.ClientFactory" Version="2.65.0" />
-		<PackageReference Include="SIL.Machine" Version="3.7.0" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
+		<PackageReference Include="SIL.Machine" Version="3.7.1" Condition="!Exists('..\..\..\..\..\machine\src\SIL.Machine\SIL.Machine.csproj')" />
 		<PackageReference Include="Microsoft.FeatureManagement.AspNetCore" Version="3.5.0" />
 	</ItemGroup>
 

src/Serval/src/Serval.Translation/Contracts/PretranslationQuotationMarkBehavior.cs

@@ -0,0 +1,7 @@
+namespace Serval.Translation.Contracts;
+
+public enum PretranslationNormalizationBehavior
+{
+    Normalized,
+    Denormalized
+}

src/Serval/src/Serval.Translation/Controllers/TranslationEnginesController.cs

@@ -901,6 +901,10 @@ CancellationToken cancellationToken
     /// * `TryToPlace`: The USFM markers (or the entire embed) are placed in approximately the right location within the verse. **This option is only available for paragraph markers. Quality of placement may differ from language to language.**.
     /// * `Strip`: The USFM markers (or the entire embed) are removed. **This is the default for style markers**.
     ///
+    /// Quote normalization behavior is controlled by the `quoteNormalizationBehavior` parameter options:
+    /// * `Normalized`: The quotes in the pretranslated USFM are normalized quotes (typically straight quotes: ', ") in the style of the source data.
+    /// * `Denormalized`: The quotes in the pretranslated USFM are denormalized into the style of the target data. Quote denormalization may not be successful in all contexts. A remark will be added to the USFM listing the chapters that were successfully denormalized.
+    ///
     /// Only pretranslations for the most recent successful build of the engine are returned.
     /// The USFM parsing and marker types used are defined here: [this wiki](https://github.com/sillsdev/serval/wiki/USFM-Parsing-and-Translation).
     /// </remarks>
@@ -912,6 +916,7 @@ CancellationToken cancellationToken
     /// <param name="paragraphMarkerBehavior">The behavior of paragraph markers.</param>
     /// <param name="embedBehavior">The behavior of embed markers.</param>
     /// <param name="styleMarkerBehavior">The behavior of style markers.</param>
+    /// <param name="quoteNormalizationBehavior">The normalization behavior of quotes.</param>
     /// <param name="cancellationToken"></param>
     /// <response code="200">The book in USFM format</response>
     /// <response code="204">The specified book does not exist in the source or target corpus.</response>
@@ -941,6 +946,7 @@ public async Task<IActionResult> GetPretranslatedUsfmAsync(
         [FromQuery(Name = "paragraph-marker-behavior")] PretranslationUsfmMarkerBehavior? paragraphMarkerBehavior,
         [FromQuery(Name = "embed-behavior")] PretranslationUsfmMarkerBehavior? embedBehavior,
         [FromQuery(Name = "style-marker-behavior")] PretranslationUsfmMarkerBehavior? styleMarkerBehavior,
+        [FromQuery(Name = "quotation-mark-behavior")] PretranslationNormalizationBehavior? quoteNormalizationBehavior,
         CancellationToken cancellationToken
     )
     {
@@ -961,6 +967,7 @@ CancellationToken cancellationToken
             paragraphMarkerBehavior ?? PretranslationUsfmMarkerBehavior.Preserve,
             embedBehavior ?? PretranslationUsfmMarkerBehavior.Preserve,
             styleMarkerBehavior ?? PretranslationUsfmMarkerBehavior.Strip,
+            quoteNormalizationBehavior ?? PretranslationNormalizationBehavior.Normalized,
             cancellationToken
         );
         if (usfm == "")

src/Serval/src/Serval.Translation/Models/Build.cs

@@ -20,5 +20,5 @@ public record Build : IInitializableEntity
     public bool? IsInitialized { get; set; }
     public DateTime? DateCreated { get; set; }
     public IReadOnlyList<BuildPhase>? Phases { get; init; }
-    public IReadOnlyCollection<ParallelCorpusAnalysis>? Analysis { get; init; }
+    public IReadOnlyList<ParallelCorpusAnalysis>? Analysis { get; init; }
 }

src/Serval/src/Serval.Translation/Services/IPretranslationService.cs

@@ -20,6 +20,7 @@ Task<string> GetUsfmAsync(
         PretranslationUsfmMarkerBehavior paragraphMarkerBehavior,
         PretranslationUsfmMarkerBehavior embedBehavior,
         PretranslationUsfmMarkerBehavior styleMarkerBehavior,
+        PretranslationNormalizationBehavior quoteNormalizationBehavior,
         CancellationToken cancellationToken = default
     );
 }