Port quotation denormalization

src/SIL.Machine/Corpora/FallbackQuotationMarkResolver.cs

@@ -0,0 +1,167 @@
+using System.Collections.Generic;
+using System.Linq;
+using SIL.Machine.PunctuationAnalysis;
+
+namespace SIL.Machine.Corpora
+{
+    public class FallbackQuotationMarkResolver : IQuotationMarkResolver
+    {
+        private readonly IQuotationMarkResolutionSettings _settings;
+        public QuotationMarkMetadata LastQuotationMark { get; set; }
+        public HashSet<QuotationMarkResolutionIssue> Issues { get; }
+
+        public FallbackQuotationMarkResolver(IQuotationMarkResolutionSettings settings)
+        {
+            _settings = settings;
+            LastQuotationMark = null;
+            Issues = new HashSet<QuotationMarkResolutionIssue>();
+        }
+
+        public void Reset()
+        {
+            LastQuotationMark = null;
+            Issues.Clear();
+        }
+
+        public IEnumerable<QuotationMarkMetadata> ResolveQuotationMarks(
+            IReadOnlyList<QuotationMarkStringMatch> quotationMarkMatches
+        )
+        {
+            foreach (QuotationMarkStringMatch quoteMatch in quotationMarkMatches)
+            {
+                foreach (QuotationMarkMetadata quotationMarkMetadata in ResolveQuotationMark(quoteMatch))
+                {
+                    yield return quotationMarkMetadata;
+                }
+            }
+        }
+
+        public IEnumerable<QuotationMarkMetadata> ResolveQuotationMark(QuotationMarkStringMatch quotationMarkMatch)
+        {
+            if (IsOpeningQuotationMark(quotationMarkMatch))
+            {
+                QuotationMarkMetadata quotationMark = ResolveOpeningMark(quotationMarkMatch);
+                if (quotationMark != null)
+                {
+                    yield return quotationMark;
+                }
+                else
+                {
+                    Issues.Add(QuotationMarkResolutionIssue.UnexpectedQuotationMark);
+                }
+            }
+            else if (IsClosingQuotationMark(quotationMarkMatch))
+            {
+                QuotationMarkMetadata quotationMark = ResolveClosingMark(quotationMarkMatch);
+                if (quotationMark != null)
+                {
+                    yield return quotationMark;
+                }
+                else
+                {
+                    Issues.Add(QuotationMarkResolutionIssue.UnexpectedQuotationMark);
+                }
+            }
+            else
+            {
+                // Make a reasonable guess about the direction of the quotation mark
+                if (LastQuotationMark == null || LastQuotationMark.Direction == QuotationMarkDirection.Closing)
+                {
+                    QuotationMarkMetadata quotationMark = ResolveOpeningMark(quotationMarkMatch);
+                    if (quotationMark != null)
+                        yield return quotationMark;
+                }
+                else
+                {
+                    QuotationMarkMetadata quotationMark = ResolveClosingMark(quotationMarkMatch);
+                    if (quotationMark != null)
+                        yield return quotationMark;
+                }
+                Issues.Add(QuotationMarkResolutionIssue.AmbiguousQuotationMark);
+            }
+        }
+
+        public bool IsOpeningQuotationMark(QuotationMarkStringMatch match)
+        {
+            if (_settings.IsValidOpeningQuotationMark(match) && _settings.IsValidClosingQuotationMark(match))
+            {
+                return (
+                        match.IsAtStartOfSegment
+                        || match.HasLeadingWhitespace()
+                        || DoesMostRecentOpeningMarkImmediatelyPrecede(match)
+                        || match.HasQuoteIntroducerInLeadingSubstring()
+                    ) && !(match.HasTrailingWhitespace() || match.HasTrailingPunctuation());
+            }
+            else if (_settings.IsValidOpeningQuotationMark(match))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        public bool DoesMostRecentOpeningMarkImmediatelyPrecede(QuotationMarkStringMatch match)
+        {
+            if (LastQuotationMark == null || LastQuotationMark.Direction != QuotationMarkDirection.Opening)
+            {
+                return false;
+            }
+            return LastQuotationMark.TextSegment.Equals(match.TextSegment)
+                && LastQuotationMark.EndIndex == match.StartIndex;
+        }
+
+        public bool IsClosingQuotationMark(QuotationMarkStringMatch match)
+        {
+            if (_settings.IsValidOpeningQuotationMark(match) && _settings.IsValidClosingQuotationMark(match))
+            {
+                return (match.HasTrailingWhitespace() || match.HasTrailingPunctuation() || match.IsAtEndOfSegment)
+                    && !match.HasLeadingWhitespace();
+            }
+            else if (_settings.IsValidClosingQuotationMark(match))
+            {
+                return true;
+            }
+
+            return false;
+        }
+
+        public QuotationMarkMetadata ResolveOpeningMark(QuotationMarkStringMatch quotationMarkMatch)
+        {
+            HashSet<int> possibleDepths = _settings.GetPossibleDepths(
+                quotationMarkMatch.QuotationMark,
+                QuotationMarkDirection.Opening
+            );
+            if (possibleDepths.Count == 0)
+                return null;
+
+            QuotationMarkMetadata quotationMark = quotationMarkMatch.Resolve(
+                possibleDepths.Min(),
+                QuotationMarkDirection.Opening
+            );
+            LastQuotationMark = quotationMark;
+            return quotationMark;
+        }
+
+        public QuotationMarkMetadata ResolveClosingMark(QuotationMarkStringMatch quotationMarkMatch)
+        {
+            HashSet<int> possibleDepths = _settings.GetPossibleDepths(
+                quotationMarkMatch.QuotationMark,
+                QuotationMarkDirection.Closing
+            );
+            if (possibleDepths.Count == 0)
+                return null;
+
+            QuotationMarkMetadata quote = quotationMarkMatch.Resolve(
+                possibleDepths.Min(),
+                QuotationMarkDirection.Closing
+            );
+            LastQuotationMark = quote;
+            return quote;
+        }
+
+        public HashSet<QuotationMarkResolutionIssue> GetIssues()
+        {
+            return Issues;
+        }
+    }
+}

src/SIL.Machine/Corpora/ParatextProjectQuoteConventionDetector.cs

@@ -0,0 +1,56 @@
+using System;
+using System.IO;
+using System.Text;
+using SIL.Machine.PunctuationAnalysis;
+
+namespace SIL.Machine.Corpora
+{
+    public abstract class ParatextProjectQuoteConventionDetector
+    {
+        private readonly ParatextProjectSettings _settings;
+
+        protected ParatextProjectQuoteConventionDetector(ParatextProjectSettings settings)
+        {
+            _settings = settings;
+        }
+
+        protected ParatextProjectQuoteConventionDetector(ParatextProjectSettingsParserBase settingsParser)
+        {
+            _settings = settingsParser.Parse();
+        }
+
+        public QuoteConventionAnalysis GetQuoteConventionAnalysis(QuoteConventionDetector handler = null)
+        {
+            handler = handler ?? new QuoteConventionDetector();
+            foreach (string fileName in _settings.GetAllScriptureBookFileNames())
+            {
+                if (!Exists(fileName))
+                    continue;
+
+                string usfm;
+                using (var reader = new StreamReader(Open(fileName)))
+                {
+                    usfm = reader.ReadToEnd();
+                }
+
+                try
+                {
+                    UsfmParser.Parse(usfm, handler, _settings.Stylesheet, _settings.Versification);
+                }
+                catch (Exception ex)
+                {
+                    var sb = new StringBuilder();
+                    sb.Append($"An error occurred while parsing the usfm for '{fileName}`");
+                    if (!string.IsNullOrEmpty(_settings.Name))
+                        sb.Append($" in project '{_settings.Name}'");
+                    sb.Append($". Error: '{ex.Message}'");
+                    throw new InvalidOperationException(sb.ToString(), ex);
+                }
+            }
+            return handler.DetectQuotationConvention();
+        }
+
+        protected abstract bool Exists(string fileName);
+        protected abstract Stream Open(string fileName);
+    }
+}

src/SIL.Machine/Corpora/ParatextProjectSettings.cs

@@ -1,4 +1,5 @@
-using System.Globalization;
+using System.Collections.Generic;
+using System.Globalization;
 using System.Text;
 using SIL.Scripture;
 
@@ -103,6 +104,16 @@ public string GetBookFileName(string bookId)
             return FileNamePrefix + bookPart + FileNameSuffix;
         }
 
+        public IEnumerable<string> GetAllScriptureBookFileNames()
+        {
+            BookSet scriptureBooks = Canon.ScriptureBooks;
+            scriptureBooks.SelectAll();
+            foreach (string bookId in scriptureBooks.SelectedBookIds)
+            {
+                yield return GetBookFileName(bookId);
+            }
+        }
+
         private static string GetBookFileNameDigits(string bookId)
         {
             int bookNum = Canon.BookIdToNumber(bookId);

src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs

@@ -21,7 +21,7 @@ protected ParatextProjectTextUpdaterBase(ParatextProjectSettingsParserBase setti
 
         public string UpdateUsfm(
             string bookId,
-            IReadOnlyList<(IReadOnlyList<ScriptureRef>, string)> rows,
+            IReadOnlyList<UpdateUsfmRow> rows,
             string fullName = null,
             UpdateUsfmTextBehavior textBehavior = UpdateUsfmTextBehavior.PreferExisting,
             UpdateUsfmMarkerBehavior paragraphBehavior = UpdateUsfmMarkerBehavior.Preserve,

src/SIL.Machine/Corpora/PlaceMarkersUsfmUpdateBlockHandler.cs

@@ -8,49 +8,60 @@ namespace SIL.Machine.Corpora
 {
     public class PlaceMarkersAlignmentInfo
     {
-        public IReadOnlyList<string> Refs { get; }
+        public const string MetadataKey = "alignment_info";
+
         public IReadOnlyList<string> SourceTokens { get; }
         public IReadOnlyList<string> TranslationTokens { get; }
         public WordAlignmentMatrix Alignment { get; }
+        public UpdateUsfmMarkerBehavior ParagraphBehavior { get; }
+        public UpdateUsfmMarkerBehavior StyleBehavior { get; }
 
         public PlaceMarkersAlignmentInfo(
-            IReadOnlyList<string> refs,
             IReadOnlyList<string> sourceTokens,
             IReadOnlyList<string> translationTokens,
-            WordAlignmentMatrix alignment
+            WordAlignmentMatrix alignment,
+            UpdateUsfmMarkerBehavior paragraphBehavior,
+            UpdateUsfmMarkerBehavior styleBehavior
         )
         {
-            Refs = refs;
             SourceTokens = sourceTokens;
             TranslationTokens = translationTokens;
             Alignment = alignment;
+            ParagraphBehavior = paragraphBehavior;
+            StyleBehavior = styleBehavior;
         }
     }
 
     public class PlaceMarkersUsfmUpdateBlockHandler : IUsfmUpdateBlockHandler
     {
-        private readonly IDictionary<string, PlaceMarkersAlignmentInfo> _alignmentInfo;
-
-        public PlaceMarkersUsfmUpdateBlockHandler(IEnumerable<PlaceMarkersAlignmentInfo> alignmentInfo)
-        {
-            _alignmentInfo = alignmentInfo.ToDictionary(info => info.Refs.First(), info => info);
-        }
-
         public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block)
         {
             string reference = block.Refs.FirstOrDefault().ToString();
             var elements = block.Elements.ToList();
 
             // Nothing to do if there are no markers to place or no alignment to use
+            if (!block.Metadata.TryGetValue(PlaceMarkersAlignmentInfo.MetadataKey, out object alignmentObject))
+            {
+                return block;
+            }
+            if (!(alignmentObject is PlaceMarkersAlignmentInfo alignmentInfo))
+            {
+                return block;
+            }
             if (
                 elements.Count == 0
-                || !_alignmentInfo.TryGetValue(reference, out PlaceMarkersAlignmentInfo alignmentInfo)
                 || alignmentInfo.Alignment.RowCount == 0
                 || alignmentInfo.Alignment.ColumnCount == 0
                 || !elements.Any(e =>
-                    e.Type.IsOneOf(UsfmUpdateBlockElementType.Paragraph, UsfmUpdateBlockElementType.Style)
-                    && !e.MarkedForRemoval
-                    && e.Tokens.Count == 1
+                    (
+                        e.Type == UsfmUpdateBlockElementType.Paragraph
+                        && alignmentInfo.ParagraphBehavior == UpdateUsfmMarkerBehavior.Preserve
+                        && e.Tokens.Count == 1
+                    )
+                    || (
+                        e.Type == UsfmUpdateBlockElementType.Style
+                        && alignmentInfo.StyleBehavior == UpdateUsfmMarkerBehavior.Preserve
+                    )
                 )
             )
             {
@@ -112,7 +123,13 @@ public UsfmUpdateBlock ProcessBlock(UsfmUpdateBlock block)
             {
                 if (element.Type == UsfmUpdateBlockElementType.Text)
                 {
-                    if (element.MarkedForRemoval)
+                    if (
+                        element.MarkedForRemoval
+                        || (
+                            element.Type == UsfmUpdateBlockElementType.Paragraph
+                            && alignmentInfo.ParagraphBehavior == UpdateUsfmMarkerBehavior.Strip
+                        )
+                    )
                     {
                         string text = element.Tokens[0].ToUsfm();
                         sourceSentence += text;

src/SIL.Machine/Corpora/QuotationMarkDenormalizationFirstPass.cs

@@ -0,0 +1,14 @@
+using SIL.Machine.PunctuationAnalysis;
+
+namespace SIL.Machine.Corpora
+{
+    // This is a convenience class so that users don't have to know to normalize the source quote convention
+    public class QuotationMarkDenormalizationFirstPass : QuotationMarkUpdateFirstPass
+    {
+        public QuotationMarkDenormalizationFirstPass(
+            QuoteConvention sourceQuoteConvention,
+            QuoteConvention targetQuoteConvention
+        )
+            : base(sourceQuoteConvention.Normalize(), targetQuoteConvention) { }
+    }
+}

src/SIL.Machine/Corpora/QuotationMarkDenormalizationUsfmUpdateBlockHandler.cs

@@ -0,0 +1,19 @@
+using SIL.Machine.PunctuationAnalysis;
+
+namespace SIL.Machine.Corpora
+{
+    public class QuotationMarkDenormalizationUsfmUpdateBlockHandler : QuoteConventionChangingUsfmUpdateBlockHandler
+    {
+        // This is a convenience class so that users don't have to know to normalize the source quote convention
+        public QuotationMarkDenormalizationUsfmUpdateBlockHandler(
+            QuoteConvention sourceQuoteConvention,
+            QuoteConvention targetQuoteConvention,
+            QuotationMarkUpdateSettings settings = null
+        )
+            : base(
+                sourceQuoteConvention.Normalize(),
+                targetQuoteConvention,
+                settings ?? new QuotationMarkUpdateSettings()
+            ) { }
+    }
+}