Validate usfm versification

src/SIL.Machine/Corpora/FileParatextProjectFileHandler.cs

@@ -0,0 +1,39 @@
+using System.IO;
+using System.Linq;
+
+namespace SIL.Machine.Corpora
+{
+    public class FileParatextProjectFileHandler : IParatextProjectFileHandler
+    {
+        private readonly string _projectDir;
+
+        public FileParatextProjectFileHandler(string projectDir)
+        {
+            _projectDir = projectDir;
+        }
+
+        public bool Exists(string fileName)
+        {
+            return File.Exists(Path.Combine(_projectDir, fileName));
+        }
+
+        public Stream Open(string fileName)
+        {
+            return File.OpenRead(Path.Combine(_projectDir, fileName));
+        }
+
+        public UsfmStylesheet CreateStylesheet(string fileName)
+        {
+            string customStylesheetFileName = Path.Combine(_projectDir, "custom.sty");
+            return new UsfmStylesheet(
+                fileName,
+                File.Exists(customStylesheetFileName) ? customStylesheetFileName : null
+            );
+        }
+
+        public string Find(string extension)
+        {
+            return Directory.EnumerateFiles(_projectDir, "*" + extension).FirstOrDefault();
+        }
+    }
+}

src/SIL.Machine/Corpora/FileParatextProjectSettingsParser.cs

@@ -1,39 +1,13 @@
-using System.IO;
-using System.Linq;
-
-namespace SIL.Machine.Corpora
+namespace SIL.Machine.Corpora
 {
     public class FileParatextProjectSettingsParser : ParatextProjectSettingsParserBase
     {
-        private readonly string _projectDir;
-
         public FileParatextProjectSettingsParser(string projectDir)
-        {
-            _projectDir = projectDir;
-        }
-
-        protected override UsfmStylesheet CreateStylesheet(string fileName)
-        {
-            string customStylesheetFileName = Path.Combine(_projectDir, "custom.sty");
-            return new UsfmStylesheet(
-                fileName,
-                File.Exists(customStylesheetFileName) ? customStylesheetFileName : null
-            );
-        }
-
-        protected override bool Exists(string fileName)
-        {
-            return File.Exists(Path.Combine(_projectDir, fileName));
-        }
-
-        protected override string Find(string extension)
-        {
-            return Directory.EnumerateFiles(_projectDir, "*" + extension).FirstOrDefault();
-        }
+            : base(new FileParatextProjectFileHandler(projectDir)) { }
 
-        protected override Stream Open(string fileName)
+        public static ParatextProjectSettings Parse(string projectDir)
         {
-            return File.OpenRead(Path.Combine(_projectDir, fileName));
+            return new FileParatextProjectSettingsParser(projectDir).Parse();
         }
     }
 }

src/SIL.Machine/Corpora/FileParatextProjectTextUpdater.cs

@@ -1,25 +1,9 @@
-using System.IO;
-
-namespace SIL.Machine.Corpora
+namespace SIL.Machine.Corpora
 {
     public class FileParatextProjectTextUpdater : ParatextProjectTextUpdaterBase
     {
-        private readonly string _projectDir;
-
         public FileParatextProjectTextUpdater(string projectDir)
-            : base(new FileParatextProjectSettingsParser(projectDir))
-        {
-            _projectDir = projectDir;
-        }
-
-        protected override bool Exists(string fileName)
-        {
-            return File.Exists(Path.Combine(_projectDir, fileName));
-        }
-
-        protected override Stream Open(string fileName)
-        {
-            return File.OpenRead(Path.Combine(_projectDir, fileName));
-        }
+            : base(new FileParatextProjectFileHandler(projectDir), FileParatextProjectSettingsParser.Parse(projectDir))
+        { }
     }
 }

src/SIL.Machine/Corpora/FileParatextProjectVersificationErrorDetector.cs

@@ -0,0 +1,9 @@
+namespace SIL.Machine.Corpora
+{
+    public class FileParatextProjectVersificationErrorDetector : ParatextProjectVersificationErrorDetectorBase
+    {
+        public FileParatextProjectVersificationErrorDetector(string projectDir)
+            : base(new FileParatextProjectFileHandler(projectDir), FileParatextProjectSettingsParser.Parse(projectDir))
+        { }
+    }
+}

src/SIL.Machine/Corpora/IParatextProjectFileHandler.cs

@@ -0,0 +1,12 @@
+using System.IO;
+
+namespace SIL.Machine.Corpora
+{
+    public interface IParatextProjectFileHandler
+    {
+        bool Exists(string fileName);
+        Stream Open(string fileName);
+        string Find(string extension);
+        UsfmStylesheet CreateStylesheet(string fileName);
+    }
+}

src/SIL.Machine/Corpora/ParatextBackupTermsCorpus.cs

@@ -15,14 +15,12 @@ public ParatextBackupTermsCorpus(
         {
             using (var archive = ZipFile.OpenRead(fileName))
             {
-                ParatextProjectSettings settings = new ZipParatextProjectSettingsParser(archive).Parse();
-                IEnumerable<(string, IReadOnlyList<string>)> glosses = new ZipParatextProjectTermsParser(
-                    archive,
-                    settings
-                )
+                IEnumerable<(string, IReadOnlyList<string>)> glosses = new ZipParatextProjectTermsParser(archive)
                     .Parse(termCategories, useTermGlosses, chapters)
                     .OrderBy(g => g.TermId);
 
+                ParatextProjectSettings settings = ZipParatextProjectSettingsParser.Parse(archive);
+
                 string textId =
                     $"{settings.BiblicalTermsListType}:{settings.BiblicalTermsProjectName}:{settings.BiblicalTermsFileName}";
 

src/SIL.Machine/Corpora/ParatextProjectSettingsParserBase.cs

@@ -8,16 +8,23 @@ namespace SIL.Machine.Corpora
 {
     public abstract class ParatextProjectSettingsParserBase
     {
+        private readonly IParatextProjectFileHandler _paratextProjectFileHandler;
+
+        public ParatextProjectSettingsParserBase(IParatextProjectFileHandler paratextProjectFileHandler)
+        {
+            _paratextProjectFileHandler = paratextProjectFileHandler;
+        }
+
         public ParatextProjectSettings Parse()
         {
             Encoding.RegisterProvider(CodePagesEncodingProvider.Instance);
             string settingsFileName = "Settings.xml";
-            if (!Exists(settingsFileName))
-                settingsFileName = Find(".ssf");
+            if (!_paratextProjectFileHandler.Exists(settingsFileName))
+                settingsFileName = _paratextProjectFileHandler.Find(".ssf");
             if (string.IsNullOrEmpty(settingsFileName))
                 throw new InvalidOperationException("The project does not contain a settings file.");
             XDocument settingsDoc;
-            using (Stream stream = Open(settingsFileName))
+            using (Stream stream = _paratextProjectFileHandler.Open(settingsFileName))
             {
                 settingsDoc = XDocument.Load(stream);
             }
@@ -36,7 +43,7 @@ public ParatextProjectSettings Parse()
 
             var scrVersType = (int?)settingsDoc.Root.Element("Versification") ?? (int)ScrVersType.English;
             var versification = new ScrVers((ScrVersType)scrVersType);
-            if (Exists("custom.vrs"))
+            if (_paratextProjectFileHandler.Exists("custom.vrs"))
             {
                 var guid = (string)settingsDoc.Root.Element("Guid");
                 string versName = ((ScrVersType)scrVersType).ToString() + "-" + guid;
@@ -46,7 +53,7 @@ public ParatextProjectSettings Parse()
                 }
                 else
                 {
-                    using (var reader = new StreamReader(Open("custom.vrs")))
+                    using (var reader = new StreamReader(_paratextProjectFileHandler.Open("custom.vrs")))
                     {
                         versification = Versification.Table.Implementation.Load(
                             reader,
@@ -60,9 +67,9 @@ public ParatextProjectSettings Parse()
             }
 
             var stylesheetFileName = (string)settingsDoc.Root.Element("StyleSheet") ?? "usfm.sty";
-            if (!Exists(stylesheetFileName) && stylesheetFileName != "usfm_sb.sty")
+            if (!_paratextProjectFileHandler.Exists(stylesheetFileName) && stylesheetFileName != "usfm_sb.sty")
                 stylesheetFileName = "usfm.sty";
-            UsfmStylesheet stylesheet = CreateStylesheet(stylesheetFileName);
+            UsfmStylesheet stylesheet = _paratextProjectFileHandler.CreateStylesheet(stylesheetFileName);
 
             string prefix = "";
             string form = "41MAT";
@@ -122,10 +129,5 @@ public ParatextProjectSettings Parse()
                 languageCode
             );
         }
-
-        protected abstract bool Exists(string fileName);
-        protected abstract string Find(string extension);
-        protected abstract Stream Open(string fileName);
-        protected abstract UsfmStylesheet CreateStylesheet(string fileName);
     }
 }

src/SIL.Machine/Corpora/ParatextProjectTermsParserBase.cs

@@ -37,15 +37,15 @@ public abstract class ParatextProjectTermsParserBase
         private static readonly Regex NumericalInformationRegex = new Regex(@"\s+\d+(\.\d+)*$", RegexOptions.Compiled);
 
         private readonly ParatextProjectSettings _settings;
+        private readonly IParatextProjectFileHandler _paratextProjectFileHandler;
 
-        protected ParatextProjectTermsParserBase(ParatextProjectSettings settings)
+        protected ParatextProjectTermsParserBase(
+            IParatextProjectFileHandler paratextProjectFileHandler,
+            ParatextProjectSettings settings
+        )
         {
             _settings = settings;
-        }
-
-        protected ParatextProjectTermsParserBase(ParatextProjectSettingsParserBase settingsParser)
-        {
-            _settings = settingsParser.Parse();
+            _paratextProjectFileHandler = paratextProjectFileHandler;
         }
 
         public IEnumerable<(string TermId, IReadOnlyList<string> Glosses)> Parse(
@@ -59,9 +59,9 @@ protected ParatextProjectTermsParserBase(ParatextProjectSettingsParserBase setti
             IDictionary<string, ImmutableHashSet<VerseRef>> termIdToReferences;
             if (_settings.BiblicalTermsListType == "Project")
             {
-                if (Exists(_settings.BiblicalTermsFileName))
+                if (_paratextProjectFileHandler.Exists(_settings.BiblicalTermsFileName))
                 {
-                    using (Stream keyTermsFile = Open(_settings.BiblicalTermsFileName))
+                    using (Stream keyTermsFile = _paratextProjectFileHandler.Open(_settings.BiblicalTermsFileName))
                     {
                         biblicalTermsDoc = XDocument.Load(keyTermsFile);
                         termIdToCategoryDictionary = GetCategoryPerId(biblicalTermsDoc);
@@ -115,9 +115,9 @@ protected ParatextProjectTermsParserBase(ParatextProjectSettingsParserBase setti
             }
 
             XDocument termRenderingsDoc = null;
-            if (Exists("TermRenderings.xml"))
+            if (_paratextProjectFileHandler.Exists("TermRenderings.xml"))
             {
-                using (Stream keyTermsFile = Open("TermRenderings.xml"))
+                using (Stream keyTermsFile = _paratextProjectFileHandler.Open("TermRenderings.xml"))
                 {
                     termRenderingsDoc = XDocument.Load(keyTermsFile);
                 }
@@ -298,9 +298,5 @@ private static IDictionary<string, ImmutableHashSet<VerseRef>> GetReferences(XDo
                             .ToImmutableHashSet()
                 );
         }
-
-        protected abstract Stream Open(string fileName);
-
-        protected abstract bool Exists(string fileName);
     }
 }

src/SIL.Machine/Corpora/ParatextProjectTextUpdaterBase.cs

@@ -8,15 +8,15 @@ namespace SIL.Machine.Corpora
     public abstract class ParatextProjectTextUpdaterBase
     {
         private readonly ParatextProjectSettings _settings;
+        private readonly IParatextProjectFileHandler _paratextProjectFileHandler;
 
-        protected ParatextProjectTextUpdaterBase(ParatextProjectSettings settings)
+        protected ParatextProjectTextUpdaterBase(
+            IParatextProjectFileHandler paratextProjectFileHandler,
+            ParatextProjectSettings settings
+        )
         {
             _settings = settings;
-        }
-
-        protected ParatextProjectTextUpdaterBase(ParatextProjectSettingsParserBase settingsParser)
-        {
-            _settings = settingsParser.Parse();
+            _paratextProjectFileHandler = paratextProjectFileHandler;
         }
 
         public string UpdateUsfm(
@@ -73,7 +73,8 @@ public string UpdateUsfm(
             }
         }
 
-        protected abstract bool Exists(string fileName);
-        protected abstract Stream Open(string fileName);
+        private bool Exists(string fileName) => _paratextProjectFileHandler.Exists(fileName);
+
+        private Stream Open(string fileName) => _paratextProjectFileHandler.Open(fileName);
     }
 }

src/SIL.Machine/Corpora/ParatextProjectVersificationErrorDetectorBase.cs

@@ -0,0 +1,55 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Text;
+
+namespace SIL.Machine.Corpora
+{
+    public abstract class ParatextProjectVersificationErrorDetectorBase
+    {
+        private readonly ParatextProjectSettings _settings;
+        private readonly IParatextProjectFileHandler _paratextProjectFileHandler;
+
+        protected ParatextProjectVersificationErrorDetectorBase(
+            IParatextProjectFileHandler paratextProjectFileHandler,
+            ParatextProjectSettings settings
+        )
+        {
+            _settings = settings;
+            _paratextProjectFileHandler = paratextProjectFileHandler;
+        }
+
+        public IReadOnlyList<UsfmVersificationError> GetUsfmVersificationErrors(
+            UsfmVersificationErrorDetector handler = null
+        )
+        {
+            handler = handler ?? new UsfmVersificationErrorDetector(_settings.Versification);
+            foreach (string fileName in _settings.GetAllScriptureBookFileNames())
+            {
+                if (!_paratextProjectFileHandler.Exists(fileName))
+                    continue;
+
+                string usfm;
+                using (var reader = new StreamReader(_paratextProjectFileHandler.Open(fileName)))
+                {
+                    usfm = reader.ReadToEnd();
+                }
+
+                try
+                {
+                    UsfmParser.Parse(usfm, handler, _settings.Stylesheet, _settings.Versification);
+                }
+                catch (Exception ex)
+                {
+                    var sb = new StringBuilder();
+                    sb.Append($"An error occurred while parsing the usfm for '{fileName}`");
+                    if (!string.IsNullOrEmpty(_settings.Name))
+                        sb.Append($" in project '{_settings.Name}'");
+                    sb.Append($". Error: '{ex.Message}'");
+                    throw new InvalidOperationException(sb.ToString(), ex);
+                }
+            }
+            return handler.Errors;
+        }
+    }
+}