Implement entry point for building content description models

Issue #252 Implement entry point (as started by Anthony) for building content description models.
Change name of ContentDescription.cs file to UseModel.cs.
Need to adjust methods to deal with new manifests data structure.

src/AnalysisPrograms/AnalysisPrograms.csproj

@@ -302,7 +302,7 @@
     <Compile Include="..\AssemblyMetadata.cs">
       <Link>AssemblyMetadata.cs</Link>
     </Compile>
-    <Compile Include="ContentDescription\ContentDescription.cs" />
+    <Compile Include="ContentDescription\UseModel.cs" />
     <Compile Include="AcousticWorkbench.Orchestration\EventMetadataResolver.cs" />
     <Compile Include="AcousticWorkbench.Orchestration\RemoteSegment.cs" />
     <Compile Include="AED.cs" />

src/AnalysisPrograms/ContentDescription/BuildModel.cs

@@ -1,52 +1,53 @@
-using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
+// <copyright file="BuildModel.cs" company="QutEcoacoustics">
+// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
+// </copyright>
 
 namespace AnalysisPrograms.ContentDescription
 {
+    using System;
+    using System.Collections.Generic;
     using System.ComponentModel.DataAnnotations;
     using System.IO;
-    using AnalysisBase;
-    using AnalysisPrograms.Production;
+    using System.Threading.Tasks;
+    using Acoustics.Shared;
     using AnalysisPrograms.Production.Arguments;
-    using AnalysisPrograms.Production.Validation;
     using AudioAnalysisTools.ContentDescriptionTools;
     using McMaster.Extensions.CommandLineUtils;
 
-    public partial class BuildModel
+    public class BuildModel
     {
         public const string CommandName = "BuildContentDescriptionModel";
 
         [Command(
             CommandName,
-            Description = "TODO")]
-        public class Arguments: SubCommandBase
+            Description = "Reads a file of template manifests and creates an output file of functional templates that are used for audio content description.")]
+        public class Arguments : SubCommandBase
         {
             [Argument(
                 0,
-                Description = "TODO")]
+                Description = "Path to an input yml file containing an array of template manifests.")]
             [Required]
             [FileExists]
             [LegalFilePath]
             public FileInfo TemplateManifest { get; set; }
 
             [Argument(
                 1,
-                Description = "TODO")]
+                Description = "Path to an output json file containing an array of functional templates.")]
             [Required]
             [FileExists]
             [LegalFilePath]
             public FileInfo TemplateDefinitions { get; set; }
 
+            /*
             [Argument(
                 2,
                 Description = "A directory to write output to")]
             [Required]
             [DirectoryExistsOrCreate(createIfNotExists: true)]
             [LegalFilePath]
             public virtual DirectoryInfo Output { get; set; }
+            */
 
             public override Task<int> Execute(CommandLineApplication app)
             {
@@ -58,11 +59,82 @@ public override Task<int> Execute(CommandLineApplication app)
 
         public static void Execute(Arguments arguments)
         {
-            // TODO: inline CreateNewFileOfTemplateDefinitions to this method.
-            TemplateManifest.CreateNewFileOfTemplateDefinitions(arguments.TemplateManifest,
-                arguments.TemplateDefinitions);
+            FileInfo manifestFile = arguments.TemplateManifest;
+            FileInfo functionalTemplatesFile = arguments.TemplateDefinitions;
 
-            LoggedConsole.WriteSuccessLine("Completed");
+            // Read in all template manifests
+            var manifests = Yaml.Deserialize<TemplateManifest[]>(manifestFile);
+
+            // Read current template definitions and convert to dictionary
+            var arrayOfFunctionalTemplates = Json.Deserialize<FunctionalTemplate[]>(functionalTemplatesFile);
+            var dictionaryOfCurrentTemplates = DataProcessing.ConvertArrayOfFunctionalTemplatesToDictionary(arrayOfFunctionalTemplates);
+
+            // init a new template list for output.
+            var newTemplateList = new List<FunctionalTemplate>();
+
+            // cycle through all the manifests
+            foreach (var manifest in manifests)
+            {
+                var name = manifest.Name;
+                if (!dictionaryOfCurrentTemplates.ContainsKey(name))
+                {
+                    // the current manifest is not an existing template - therefore make it.
+                    var newTemplate = new FunctionalTemplate(manifest)
+                    {
+                        Template = TemplateManifest.CreateTemplateDefinition(manifest),
+                        MostRecentEdit = DateTime.Now,
+                    };
+                    newTemplateList.Add(newTemplate);
+                    continue;
+                }
+
+                if (manifest.EditStatus == EditStatus.Edit)
+                {
+                    // This option edits an existing functional template in the json file. The template definition is (re)calculated.
+                    // Effectively the same as creating a new template.
+                    var newTemplate = new FunctionalTemplate(manifest)
+                    {
+                        Template = TemplateManifest.CreateTemplateDefinition(manifest),
+                        MostRecentEdit = DateTime.Now,
+                    };
+                    newTemplateList.Add(newTemplate);
+                    continue;
+                }
+
+                if (manifest.EditStatus == EditStatus.Copy)
+                {
+                    // This option keeps an existing functional template unchanged.
+                    var existingTemplate = dictionaryOfCurrentTemplates[name];
+                    newTemplateList.Add(existingTemplate);
+                    continue;
+                }
+
+                if (manifest.EditStatus == EditStatus.Ignore)
+                {
+                    // Do not output this template to the list of functional templates.
+                    continue;
+                }
+            }
+
+            var functionalTemplatesFileName = functionalTemplatesFile.Name;
+            var functionalTemplatesFilePath = Path.Combine(manifestFile.DirectoryName ?? throw new InvalidOperationException(), functionalTemplatesFileName);
+
+            // Save the previous templates file
+            string backupPath = Path.Combine(manifestFile.DirectoryName, functionalTemplatesFileName + ".Backup.json");
+            if (File.Exists(backupPath))
+            {
+                File.Delete(backupPath);
+            }
+
+            //Now copy the file first
+            File.Copy(functionalTemplatesFilePath, backupPath, true);
+
+            //Now Rename the File
+            //File.Move(NewFilePath, Path.Combine(NewFileLocation, "File.txt"));
+
+            // No need to move the backup because serializing over-writes the current templates file.
+            var opTemplatesFile = new FileInfo(functionalTemplatesFilePath);
+            Json.Serialise(opTemplatesFile, newTemplateList.ToArray());
         }
     }
 }

src/AnalysisPrograms/ContentDescription/ContentDescription.cs → src/AnalysisPrograms/ContentDescription/UseModel.cs

@@ -1,4 +1,4 @@
-// <copyright file="ContentDescription.cs" company="QutEcoacoustics">
+// <copyright file="UseModel.cs" company="QutEcoacoustics">
 // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group).
 // </copyright>
 
@@ -28,7 +28,7 @@ namespace AnalysisPrograms.ContentDescription
     /// To call this class, the first argument on the commandline must be 'audio2csv'.
     /// Given a one-minute recording segment, the ContentDescription.Analyze() method calls AudioAnalysisTools.Indices.IndexCalculateSixOnly.Analysis().
     /// This calculates six spectral indices, ACI, ENT, EVN, BGN, PMN, OSC. This set of 6x256 acoustic features is used for content description.
-    /// The content description methods are called from ContentDescription.SummariseResults() method.
+    /// The content description methods are called from UseModel.SummariseResults() method.
     /// </summary>
     public class UseModel : AbstractStrongAnalyser
     {
@@ -52,9 +52,9 @@ public class UseModel : AbstractStrongAnalyser
             AnalysisTargetSampleRate = ContentSignatures.SampleRate,
         };
 
-        private FunctionalTemplate[] functionalTemplates = null;
+        private FunctionalTemplate[] functionalTemplates;
 
-        private Dictionary<string, Dictionary<string, double[]>> templatesAsDictionary = null;
+        private Dictionary<string, Dictionary<string, double[]>> templatesAsDictionary;
 
         public override void BeforeAnalyze(AnalysisSettings analysisSettings)
         {
@@ -94,11 +94,7 @@ public class CdConfig : AnalyzerConfig
         public override AnalysisResult2 Analyze<T>(AnalysisSettings analysisSettings, SegmentSettings<T> segmentSettings)
         {
             // set the start time for the current recording segment. Default is zero.
-            var elapsedTimeAtStartOfRecording = TimeSpan.Zero;
-            if (segmentSettings.SegmentStartOffset != null)
-            {
-                elapsedTimeAtStartOfRecording = (TimeSpan)segmentSettings.SegmentStartOffset;
-            }
+            var elapsedTimeAtStartOfRecording = segmentSettings.SegmentStartOffset;
 
             var startMinuteId = (int)Math.Round(elapsedTimeAtStartOfRecording.TotalMinutes);
 
@@ -114,7 +110,7 @@ public override AnalysisResult2 Analyze<T>(AnalysisSettings analysisSettings, Se
             segmentResults.SpectralIndexValues.FileName = segmentSettings.Segment.SourceMetadata.Identifier;
 
             // DO THE CONTENT DESCRIPTION FOR ONE MINUTE HERE
-            // First get acoustic indices for one minute, convert to Dictionary and normalise the values.
+            // First get acoustic indices for one minute, convert to Dictionary and normalize the values.
             var indicesDictionary = IndexCalculateSixOnly.ConvertIndicesToDictionary(segmentResults.SpectralIndexValues);
             foreach (string key in ContentSignatures.IndexNames)
             {
@@ -196,8 +192,9 @@ public override void SummariseResults(
             //sampleRate = analysisSettings.Configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? sampleRate;
             var cdConfiguration = (CdConfig)analysisSettings.Configuration;
             var ldSpectrogramConfig = cdConfiguration.LdSpectrogramConfig;
-            var cdConfigFile = analysisSettings.ConfigFile;
-            var configDirectory = cdConfigFile.DirectoryName ?? throw new ArgumentNullException(nameof(cdConfigFile), "Null value");
+
+            //var cdConfigFile = analysisSettings.ConfigFile;
+            //var configDirectory = cdConfigFile.DirectoryName ?? throw new ArgumentNullException(nameof(cdConfigFile), "Null value");
             var sourceAudio = inputFileSegment.Source;
             string basename = Path.GetFileNameWithoutExtension(sourceAudio.Name);
             var resultsDirectory = AnalysisCoordinator.GetNamedDirectory(analysisSettings.AnalysisOutputDirectory, this);
@@ -275,6 +272,7 @@ public override void SummariseResults(
             image.Save(path3);
         }
 
+        /*
         /// <summary>
         /// NOTE: THIS METHOD SHOULD EVENTUALLY BE DELETED. NO LONGER CALLED.
         /// Calculate the content description for each minute.
@@ -327,6 +325,7 @@ private static Dictionary<string, double[]> GetContentDescription(
             var dictionaryOfScores = DataProcessing.ConvertResultsToDictionaryOfArrays(results, length, startMinuteId);
             return dictionaryOfScores;
         }
+        */
 
         /// <summary>
         /// Produce plots for graphical display.

src/AnalysisPrograms/Sandpit.cs

@@ -16,7 +16,8 @@ namespace AnalysisPrograms
     using Acoustics.Shared;
     using Acoustics.Shared.Csv;
     using Acoustics.Tools.Wav;
-    using AnalyseLongRecordings;
+    using AnalysisPrograms.AnalyseLongRecordings;
+    using AnalysisPrograms.ContentDescription;
     using AudioAnalysisTools;
     using AudioAnalysisTools.ContentDescriptionTools;
     using AudioAnalysisTools.DSP;
@@ -25,7 +26,7 @@ namespace AnalysisPrograms
     using AudioAnalysisTools.StandardSpectrograms;
     using AudioAnalysisTools.WavTools;
     using McMaster.Extensions.CommandLineUtils;
-    using Production.Arguments;
+    using AnalysisPrograms.Production.Arguments;
     using TowseyLibrary;
 
     /// <summary>
@@ -121,9 +122,17 @@ public override Task<int> Execute(CommandLineApplication app)
 
         public static void ContentDescriptionCreateTemplates()
         {
-            var templateManifests = new FileInfo(@"C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml");
-            var templateDefinitions = new FileInfo(@"C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json");
-            TemplateManifest.CreateNewFileOfTemplateDefinitions(templateManifests, templateDefinitions);
+            /* COMMAND LINE FOR BUILDING CONTENT DEFINITIONS MODEL OF TEMPLATES
+             * .\AnalysisPrograms.exe BuildContentDescriptionModel "C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml" "C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json"
+             */
+            var templateManifestsPath = @"C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml";
+            var templateDefinitionsPath = @"C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json";
+            var arguments = new BuildModel.Arguments
+            {
+                TemplateManifest = templateManifestsPath.ToFileInfo(),
+                TemplateDefinitions = templateDefinitionsPath.ToFileInfo(),
+            };
+            BuildModel.Execute(arguments);
             Console.WriteLine("# Finished creation of new manifest");
         }
 

src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs

@@ -153,11 +153,6 @@ public static DescriptionResult AnalyzeOneMinute(
             // now subject the indices to various content searches
             foreach (var template in templates)
             {
-                if (template.UseStatus == false)
-                {
-                    continue;
-                }
-
                 var algorithmType = template.Manifest.FeatureExtractionAlgorithm;
                 var templateIndices = templatesAsDictionary[template.Manifest.Name];
                 double score;

src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs

@@ -19,12 +19,12 @@ public static class DataProcessing
         /// </summary>
         /// <param name="array">An array of templates.</param>
         /// <returns>A dictionary of templates.</returns>
-        public static Dictionary<string, TemplateManifest> ConvertTemplateArrayToDictionary(TemplateManifest[] array)
+        public static Dictionary<string, FunctionalTemplate> ConvertArrayOfFunctionalTemplatesToDictionary(FunctionalTemplate[] array)
         {
-            var dictionary = new Dictionary<string, TemplateManifest>();
+            var dictionary = new Dictionary<string, FunctionalTemplate>();
             foreach (var template in array)
             {
-                dictionary.Add(template.Name, template);
+                dictionary.Add(template.Manifest.Name, template);
             }
 
             return dictionary;