diff --git a/src/AnalysisPrograms/AnalysisPrograms.csproj b/src/AnalysisPrograms/AnalysisPrograms.csproj index 5c7be8a31..ed141906c 100644 --- a/src/AnalysisPrograms/AnalysisPrograms.csproj +++ b/src/AnalysisPrograms/AnalysisPrograms.csproj @@ -302,7 +302,7 @@ AssemblyMetadata.cs - + diff --git a/src/AnalysisPrograms/ContentDescription/BuildModel.cs b/src/AnalysisPrograms/ContentDescription/BuildModel.cs index 1a745daad..e22b75f92 100644 --- a/src/AnalysisPrograms/ContentDescription/BuildModel.cs +++ b/src/AnalysisPrograms/ContentDescription/BuildModel.cs @@ -1,32 +1,31 @@ -using System; -using System.Collections.Generic; -using System.Linq; -using System.Text; -using System.Threading.Tasks; +// +// All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). +// namespace AnalysisPrograms.ContentDescription { + using System; + using System.Collections.Generic; using System.ComponentModel.DataAnnotations; using System.IO; - using AnalysisBase; - using AnalysisPrograms.Production; + using System.Threading.Tasks; + using Acoustics.Shared; using AnalysisPrograms.Production.Arguments; - using AnalysisPrograms.Production.Validation; using AudioAnalysisTools.ContentDescriptionTools; using McMaster.Extensions.CommandLineUtils; - public partial class BuildModel + public class BuildModel { public const string CommandName = "BuildContentDescriptionModel"; [Command( CommandName, - Description = "TODO")] - public class Arguments: SubCommandBase + Description = "Reads a file of template manifests and creates an output file of functional templates that are used for audio content description.")] + public class Arguments : SubCommandBase { [Argument( 0, - Description = "TODO")] + Description = "Path to an input yml file containing an array of template manifests.")] [Required] [FileExists] [LegalFilePath] @@ -34,12 +33,13 @@ public class Arguments: SubCommandBase [Argument( 1, - Description = "TODO")] + Description = "Path to an output json file containing an array of functional templates.")] [Required] [FileExists] [LegalFilePath] public FileInfo TemplateDefinitions { get; set; } + /* [Argument( 2, Description = "A directory to write output to")] @@ -47,6 +47,7 @@ public class Arguments: SubCommandBase [DirectoryExistsOrCreate(createIfNotExists: true)] [LegalFilePath] public virtual DirectoryInfo Output { get; set; } + */ public override Task Execute(CommandLineApplication app) { @@ -58,11 +59,82 @@ public override Task Execute(CommandLineApplication app) public static void Execute(Arguments arguments) { - // TODO: inline CreateNewFileOfTemplateDefinitions to this method. - TemplateManifest.CreateNewFileOfTemplateDefinitions(arguments.TemplateManifest, - arguments.TemplateDefinitions); + FileInfo manifestFile = arguments.TemplateManifest; + FileInfo functionalTemplatesFile = arguments.TemplateDefinitions; - LoggedConsole.WriteSuccessLine("Completed"); + // Read in all template manifests + var manifests = Yaml.Deserialize(manifestFile); + + // Read current template definitions and convert to dictionary + var arrayOfFunctionalTemplates = Json.Deserialize(functionalTemplatesFile); + var dictionaryOfCurrentTemplates = DataProcessing.ConvertArrayOfFunctionalTemplatesToDictionary(arrayOfFunctionalTemplates); + + // init a new template list for output. + var newTemplateList = new List(); + + // cycle through all the manifests + foreach (var manifest in manifests) + { + var name = manifest.Name; + if (!dictionaryOfCurrentTemplates.ContainsKey(name)) + { + // the current manifest is not an existing template - therefore make it. + var newTemplate = new FunctionalTemplate(manifest) + { + Template = TemplateManifest.CreateTemplateDefinition(manifest), + MostRecentEdit = DateTime.Now, + }; + newTemplateList.Add(newTemplate); + continue; + } + + if (manifest.EditStatus == EditStatus.Edit) + { + // This option edits an existing functional template in the json file. The template definition is (re)calculated. + // Effectively the same as creating a new template. + var newTemplate = new FunctionalTemplate(manifest) + { + Template = TemplateManifest.CreateTemplateDefinition(manifest), + MostRecentEdit = DateTime.Now, + }; + newTemplateList.Add(newTemplate); + continue; + } + + if (manifest.EditStatus == EditStatus.Copy) + { + // This option keeps an existing functional template unchanged. + var existingTemplate = dictionaryOfCurrentTemplates[name]; + newTemplateList.Add(existingTemplate); + continue; + } + + if (manifest.EditStatus == EditStatus.Ignore) + { + // Do not output this template to the list of functional templates. + continue; + } + } + + var functionalTemplatesFileName = functionalTemplatesFile.Name; + var functionalTemplatesFilePath = Path.Combine(manifestFile.DirectoryName ?? throw new InvalidOperationException(), functionalTemplatesFileName); + + // Save the previous templates file + string backupPath = Path.Combine(manifestFile.DirectoryName, functionalTemplatesFileName + ".Backup.json"); + if (File.Exists(backupPath)) + { + File.Delete(backupPath); + } + + //Now copy the file first + File.Copy(functionalTemplatesFilePath, backupPath, true); + + //Now Rename the File + //File.Move(NewFilePath, Path.Combine(NewFileLocation, "File.txt")); + + // No need to move the backup because serializing over-writes the current templates file. + var opTemplatesFile = new FileInfo(functionalTemplatesFilePath); + Json.Serialise(opTemplatesFile, newTemplateList.ToArray()); } } } diff --git a/src/AnalysisPrograms/ContentDescription/ContentDescription.cs b/src/AnalysisPrograms/ContentDescription/UseModel.cs similarity index 96% rename from src/AnalysisPrograms/ContentDescription/ContentDescription.cs rename to src/AnalysisPrograms/ContentDescription/UseModel.cs index 607e4bba7..0e41a3eb8 100644 --- a/src/AnalysisPrograms/ContentDescription/ContentDescription.cs +++ b/src/AnalysisPrograms/ContentDescription/UseModel.cs @@ -1,4 +1,4 @@ -// +// // All code in this file and all associated files are the copyright and property of the QUT Ecoacoustics Research Group (formerly MQUTeR, and formerly QUT Bioacoustics Research Group). // @@ -28,7 +28,7 @@ namespace AnalysisPrograms.ContentDescription /// To call this class, the first argument on the commandline must be 'audio2csv'. /// Given a one-minute recording segment, the ContentDescription.Analyze() method calls AudioAnalysisTools.Indices.IndexCalculateSixOnly.Analysis(). /// This calculates six spectral indices, ACI, ENT, EVN, BGN, PMN, OSC. This set of 6x256 acoustic features is used for content description. - /// The content description methods are called from ContentDescription.SummariseResults() method. + /// The content description methods are called from UseModel.SummariseResults() method. /// public class UseModel : AbstractStrongAnalyser { @@ -52,9 +52,9 @@ public class UseModel : AbstractStrongAnalyser AnalysisTargetSampleRate = ContentSignatures.SampleRate, }; - private FunctionalTemplate[] functionalTemplates = null; + private FunctionalTemplate[] functionalTemplates; - private Dictionary> templatesAsDictionary = null; + private Dictionary> templatesAsDictionary; public override void BeforeAnalyze(AnalysisSettings analysisSettings) { @@ -94,11 +94,7 @@ public class CdConfig : AnalyzerConfig public override AnalysisResult2 Analyze(AnalysisSettings analysisSettings, SegmentSettings segmentSettings) { // set the start time for the current recording segment. Default is zero. - var elapsedTimeAtStartOfRecording = TimeSpan.Zero; - if (segmentSettings.SegmentStartOffset != null) - { - elapsedTimeAtStartOfRecording = (TimeSpan)segmentSettings.SegmentStartOffset; - } + var elapsedTimeAtStartOfRecording = segmentSettings.SegmentStartOffset; var startMinuteId = (int)Math.Round(elapsedTimeAtStartOfRecording.TotalMinutes); @@ -114,7 +110,7 @@ public override AnalysisResult2 Analyze(AnalysisSettings analysisSettings, Se segmentResults.SpectralIndexValues.FileName = segmentSettings.Segment.SourceMetadata.Identifier; // DO THE CONTENT DESCRIPTION FOR ONE MINUTE HERE - // First get acoustic indices for one minute, convert to Dictionary and normalise the values. + // First get acoustic indices for one minute, convert to Dictionary and normalize the values. var indicesDictionary = IndexCalculateSixOnly.ConvertIndicesToDictionary(segmentResults.SpectralIndexValues); foreach (string key in ContentSignatures.IndexNames) { @@ -196,8 +192,9 @@ public override void SummariseResults( //sampleRate = analysisSettings.Configuration.GetIntOrNull(AnalysisKeys.ResampleRate) ?? sampleRate; var cdConfiguration = (CdConfig)analysisSettings.Configuration; var ldSpectrogramConfig = cdConfiguration.LdSpectrogramConfig; - var cdConfigFile = analysisSettings.ConfigFile; - var configDirectory = cdConfigFile.DirectoryName ?? throw new ArgumentNullException(nameof(cdConfigFile), "Null value"); + + //var cdConfigFile = analysisSettings.ConfigFile; + //var configDirectory = cdConfigFile.DirectoryName ?? throw new ArgumentNullException(nameof(cdConfigFile), "Null value"); var sourceAudio = inputFileSegment.Source; string basename = Path.GetFileNameWithoutExtension(sourceAudio.Name); var resultsDirectory = AnalysisCoordinator.GetNamedDirectory(analysisSettings.AnalysisOutputDirectory, this); @@ -275,6 +272,7 @@ public override void SummariseResults( image.Save(path3); } + /* /// /// NOTE: THIS METHOD SHOULD EVENTUALLY BE DELETED. NO LONGER CALLED. /// Calculate the content description for each minute. @@ -327,6 +325,7 @@ private static Dictionary GetContentDescription( var dictionaryOfScores = DataProcessing.ConvertResultsToDictionaryOfArrays(results, length, startMinuteId); return dictionaryOfScores; } + */ /// /// Produce plots for graphical display. diff --git a/src/AnalysisPrograms/Sandpit.cs b/src/AnalysisPrograms/Sandpit.cs index b6383f106..a9939c4c5 100644 --- a/src/AnalysisPrograms/Sandpit.cs +++ b/src/AnalysisPrograms/Sandpit.cs @@ -16,7 +16,8 @@ namespace AnalysisPrograms using Acoustics.Shared; using Acoustics.Shared.Csv; using Acoustics.Tools.Wav; - using AnalyseLongRecordings; + using AnalysisPrograms.AnalyseLongRecordings; + using AnalysisPrograms.ContentDescription; using AudioAnalysisTools; using AudioAnalysisTools.ContentDescriptionTools; using AudioAnalysisTools.DSP; @@ -25,7 +26,7 @@ namespace AnalysisPrograms using AudioAnalysisTools.StandardSpectrograms; using AudioAnalysisTools.WavTools; using McMaster.Extensions.CommandLineUtils; - using Production.Arguments; + using AnalysisPrograms.Production.Arguments; using TowseyLibrary; /// @@ -121,9 +122,17 @@ public override Task Execute(CommandLineApplication app) public static void ContentDescriptionCreateTemplates() { - var templateManifests = new FileInfo(@"C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml"); - var templateDefinitions = new FileInfo(@"C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json"); - TemplateManifest.CreateNewFileOfTemplateDefinitions(templateManifests, templateDefinitions); + /* COMMAND LINE FOR BUILDING CONTENT DEFINITIONS MODEL OF TEMPLATES + * .\AnalysisPrograms.exe BuildContentDescriptionModel "C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml" "C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json" + */ + var templateManifestsPath = @"C:\Ecoacoustics\ContentDescription\ContentDescriptionTemplateManifests.yml"; + var templateDefinitionsPath = @"C:\Ecoacoustics\ContentDescription\Towsey.TemplateDefinitions.json"; + var arguments = new BuildModel.Arguments + { + TemplateManifest = templateManifestsPath.ToFileInfo(), + TemplateDefinitions = templateDefinitionsPath.ToFileInfo(), + }; + BuildModel.Execute(arguments); Console.WriteLine("# Finished creation of new manifest"); } diff --git a/src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs b/src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs index 4178c9173..2bd562244 100644 --- a/src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs +++ b/src/AudioAnalysisTools/ContentDescriptionTools/ContentSignatures.cs @@ -153,11 +153,6 @@ public static DescriptionResult AnalyzeOneMinute( // now subject the indices to various content searches foreach (var template in templates) { - if (template.UseStatus == false) - { - continue; - } - var algorithmType = template.Manifest.FeatureExtractionAlgorithm; var templateIndices = templatesAsDictionary[template.Manifest.Name]; double score; diff --git a/src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs b/src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs index ab0bb53fd..aa3330d8c 100644 --- a/src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs +++ b/src/AudioAnalysisTools/ContentDescriptionTools/DataProcessing.cs @@ -19,12 +19,12 @@ public static class DataProcessing /// /// An array of templates. /// A dictionary of templates. - public static Dictionary ConvertTemplateArrayToDictionary(TemplateManifest[] array) + public static Dictionary ConvertArrayOfFunctionalTemplatesToDictionary(FunctionalTemplate[] array) { - var dictionary = new Dictionary(); + var dictionary = new Dictionary(); foreach (var template in array) { - dictionary.Add(template.Name, template); + dictionary.Add(template.Manifest.Name, template); } return dictionary; diff --git a/src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs b/src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs index dfb44885c..5a42ad622 100644 --- a/src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs +++ b/src/AudioAnalysisTools/ContentDescriptionTools/TemplateManifest.cs @@ -7,8 +7,6 @@ namespace AudioAnalysisTools.ContentDescriptionTools using System; using System.Collections.Generic; using System.IO; - using Accord.IO; - using Acoustics.Shared; /// /// Templates are initially defined manually in a YAML file. Each template in a YAML file is called a "manifest". @@ -20,112 +18,25 @@ namespace AudioAnalysisTools.ContentDescriptionTools /// The functional template also contains information required to scan new recordings with the template definition. /// /// Each template manifest in a yml file contains an EditStatus field which describes what to with the manifest. - /// There are there options as described below. + /// There are three options as described below. /// public enum EditStatus { Edit, // This option edits an existing functional template in the json file. The template definition is (re)calculated. Copy, // This option keeps an existing functional template unchanged. - Ignore, // This option keeps an existing functional template unchanged except changes its UseStatus boolean field to FALSE. + Ignore, // This option ignores the manifest, i.e. does not transfer it to the array of functional templates. } /// /// This is base class for both template manifests and functional templates. /// Most of the fields and properties are common to both manifests and functional templates. /// Manifests contain the template provenance. This does not appear in the functional template because provenance includes path data. - /// TODO Set up inheritance from base class so that there is separate class for manifests and functional templates. - /// /// This class also contains methods to create new or edit existing functional templates based on info in the manifests. /// public class TemplateManifest { - public static void CreateNewFileOfTemplateDefinitions(FileInfo manifestFile, FileInfo templateDefinitionsFile) - { - // Read in all template manifests - var manifests = Yaml.Deserialize(manifestFile); - - // Read current template definitions and convert to dictionary - var arrayOfTemplates = Json.Deserialize(templateDefinitionsFile); - var dictionaryOfCurrentTemplates = DataProcessing.ConvertTemplateArrayToDictionary(arrayOfTemplates); - - // init a new template list for output. - var newTemplateList = new List(); - - // cycle through all the manifests - for (var i = 0; i < manifests.Length; i++) - { - var manifest = manifests[i]; - var name = manifest.Name; - if (!dictionaryOfCurrentTemplates.ContainsKey(name)) - { - // the current manifest is not an existing template - therefore make it. - var newTemplate = CreateNewTemplateFromManifest(manifest); - newTemplate.Template = CreateTemplateDefinition(manifest); - newTemplate.MostRecentEdit = DateTime.Now; - newTemplateList.Add(newTemplate); - continue; - } - - if (manifest.EditStatus == EditStatus.Edit) - { - // This option edits an existing functional template in the json file. The template definition is (re)calculated. - var newTemplate = CreateNewTemplateFromManifest(manifest); - newTemplate.Template = CreateTemplateDefinition(manifest); - newTemplate.MostRecentEdit = DateTime.Now; - newTemplateList.Add(newTemplate); - continue; - } - - if (manifest.EditStatus == EditStatus.Copy) - { - // TODO: intentionally broken. FunctionalTemplates should be immutable. If they need to change create a new one (could be a copy, but it would have a version or edit date etc...). - throw new NotImplementedException(); - // This option keeps an existing functional template unchanged. - //var existingTemplate = dictionaryOfCurrentTemplates[name]; - //existingTemplate.UseStatus = true; - //existingTemplate.Provenance = null; - //newTemplateList.Add(existingTemplate); - //continue; - } - - if (manifest.EditStatus == EditStatus.Ignore) - { - // TODO: intentionally broken. FunctionalTemplates should be immutable. If they need to change create a new one (could be a copy, but it would have a version or edit date etc...). - // TODO: Per the comment above, if they're regenerated, there's no need to ignore some - throw new NotImplementedException(); - // This option keeps an existing functional template unchanged except changes its UseStatus boolean field to FALSE. - //var existingTemplate = dictionaryOfCurrentTemplates[name]; - //existingTemplate.Provenance = null; - //existingTemplate.UseStatus = false; - //newTemplateList.Add(existingTemplate); - } - } - - var templatesFileName = templateDefinitionsFile.Name; - var templatesFilePath = Path.Combine(manifestFile.DirectoryName ?? throw new InvalidOperationException(), templatesFileName); - - // Save the previous templates file - string backupTemplatesFilePath = Path.Combine(manifestFile.DirectoryName, templatesFileName + ".Backup.json"); - if (File.Exists(backupTemplatesFilePath)) - { - File.Delete(backupTemplatesFilePath); - } - - //Now copy the file first - File.Copy(templatesFilePath, backupTemplatesFilePath, true); - - //Now Rename the File - //File.Move(NewFilePath, Path.Combine(NewFileLocation, "File.txt")); - - // No need to move the backup because serializing over-writes the current templates file. - var templatesFile = new FileInfo(templatesFilePath); - - //Yaml.Serialize(templatesFile, newTemplateList.ToArray()); - Json.Serialise(templatesFile, newTemplateList.ToArray()); - } - /// - /// THis method calculates new template based on passed manifest. + /// This method calculates new template based on passed manifest. /// public static Dictionary CreateTemplateDefinition(TemplateManifest templateManifest) { @@ -139,44 +50,26 @@ public static Dictionary CreateTemplateDefinition(TemplateMani var path = Path.Combine(sourceDirectory, baseName + ContentSignatures.AnalysisString); var dictionaryOfIndices = DataProcessing.ReadIndexMatrices(path); var algorithmType = templateManifest.FeatureExtractionAlgorithm; - Dictionary newTemplateDeftn; + Dictionary newTemplateDefinition; switch (algorithmType) { case 1: - newTemplateDeftn = ContentAlgorithms.CreateFullBandTemplate1(templateManifest, dictionaryOfIndices); + newTemplateDefinition = ContentAlgorithms.CreateFullBandTemplate1(templateManifest, dictionaryOfIndices); break; case 2: - newTemplateDeftn = ContentAlgorithms.CreateBroadbandTemplate1(templateManifest, dictionaryOfIndices); + newTemplateDefinition = ContentAlgorithms.CreateBroadbandTemplate1(templateManifest, dictionaryOfIndices); break; case 3: - newTemplateDeftn = ContentAlgorithms.CreateNarrowBandTemplate1(templateManifest, dictionaryOfIndices); + newTemplateDefinition = ContentAlgorithms.CreateNarrowBandTemplate1(templateManifest, dictionaryOfIndices); break; default: //LoggedConsole.WriteWarnLine("Algorithm " + algorithmType + " does not exist."); - newTemplateDeftn = null; + newTemplateDefinition = null; break; } - return newTemplateDeftn; - } - - public static FunctionalTemplate CreateNewTemplateFromManifest(TemplateManifest templateManifest) - { - var newTemplate = new FunctionalTemplate() - { - // TODO: is clone actually needed here? - // I chose clone because it mirrors the functionality that *was* here - i.e. a refactor - Manifest = templateManifest.DeepClone(), - UseStatus = true, - }; - - if (templateManifest.EditStatus == EditStatus.Ignore) - { - newTemplate.UseStatus = false; - } - - return newTemplate; + return newTemplateDefinition; } //TEMPLATE DESCRIPTION @@ -199,7 +92,6 @@ public static FunctionalTemplate CreateNewTemplateFromManifest(TemplateManifest /// public EditStatus EditStatus { get; set; } - //ALGORITHMIC PARAMETERS ASSOCIATED WITH TEMPLATE public byte FeatureExtractionAlgorithm { get; set; } @@ -224,14 +116,7 @@ public static FunctionalTemplate CreateNewTemplateFromManifest(TemplateManifest /// public int BandMaxHz { get; set; } - - public SourceAudioProvenance[] Provenance { get; set; } - - // The following random data was used to try some statistical experiments. - // get dummy data - //var rn = new RandomNumber(DateTime.Now.Second + (int)DateTime.Now.Ticks + 333); - //var distance = rn.GetDouble(); } /// @@ -239,11 +124,13 @@ public static FunctionalTemplate CreateNewTemplateFromManifest(TemplateManifest /// public class SourceAudioProvenance { + /* // TODO: use this property as the source audio for which indices will be calculated from /// /// Gets or sets the template Recording Location. /// public string Path { get; set; } + */ // TODO: remove when calculating indices directly from audio segments /// @@ -275,13 +162,14 @@ public class SourceAudioProvenance public class FunctionalTemplate { - public TemplateManifest Manifest { get; set; } - /// - /// Gets or sets a value indicating whether to use the template or not. - /// UseStatus can be true or false. + /// Initializes a new instance of the class. + /// CONSTRUCTOR must initialise the info from the Manifest. /// - public bool UseStatus { get; set; } + /// The template manifest. + public FunctionalTemplate(TemplateManifest templateManifest) => this.Manifest = templateManifest; + + public TemplateManifest Manifest { get; set; } /// /// Gets or sets the date the functional template was created.