diff --git a/Directory.Packages.props b/Directory.Packages.props index 655ef9d9..27c99cd4 100644 --- a/Directory.Packages.props +++ b/Directory.Packages.props @@ -12,7 +12,7 @@ - + @@ -25,7 +25,6 @@ - @@ -41,9 +40,11 @@ - - + + + + diff --git a/Microsoft.Sbom.sln b/Microsoft.Sbom.sln index 6069741f..54633129 100644 --- a/Microsoft.Sbom.sln +++ b/Microsoft.Sbom.sln @@ -19,6 +19,10 @@ Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.SPDX22SBOMPa EndProject Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.SPDX22SBOMParser.Tests", "test\Microsoft.Sbom.SPDX22SBOMParser.Tests\Microsoft.Sbom.SPDX22SBOMParser.Tests.csproj", "{ADDEE422-40D1-48D9-A5FB-BBE990272B78}" EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Api", "src\Microsoft.Sbom.Api\Microsoft.Sbom.Api.csproj", "{725723C5-DCA4-4BAD-8883-CC94E5F5A5A8}" +EndProject +Project("{9A19103F-16F7-4668-BE54-9A1E7A4F7556}") = "Microsoft.Sbom.Api.Tests", "test\Microsoft.Sbom.Api.Tests\Microsoft.Sbom.Api.Tests.csproj", "{4F94EA4F-CC6B-4FA0-8A7E-654EAA26B625}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -53,6 +57,14 @@ Global {ADDEE422-40D1-48D9-A5FB-BBE990272B78}.Debug|Any CPU.Build.0 = Debug|Any CPU {ADDEE422-40D1-48D9-A5FB-BBE990272B78}.Release|Any CPU.ActiveCfg = Release|Any CPU {ADDEE422-40D1-48D9-A5FB-BBE990272B78}.Release|Any CPU.Build.0 = Release|Any CPU + {725723C5-DCA4-4BAD-8883-CC94E5F5A5A8}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {725723C5-DCA4-4BAD-8883-CC94E5F5A5A8}.Debug|Any CPU.Build.0 = Debug|Any CPU + {725723C5-DCA4-4BAD-8883-CC94E5F5A5A8}.Release|Any CPU.ActiveCfg = Release|Any CPU + {725723C5-DCA4-4BAD-8883-CC94E5F5A5A8}.Release|Any CPU.Build.0 = Release|Any CPU + {4F94EA4F-CC6B-4FA0-8A7E-654EAA26B625}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {4F94EA4F-CC6B-4FA0-8A7E-654EAA26B625}.Debug|Any CPU.Build.0 = Debug|Any CPU + {4F94EA4F-CC6B-4FA0-8A7E-654EAA26B625}.Release|Any CPU.ActiveCfg = Release|Any CPU + {4F94EA4F-CC6B-4FA0-8A7E-654EAA26B625}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(SolutionProperties) = preSolution HideSolutionNode = FALSE diff --git a/src/Microsoft.Sbom.Api/Bindings.cs b/src/Microsoft.Sbom.Api/Bindings.cs new file mode 100644 index 00000000..6f3975ea --- /dev/null +++ b/src/Microsoft.Sbom.Api/Bindings.cs @@ -0,0 +1,247 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Converters; +using Microsoft.Sbom.Api.Convertors; +using Microsoft.Sbom.Api.Entities.Output; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Filters; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Logging; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Providers; +using Microsoft.Sbom.Api.SignValidator; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Api.Workflows; +using Microsoft.Sbom.Api.Workflows.Helpers; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts.Interfaces; +using Ninject; +using Ninject.Extensions.Conventions; +using Ninject.Modules; +using Serilog; +using Microsoft.Sbom.Api.Config; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Common.Extensions; + +namespace Microsoft.Sbom.Api +{ + /// + /// Creates the Ninject bindings for the whole project. + /// + /// + /// Microsoft.ManifestTool.Api.dll is the assembly name of the SBOM API project. + /// Using pattern matching until all bindings are in the same assembly. + /// + public class Bindings : NinjectModule + { + [System.Diagnostics.CodeAnalysis.SuppressMessage("StyleCop.CSharp.ReadabilityRules", "SA1123:Do not place regions within elements", Justification = "Enable documentation of code")] + public override void Load() + { + Bind().ToProvider().InSingletonScope(); + + Bind().ToSelf(); + Bind().To(); + Bind().To().InSingletonScope(); + Bind().To().InSingletonScope(); + Bind().ToSelf(); + Bind().To().Named(nameof(FileArrayGenerator)); + Bind().To().Named(nameof(PackageArrayGenerator)); + Bind().To().Named(nameof(RelationshipsArrayGenerator)); + Bind().To().Named(nameof(ExternalDocumentReferenceGenerator)); + Bind().ToSelf(); + Bind().To().InSingletonScope(); + + Bind().To().Named(nameof(DownloadedRootPathFilter)).OnActivation(f => f.Init()); + Bind().To().Named(nameof(ManifestFolderFilter)).OnActivation(f => f.Init()); + + Bind().ToProvider(); + + #region Bind all manifest parsers + + // Search external assemblies + Kernel.Bind(scan => scan + .FromAssembliesMatching("*Parsers*") + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + // Search this assembly in case --self-contained is used with dotnet publish + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().ToProvider().InSingletonScope(); + Bind().ToSelf().InSingletonScope().OnActivation(m => m.Init()); + + #endregion + + #region Bind all manifest generators + + // Search external assemblies + Kernel.Bind(scan => scan + .FromAssembliesMatching("*Parsers*") + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + // Search this assembly in case --self-contained is used with dotnet publish + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().ToSelf().InSingletonScope().OnActivation(mg => mg.Init()); + + #endregion + + #region Bind all signature validators + Kernel.Bind(scan => scan + .FromAssembliesMatching("*Parsers*") + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().ToSelf().InSingletonScope().OnActivation(s => s.Init()); + + #endregion + + #region Manifest Config + + Kernel.Bind(scan => scan + .FromAssembliesMatching("*Parsers*") + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().To().InSingletonScope(); + Bind().To(); + + #endregion + + #region QuickBuild Manifest workflow bindings + Bind().To(); + Bind().To(); + #endregion + + #region AutoMapper bindings + var mapperConfiguration = new MapperConfiguration(cfg => cfg.AddProfile()); + mapperConfiguration.AssertConfigurationIsValid(); + Bind().ToConstant(mapperConfiguration).InSingletonScope(); + Bind().ToMethod(ctx => + new Mapper(mapperConfiguration, type => ctx.Kernel.Get(type))); + + #endregion + + #region Workflows + + Bind().To().Named(nameof(DropValidatorWorkflow)); + Bind().To().Named(nameof(SBOMGenerationWorkflow)); + + #endregion + + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllBaseClasses()); + + #region Bind metadata providers + + Kernel.Bind(scan => scan + .FromAssembliesInPath(new AssemblyConfig().AssemblyDirectory) + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().To(); + + #endregion + + #region Bind all sources providers. + Kernel.Bind(scan => scan + .FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + #endregion + + #region Converters + + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + + #endregion + + #region Executors + + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().ToSelf().InThreadScope(); + Bind().To().InThreadScope(); + + #endregion + + #region Bind all hash algorithm providers + + // TODO: Put all dependent assemblies in the plugins folder and search using + // that path here. + Kernel.Bind(scan => scan + .FromAssembliesMatching("*Parsers*", "*Contract*") + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + // We should move all algorithm implementations into their own lib, so that + // we can remove this additional scan. + Kernel.Bind(scan => scan. + FromThisAssembly() + .SelectAllClasses() + .InheritedFrom() + .BindAllInterfaces()); + + Bind().To().InSingletonScope(); + + #endregion + + Bind().To().InSingletonScope(); + Bind().ToSelf().InSingletonScope(); + Bind().ToSelf().InSingletonScope(); + Bind().ToSelf().InSingletonScope(); + Bind().To().InSingletonScope(); + Bind().To().InSingletonScope(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs new file mode 100644 index 00000000..eb47d576 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ApiConfigurationBuilder.cs @@ -0,0 +1,168 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts; +using Serilog.Events; +using Constants = Microsoft.Sbom.Common.Constants; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// Builds the configuration object for the SBOM api. + /// + internal class ApiConfigurationBuilder + { + /// + /// Gets a generate configuration. + /// + /// Path where package exists. If scanning start here. + /// Output path to where manifest is generated. + /// Use null to scan. + /// Use null to scan. + /// + /// + /// + /// + /// A generate configuration. + internal Configuration GetConfiguration( + string rootPath, + string manifestDirPath, + IEnumerable files, + IEnumerable packages, + SBOMMetadata metadata, + IList specifications = null, + RuntimeConfiguration runtimeConfiguration = null, + string externalDocumentReferenceListFile = null, + string componentPath = null) + { + if (string.IsNullOrWhiteSpace(rootPath)) + { + throw new ArgumentException($"'{nameof(rootPath)}' cannot be null or whitespace.", nameof(rootPath)); + } + + if (metadata is null) + { + throw new ArgumentNullException(nameof(metadata)); + } + + RuntimeConfiguration sanitizedRuntimeConfiguration = SanitiseRuntimeConfiguration(runtimeConfiguration); + + var configuration = new Configuration(); + configuration.BuildDropPath = GetConfigurationSetting(rootPath); + configuration.ManifestDirPath = GetConfigurationSetting(manifestDirPath); + configuration.ManifestToolAction = ManifestToolActions.Generate; + configuration.PackageName = GetConfigurationSetting(metadata.PackageName); + configuration.PackageVersion = GetConfigurationSetting(metadata.PackageVersion); + configuration.Parallelism = GetConfigurationSetting(sanitizedRuntimeConfiguration.WorkflowParallelism); + configuration.GenerationTimestamp = GetConfigurationSetting(sanitizedRuntimeConfiguration.GenerationTimestamp); + configuration.NamespaceUriBase = GetConfigurationSetting(sanitizedRuntimeConfiguration.NamespaceUriBase); + configuration.NamespaceUriUniquePart = GetConfigurationSetting(sanitizedRuntimeConfiguration.NamespaceUriUniquePart); + configuration.FollowSymlinks = GetConfigurationSetting(sanitizedRuntimeConfiguration.FollowSymlinks); + + SetVerbosity(sanitizedRuntimeConfiguration, configuration); + + if (packages != null) + { + configuration.PackagesList = GetConfigurationSetting(packages); + } + + if (files != null) + { + configuration.FilesList = GetConfigurationSetting(files); + } + + if (externalDocumentReferenceListFile != null) + { + configuration.ExternalDocumentReferenceListFile = GetConfigurationSetting(externalDocumentReferenceListFile); + } + + if (!string.IsNullOrWhiteSpace(componentPath)) + { + configuration.BuildComponentPath = GetConfigurationSetting(componentPath); + } + + // Convert sbom specifications to manifest info. + if (specifications != null) + { + if (specifications.Count == 0) + { + throw new ArgumentException($"'{nameof(specifications)}' must have at least 1 specification.", nameof(specifications)); + } + + IList manifestInfos = specifications + .Select(s => s.ToManifestInfo()) + .ToList(); + + configuration.ManifestInfo = GetConfigurationSetting(manifestInfos); + } + + return configuration; + } + + private void SetVerbosity(RuntimeConfiguration sanitizedRuntimeConfiguration, Configuration configuration) + { + switch (sanitizedRuntimeConfiguration.Verbosity) + { + case System.Diagnostics.Tracing.EventLevel.Critical: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Fatal); + break; + case System.Diagnostics.Tracing.EventLevel.Informational: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Information); + break; + case System.Diagnostics.Tracing.EventLevel.Error: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Error); + break; + case System.Diagnostics.Tracing.EventLevel.LogAlways: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Verbose); + break; + case System.Diagnostics.Tracing.EventLevel.Warning: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Warning); + break; + case System.Diagnostics.Tracing.EventLevel.Verbose: + configuration.Verbosity = GetConfigurationSetting(LogEventLevel.Verbose); + break; + default: + configuration.Verbosity = GetConfigurationSetting(Constants.DefaultLogLevel); + break; + } + } + + private ConfigurationSetting GetConfigurationSetting(T value) + { + return new ConfigurationSetting + { + Value = value, + Source = SettingSource.SBOMApi + }; + } + + private RuntimeConfiguration SanitiseRuntimeConfiguration(RuntimeConfiguration runtimeConfiguration) + { + if (runtimeConfiguration == null) + { + runtimeConfiguration = new RuntimeConfiguration + { + WorkflowParallelism = Constants.DefaultParallelism, + Verbosity = System.Diagnostics.Tracing.EventLevel.Warning, + DeleteManifestDirectoryIfPresent = false, + FollowSymlinks = true + }; + } + + if (runtimeConfiguration.WorkflowParallelism < Constants.MinParallelism + || runtimeConfiguration.WorkflowParallelism > Constants.MaxParallelism) + { + runtimeConfiguration.WorkflowParallelism = Constants.DefaultParallelism; + } + + return runtimeConfiguration; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ArgRevivers.cs b/src/Microsoft.Sbom.Api/Config/ArgRevivers.cs new file mode 100644 index 00000000..a83f6355 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ArgRevivers.cs @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts.Enums; +using PowerArgs; +using System; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Config +{ + public class ArgRevivers + { + /// + /// Creates a list of objects from a string value + /// The string manifest infos are seperated by commas. + /// + [ArgReviver] + public static IList ReviveManifestInfo(string _, string value) + { + try + { + IList manifestInfos = new List(); + string[] values = value.Split(','); + foreach (var manifestInfoStr in values) + { + manifestInfos.Add(ManifestInfo.Parse(manifestInfoStr)); + } + + return manifestInfos; + } + catch (Exception e) + { + throw new ValidationArgException($"Unable to parse manifest info string list: {value}. Error: {e.Message}"); + } + } + + /// + /// Creates an object from a string value. + /// + [ArgReviver] + public static AlgorithmName ReviveAlgorithmName(string _, string value) + { + try + { + // Return a placeholder object for now. The config post processor will convert this into + // a real AlgorithmName object. We only need to preserve the string value (name) of the algorithm. + return new AlgorithmName(value, null); + } + catch (Exception e) + { + throw new ValidationArgException($"Unable to parse algorithm name: {value}. Error: {e.Message}"); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Args/CommonArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/CommonArgs.cs new file mode 100644 index 00000000..bb7a2139 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Args/CommonArgs.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using PowerArgs; +using Serilog.Events; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Config.Args +{ + /// + /// Defines the common arguments used by all actions of the ManifestTool + /// + public class CommonArgs + { + /// + /// Gets or sets display this amount of detail in the logging output. + /// + [ArgDescription("Display this amount of detail in the logging output.")] + public LogEventLevel? Verbosity { get; set; } + + /// + /// Gets or sets the number of parallel threads to use for the workflows. + /// + [ArgDescription("The number of parallel threads to use for the workflows.")] + public int? Parallelism { get; set; } + + /// + /// Gets or sets a JSON config file that can be used to specify all the arguments for an action + /// + [ArgDescription("The json file that contains the configuration for the DropValidator.")] + public string ConfigFilePath { get; set; } + + /// + /// Gets or sets the action currently being performed by the manifest tool. + /// + [ArgIgnore] + public ManifestToolActions ManifestToolAction { get; set; } + + [ArgShortcut("t")] + [ArgDescription("Specify a file where we should write detailed telemetry for the workflow.")] + public string TelemetryFilePath { get; set; } + + /// + /// Gets or sets if set to false, we will not follow symlinks while traversing the build drop folder. Default is set to 'true'. + /// + [ArgDescription("If set to false, we will not follow symlinks while traversing the build drop folder. Default is set to 'true'.")] + public bool? FollowSymlinks { get; set; } + + /// + /// Gets or sets the name and version of the manifest format that we are using. + /// + [ArgDescription("A list of the name and version of the manifest format that we are using.")] + [ArgShortcut("mi")] + public IList ManifestInfo { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs new file mode 100644 index 00000000..0309570d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Args/GenerationArgs.cs @@ -0,0 +1,98 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; + +namespace Microsoft.Sbom.Api.Config.Args +{ + /// + /// The command line arguments provided for the generate action in ManifestTool + /// + public class GenerationArgs : CommonArgs + { + /// + /// Gets or sets the root folder of the drop directory for which the manifest file will be generated. + /// + [ArgShortcut("b")] + [ArgRequired(IfNot = "ConfigFilePath")] + [ArgDescription("The root folder of the drop directory for which the manifest file will be generated.")] + public string BuildDropPath { get; set; } + + /// + /// Gets or sets the folder containing the build components and packages. + /// + [ArgShortcut("bc")] + [ArgDescription("The folder containing the build components and packages.")] + public string BuildComponentPath { get; set; } + + /// + /// Gets or sets the file path containing a list of files for which the manifest file will be generated. + /// List file is an unordered set of files formated as one file per line separated + /// by Environment.NewLine. Blank lines are discarded. + /// + [ArgShortcut("bl")] + [ArgDescription("The file path to a file containing a list of files one file per line for which the manifest file will be generated. Only files listed in the file will be inlcuded in the generated manifest.")] + public string BuildListFile { get; set; } + + /// + /// Gets or sets the root folder where the generated manifest (and other files like bsi.json) files will be placed. + /// By default we will generate this folder in the same level as the build drop with the name '_manifest' + /// + [ArgShortcut("m")] + [ArgDescription("The path of the directory where the generated manifest files will be placed." + + " If this parameter is not specified, the files will be placed in {BuildDropPath}/_manifest directory.")] + public string ManifestDirPath { get; set; } + + /// + /// Gets or sets the name of the package this SBOM represents. + /// + [ArgShortcut("pn")] + [ArgDescription("The name of the package this SBOM represents. If this is not provided, we will try to infer this " + + "name from the build that generated this package, if that also fails, the SBOM generation fails.")] + public string PackageName { get; set; } + + [ArgShortcut("pv")] + [ArgDescription("The version of the package this SBOM represents. If this is not provided, we will " + + "try to infer the version from the build that generated this package, if that also fails, the " + + "SBOM generation fails.")] + public string PackageVersion { get; set; } + + [ArgDescription("Comma separated list of docker image names or hashes to be scanned for packages, ex: ubuntu:16.04, 56bab49eef2ef07505f6a1b0d5bd3a601dfc3c76ad4460f24c91d6fa298369ab.")] + [ArgShortcut("di")] + public string DockerImagesToScan { get; set; } + + [ArgShortcut("cd")] + [ArgDescription("Additional set of arguments for Component Detector. An appropriate usage of this would be a space-delimited list of `--key value` pairs, respresenting command-line switches.")] + public string AdditionalComponentDetectorArgs { get; set; } + + /// + /// Gets or sets the path to a file containing a list of external SBOMs that will be included as external document reference in the output SBOM. + /// + [ArgShortcut("er")] + [ArgDescription("The path to a file containing a list of external SBOMs that will be included as external document reference in the output SBOM. SPDX 2.2 is the only supported format for now.")] + public string ExternalDocumentReferenceListFile { get; set; } + + /// + /// Gets or sets unique part of the namespace uri for SPDX 2.2 SBOMs. This value should be globally unique. + /// If this value is not provided, we generate a unique guid that will make the namespace globally unique. + /// + [ArgShortcut("nsu")] + [ArgDescription("A unique valid URI part that will be appended to the SPDX SBOM namespace URI. This value should be globally unique.")] + public string NamespaceUriUniquePart { get; set; } + + /// + /// Gets or sets the base of the URI that will be used to generate this SBOM. This should be a value that identifies that + /// the SBOM belongs to a single publisher (or company) + /// + [ArgShortcut("nsb")] + [ArgDescription("The base path of the SBOM namespace URI.")] + public string NamespaceUriBase { get; set; } + + /// + /// Gets or sets a timestamp in the format yyyy-MM-ddTHH:mm:ssZ that will be used as the generated timestamp for the SBOM. + /// + [ArgShortcut("gt")] + [ArgDescription("A timestamp in the format 'yyyy-MM-ddTHH:mm:ssZ' that will be used as the generated timestamp for the SBOM.")] + public string GenerationTimestamp { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs new file mode 100644 index 00000000..f8f461ad --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Args/ValidationArgs.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using Microsoft.Sbom.Extensions.Entities; +using System.Collections.Generic; +using Microsoft.Sbom.Contracts.Enums; + +namespace Microsoft.Sbom.Api.Config.Args +{ + /// + /// The command line arguments provided for the validate action in ManifestTool + /// + public class ValidationArgs : CommonArgs + { + /// + /// Gets or sets the root folder of the drop directory to validate. + /// + [ArgShortcut("b")] + [ArgRequired(IfNot = "ConfigFilePath")] + [ArgDescription("The root folder of the drop directory to validate.")] + public string BuildDropPath { get; set; } + + /// + /// Gets or sets the path to the _manifest folder.. + /// + [ArgShortcut("m")] + [ArgDescription("The path of the directory where the manifest will be validated." + + " If this parameter is not specified, the manifest will be validated in {BuildDropPath}/_manifest directory.")] + public string ManifestDirPath { get; set; } + + /// + /// Gets or sets the path where the output json should be written. + /// + [ArgShortcut("o")] + [ArgRequired(IfNot = "ConfigFilePath")] + [ArgDescription("The path where the output json should be written.")] + public string OutputPath { get; set; } + + /// + /// Gets or sets the path of the signed catalog file used to validate the manifest.json + /// + [ArgDescription("The path of signed catalog file that is used to verify the signature of the manifest json file.")] + public string CatalogFilePath { get; set; } + + /// + /// Gets or sets a value indicating whether if set, will validate the manifest using the signed catalog file. + /// + [ArgShortcut("s")] + [ArgDescription("If set, will validate the manifest using the signed catalog file.")] + public bool ValidateSignature { get; set; } + + /// + /// Gets or sets a value indicating whether if set, will not fail validation on the files presented in Manifest but missing on the disk. + /// + [ArgShortcut("im")] + [ArgDescription("If set, will not fail validation on the files presented in Manifest but missing on the disk.")] + public bool IgnoreMissing { get; set; } + + /// + /// Gets or sets if you're downloading only a part of the drop using the '-r' or 'root' parameter + /// in the drop client, specify the same string value here in order to skip + /// validating paths that are not downloaded. + /// + [ArgDescription(@"If you're downloading only a part of the drop using the '-r' or 'root' parameter in the drop client, specify the same string value here in order to skip validating paths that are not downloaded.")] + [ArgShortcut("r")] + public string RootPathFilter { get; set; } + + /// + /// Gets or sets the Hash algorithm to use while verifying or generating the hash value of a file. + /// + [ArgDescription("The Hash algorithm to use while verifying or generating the hash value of a file")] + public AlgorithmName HashAlgorithm { get; set; } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Config/ConfigFile.cs b/src/Microsoft.Sbom.Api/Config/ConfigFile.cs new file mode 100644 index 00000000..5f19f43b --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ConfigFile.cs @@ -0,0 +1,159 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts.Enums; +using Serilog.Events; +using System; +using System.Collections.Generic; +using System.Text.Json.Serialization; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// This is the schema for the config file that is used to provide + /// the validator with additional params in a JSON format. Most of + /// these fields can also be provided through the command line. In case + /// of a conflict (same value provided in config file and command line, we + /// throw an input error. + /// + public class ConfigFile + { + /// + /// Gets or sets the root folder of the drop directory to validate. + /// + public string BuildDropPath { get; set; } + + /// + /// Gets or sets the folder containing the build components and packages. + /// + public string BuildComponentPath { get; set; } + + /// + /// Gets or sets the file path containing a list of files for which the manifest file will be generated. + /// + public string BuildListFile { get; set; } + + /// + /// Gets or sets the path of the manifest json to use for validation. + /// + [Obsolete("This property is obsolete. Value will by generated by the system.")] + public string ManifestPath { get; set; } + + /// + /// Gets or sets the root folder where the generated manifest (and other files like bsi.json) files will be placed. + /// By default we will generate this folder in the same level as the build drop with the name '_manifest' + /// + public string ManifestDirPath { get; set; } + + /// + /// Gets or sets the path where the output json should be written. + /// + public string OutputPath { get; set; } + + /// + /// Gets or sets the path of the signed catalog file used to validate the manifest.json + /// + public string CatalogFilePath { get; set; } + + /// + /// Gets or sets if set, will validate the manifest using the signed catalog file. + /// + public bool? ValidateSignature { get; set; } + + /// + /// Gets or sets if set, will not fail validation on the files presented in Manifest but missing on the disk. + /// + public bool? IgnoreMissing { get; set; } + + /// + /// Gets or sets if you're downloading only a part of the drop using the '-r' or 'root' parameter + /// in the drop client, specify the same string value here in order to skip + /// validating paths that are not downloaded. + /// + public string RootPathFilter { get; set; } + + /// + /// Gets or sets display this amount of detail in the logging output. + /// + public LogEventLevel? Verbosity { get; set; } + + /// + /// Gets or sets the number of parallel threads to run for the validator. + /// + public int? Parallelism { get; set; } + + /// + /// Gets or sets a list of the name and version of the manifest format that we are using. + /// + public IList ManifestInfo { get; set; } + + /// + /// Gets or sets the Hash algorithm to use while verifying or generating the hash value of a file. + /// + public AlgorithmName HashAlgorithm { get; set; } + + /// + /// Gets or sets the name of the package this SBOM represents. + /// + public string PackageName { get; set; } + + /// + /// Gets or sets the version of the package this SBOM represents. + /// + public string PackageVersion { get; set; } + + /// + /// Gets or sets a JSON config file that can be used to specify all the arguments for an action + /// + [JsonIgnore] + public string ConfigFilePath { get; set; } + + [JsonIgnore] + public ManifestToolActions ManifestToolAction { get; set; } + + /// + /// Gets or sets if specified, we will store the generated telemetry for the execution + /// of the SBOM tool at this path. + /// + public string TelemetryFilePath { get; set; } + + /// + /// Gets or sets comma separated list of docker image names or hashes to be scanned for packages, ex: ubuntu:16.04, 56bab49eef2ef07505f6a1b0d5bd3a601dfc3c76ad4460f24c91d6fa298369ab. + /// + public string DockerImagesToScan { get; set; } + + /// + /// Gets or sets the file path containing a list of external SBOMs to include as external document reference. + /// + public string ExternalDocumentReferenceListFile { get; set; } + + /// + /// Gets or sets additional set of command-line arguments for Component Detector. + /// + public string AdditionalComponentDetectorArgs { get; set; } + + /// + /// Gets or sets unique part of the namespace uri for SPDX 2.2 SBOMs. This value should be globally unique. + /// If this value is not provided, we generate a unique guid that will make the namespace globally unique. + /// + public string NamespaceUriUniquePart { get; set; } + + /// + /// Gets or sets the base of the URI that will be used to generate this SBOM. This should be a value that identifies that + /// the SBOM belongs to a single publisher (or company) + /// + public string NamespaceUriBase { get; set; } + + /// + /// Gets or sets a timestamp in the format yyyy-MM-ddTHH:mm:ssZ that will be used as the generated timestamp for the SBOM. + /// + public string GenerationTimestamp { get; set; } + + /// + /// Gets or sets if set to false, we will not follow symlinks while traversing the build drop folder. + /// + public bool? FollowSymlinks { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ConfigFileParser.cs b/src/Microsoft.Sbom.Api/Config/ConfigFileParser.cs new file mode 100644 index 00000000..a4d80bba --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ConfigFileParser.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// Used to parse the configuration as a from a JSON file. + /// + public class ConfigFileParser + { + private readonly IFileSystemUtils fileSystemUtils; + + public ConfigFileParser(IFileSystemUtils fileSystemUtils) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + public async Task ParseFromJsonFile(string filePath) + { + if (string.IsNullOrEmpty(filePath)) + { + throw new ArgumentNullException($"{nameof(filePath)} cannot be emtpy."); + } + + using Stream openStream = fileSystemUtils.OpenRead(filePath); + return await JsonSerializer.DeserializeAsync(openStream); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ConfigPostProcessor.cs b/src/Microsoft.Sbom.Api/Config/ConfigPostProcessor.cs new file mode 100644 index 00000000..0a9a49a3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ConfigPostProcessor.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using DropValidator.Api.Config; +using Microsoft.Sbom.Api.Config.Validators; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Contracts.Enums; +using PowerArgs; +using System; +using System.ComponentModel; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// Runs finalizing operations on the configuration once it has been successfully parsed. + /// + public class ConfigPostProcessor : IMappingAction + { + private readonly ConfigValidator[] configValidators; + private readonly ConfigSanitizer configSanitizer; + + public ConfigPostProcessor(ConfigValidator[] configValidators, ConfigSanitizer configSanitizer) + { + this.configValidators = configValidators ?? throw new ArgumentNullException(nameof(configValidators)); + this.configSanitizer = configSanitizer ?? throw new ArgumentNullException(nameof(configSanitizer)); + } + + public void Process(Configuration source, Configuration destination, ResolutionContext context) + { + // Set current action on config validators + configValidators.ForEach(c => c.CurrentAction = destination.ManifestToolAction); + + foreach (PropertyDescriptor property in TypeDescriptor.GetProperties(destination)) + { + // Assign default values if any using the default value attribute. + if (property.GetValue(destination) == null && + property.Attributes[typeof(System.ComponentModel.DefaultValueAttribute)] + is System.ComponentModel.DefaultValueAttribute defaultValueAttribute) + { + SetDefautValue(destination, defaultValueAttribute.Value, property); + } + + // Run validators on all properties. + configValidators.ForEach(v => v.Validate(property.DisplayName, property.GetValue(destination), property.Attributes)); + } + + // Sanitize configuration + destination = configSanitizer.SanitizeConfig(destination); + } + + private void SetDefautValue(Configuration destination, object value, PropertyDescriptor property) + { + if (value is string valueString) + { + property.SetValue(destination, new ConfigurationSetting + { + Value = valueString, + Source = SettingSource.Default + }); + } + + if (value is int valueInt) + { + property.SetValue(destination, new ConfigurationSetting + { + Value = valueInt, + Source = SettingSource.Default + }); + } + + if (value is bool valueBool) + { + property.SetValue(destination, new ConfigurationSetting + { + Value = valueBool, + Source = SettingSource.Default + }); + } + + // Fall through, only primitive types are currently supported. + // Add more primitive types if needed here. + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs new file mode 100644 index 00000000..614503b3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ConfigSanitizer.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts.Enums; +using PowerArgs; +using System; +using System.Collections.Generic; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// Sanitizes a validated configuration by setting additional parameters or fixing default parameters if needed. + /// + public class ConfigSanitizer + { + private readonly IHashAlgorithmProvider hashAlgorithmProvider; + private readonly IFileSystemUtils fileSystemUtils; + private readonly IAssemblyConfig assemblyConfig; + + public ConfigSanitizer(IHashAlgorithmProvider hashAlgorithmProvider, IFileSystemUtils fileSystemUtils, IAssemblyConfig assemblyConfig) + { + this.hashAlgorithmProvider = hashAlgorithmProvider ?? throw new ArgumentNullException(nameof(hashAlgorithmProvider)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.assemblyConfig = assemblyConfig ?? throw new ArgumentNullException(nameof(assemblyConfig)); + } + + public Configuration SanitizeConfig(Configuration configuration) + { + if (configuration is null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + configuration.HashAlgorithm = GetHashAlgorithmName(configuration); + + // set ManifestDirPath after validation of DirectoryExist and DirectoryPathIsWritable, this wouldn't exist because it needs to be created by the tool. + configuration.ManifestDirPath = GetManifestDirPath(configuration.ManifestDirPath, configuration.BuildDropPath.Value, configuration.ManifestToolAction); + + // Set namespace value if provided in the assembly + configuration.NamespaceUriBase = GetNamespaceBaseUriFromAssembly(configuration); + + // Set default ManifestInfo for validation in case user doesn't provide a value. + configuration.ManifestInfo = GetDefaultManifestInfoForValidationAction(configuration); + + return configuration; + } + + private ConfigurationSetting> GetDefaultManifestInfoForValidationAction(Configuration configuration) + { + if (configuration.ManifestToolAction != ManifestToolActions.Validate + || (configuration.ManifestInfo.Value != null && configuration.ManifestInfo.Value.Count != 0)) + { + return configuration.ManifestInfo; + } + + var defaultManifestInfo = assemblyConfig.DefaultManifestInfoForValidationAction; + if (defaultManifestInfo == null && (configuration.ManifestInfo.Value == null || configuration.ManifestInfo.Value.Count == 0)) + { + throw new ValidationArgException($"Please provide a value for the ManifestInfo (-mi) parameter to validate the SBOM."); + } + + return new ConfigurationSetting> + { + Source = SettingSource.Default, + Value = new List() + { + defaultManifestInfo + } + }; + } + + private ConfigurationSetting GetHashAlgorithmName(Configuration configuration) + { + if (configuration.ManifestToolAction != ManifestToolActions.Validate) + { + return configuration.HashAlgorithm; + } + + // Convert to actual hash algorithm values. + var oldValue = configuration.HashAlgorithm; + var newValue = hashAlgorithmProvider.Get(oldValue?.Value?.Name); + + return new ConfigurationSetting + { + Source = oldValue.Source, + Value = newValue + }; + } + + private ConfigurationSetting GetNamespaceBaseUriFromAssembly(Configuration configuration) + { + // If assembly name is not defined returned the current value. + if (string.IsNullOrWhiteSpace(assemblyConfig.DefaultSBOMNamespaceBaseUri)) + { + return configuration.NamespaceUriBase; + } + + // If the user provides the parameter even when the assembly attribute is provided, + // show a warning on the console. + if (!string.IsNullOrWhiteSpace(configuration.NamespaceUriBase?.Value)) + { + Console.WriteLine(assemblyConfig.DefaultSBOMNamespaceBaseUriWarningMessage); + } + + return new ConfigurationSetting + { + Source = SettingSource.Default, + Value = assemblyConfig.DefaultSBOMNamespaceBaseUri + }; + } + + /// + /// Set ManifestDirPath if the value is null or empty to default value + /// + private ConfigurationSetting GetManifestDirPath(ConfigurationSetting manifestDirPathConfig, string buildDropPath, ManifestToolActions manifestToolAction) + { + if (string.IsNullOrEmpty(manifestDirPathConfig?.Value)) + { + return new ConfigurationSetting + { + Value = fileSystemUtils.JoinPaths(buildDropPath, Constants.ManifestFolder), + Source = SettingSource.Default + }; + } + + return new ConfigurationSetting + { + Value = EnsurePathEndsWithManifestFolderForGenerate(manifestDirPathConfig.Value, manifestToolAction), + Source = manifestDirPathConfig.Source + }; + } + + private string EnsurePathEndsWithManifestFolderForGenerate(string value, ManifestToolActions manifestToolAction) + { + if (manifestToolAction == ManifestToolActions.Generate) + { + // For generate action, add the _manifest folder at the end of the path + return fileSystemUtils.JoinPaths(value, Constants.ManifestFolder); + } + + return value; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ConfigurationBuilder.cs b/src/Microsoft.Sbom.Api/Config/ConfigurationBuilder.cs new file mode 100644 index 00000000..05a4586d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ConfigurationBuilder.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using PowerArgs; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config +{ + /// + /// Converts the command line arguments and config file parameters to objects. + /// Finally combines the two into one object. + /// + /// Throws an error if the same parameters are defined in both the config file and command line. + /// + /// The action args parameter. + public class ConfigurationBuilder + { + private readonly IMapper mapper; + private readonly ConfigFileParser configFileParser; + + public ConfigurationBuilder(IMapper mapper, ConfigFileParser configFileParser) + { + this.mapper = mapper; + this.configFileParser = configFileParser; + } + + public async Task GetConfiguration(T args) + { + Configuration commandLineArgs; + + // Set current action for the config validators and convert command line arguments to configuration + switch (args) + { + case ValidationArgs validationArgs: + validationArgs.ManifestToolAction = ManifestToolActions.Validate; + commandLineArgs = mapper.Map(validationArgs); + break; + case GenerationArgs generationArgs: + generationArgs.ManifestToolAction = ManifestToolActions.Generate; + commandLineArgs = mapper.Map(generationArgs); + break; + default: + throw new ValidationArgException($"Unsupported configuration type found {typeof(T)}"); + } + + // Read config file if present, or use default. + var configFromFile = commandLineArgs.ConfigFilePath != null ? + await configFileParser.ParseFromJsonFile(commandLineArgs.ConfigFilePath.Value) : + new ConfigFile(); + + // Convert config file arguments to configuration. + var configFileArgs = mapper.Map(configFromFile); + + // Combine both configs, include defaults. + return mapper.Map(commandLineArgs, configFileArgs); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Extensions/ConfigurationExtensions.cs b/src/Microsoft.Sbom.Api/Config/Extensions/ConfigurationExtensions.cs new file mode 100644 index 00000000..00d35d98 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Extensions/ConfigurationExtensions.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common.Config.Attributes; +using PowerArgs; +using System.Collections.Generic; +using System.Linq; + +namespace DropValidator.Api.Config.Extensions +{ + /// + /// Provides extension methods for an instance of . + /// + public static class ConfigurationExtensions + { + /// + /// Get the name and value of each IConfiguration property that is annotated with . + /// + /// + /// + private static IEnumerable<(string Name, object Value)> GetComponentDetectorArgs(this IConfiguration configuration) => typeof(IConfiguration) + .GetProperties() + .Where(prop => prop.GetCustomAttributes(typeof(ComponentDetectorArgumentAttribute), true).Any() + && prop.PropertyType.GetGenericTypeDefinition() == typeof(ConfigurationSetting<>) + && prop.GetValue(configuration) != null) + .Select(prop => (prop.Attr().ParameterName, prop.GetValue(configuration))); + + /// + /// Adds component detection arguments to the builder. + /// + /// + /// + /// + private static ComponentDetectionCliArgumentBuilder AddToCommandLineBuilder(this (string Name, object Value) arg, ComponentDetectionCliArgumentBuilder builder) => + !string.IsNullOrWhiteSpace(arg.Name) ? builder.AddArg(arg.Name, arg.Value.ToString()) : builder.ParseAndAddArgs(arg.Value.ToString()); + + /// + /// Adds command line arguments for all properties annotated with to the current CD CLI arguments builder and returns array of arguments. + /// + /// + /// + /// + public static string[] ToComponentDetectorCommandLineParams(this IConfiguration configuration, ComponentDetectionCliArgumentBuilder builder) + { + configuration + .GetComponentDetectorArgs() + .ForEach(arg => arg.AddToCommandLineBuilder(builder)); + return builder.Build(); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Config/ManifestToolCmdRunner.cs b/src/Microsoft.Sbom.Api/Config/ManifestToolCmdRunner.cs new file mode 100644 index 00000000..a5e52e0e --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ManifestToolCmdRunner.cs @@ -0,0 +1,121 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Ninject; +using PowerArgs; +using System; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Workflows; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config +{ + [ArgDescription("The manifest tool validates or generates a manifest for a build artifact.")] + [ArgExceptionBehavior(ArgExceptionPolicy.StandardExceptionHandling)] + [ArgProductName("ManifestTool.exe")] + public class ManifestToolCmdRunner + { + private readonly StandardKernel kernel; + + public ManifestToolCmdRunner() + { + IsFailed = false; + kernel = new StandardKernel(new Bindings()); + } + + public ManifestToolCmdRunner(StandardKernel kernel) + { + IsFailed = false; + this.kernel = kernel ?? throw new ArgumentNullException(nameof(kernel)); + } + + /// + /// Gets or sets a value indicating whether displays help info. + /// + [ArgShortcut("?")] + [ArgShortcut("h")] + [HelpHook] + [JsonIgnore] + [ArgDescription("Prints this help message")] + public bool Help { get; set; } + + /// + /// Gets a value indicating whether if set to true, indicates that there was a problem while parsing the configuration. + /// + [ArgIgnore] + public bool IsFailed { get; private set; } + + /// + /// Gets a value indicating whether if set to true, indicates that there was a problem accesing a path specified in the parameters. + /// + [ArgIgnore] + public bool IsAccessError { get; private set; } + + /// + /// Validate a build artifact using the manifest. Optionally also verify the signing certificate of the manfiest. + /// + /// + [ArgActionMethod, ArgDescription("Validate a build artifact using the manifest. " + + "Optionally also verify the signing certificate of the manfiest.")] + public async Task Validate(ValidationArgs validationArgs) + { + try + { + var mapper = kernel.Get(); + var configFileParser = kernel.Get(); + var configBuilder = new ConfigurationBuilder(mapper, configFileParser); + + kernel.Bind().ToConstant(await configBuilder.GetConfiguration(validationArgs)); + var result = await kernel.Get(nameof(DropValidatorWorkflow)).RunAsync(); + await kernel.Get().FinalizeAndLogTelemetryAsync(); + + IsFailed = !result; + } + catch (Exception e) + { + var message = e.InnerException != null ? e.InnerException.Message : e.Message; + Console.WriteLine($"Encountered error while running ManifestTool validation workflow. Error: {message}"); + IsFailed = true; + } + } + + /// + /// Generate a manifest.json and a bsi.json for all the files in the given build drop folder. + /// + [ArgActionMethod, ArgDescription("Generate a manifest.json and a bsi.json for all the files " + + "in the given build drop folder.")] + public async Task Generate(GenerationArgs generationArgs) + { + try + { + var mapper = kernel.Get(); + var configFileParser = kernel.Get(); + var configBuilder = new ConfigurationBuilder(mapper, configFileParser); + + kernel.Bind().ToConstant(await configBuilder.GetConfiguration(generationArgs)); + + var result = await kernel.Get(nameof(SBOMGenerationWorkflow)).RunAsync(); + await kernel.Get().FinalizeAndLogTelemetryAsync(); + IsFailed = !result; + } + catch (AccessDeniedValidationArgException e) + { + var message = e.InnerException != null ? e.InnerException.Message : e.Message; + Console.WriteLine($"Encountered error while running ManifestTool generation workflow. Error: {message}"); + IsFailed = true; + IsAccessError = true; + } + catch (Exception e) + { + var message = e.InnerException != null ? e.InnerException.Message : e.Message; + Console.WriteLine($"Encountered error while running ManifestTool generation workflow. Error: {message}"); + IsFailed = true; + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/ConfigValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/ConfigValidator.cs new file mode 100644 index 00000000..4e82b505 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/ConfigValidator.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using System; +using System.ComponentModel; + +namespace Microsoft.Sbom.Common.Config.Validators +{ + /// + /// Abstract class from which all validators must inherit. + /// This class only validates configuration properties that are of the type + /// + public abstract class ConfigValidator + { + private readonly IAssemblyConfig assemblyConfig; + + /// + /// This is the attribute that a property must have in order to be validated by this validator. + /// + private readonly Type supportedAttribute; + + /// + /// Gets or sets the current action being performed on the manifest tool. + /// + public ManifestToolActions CurrentAction { get; set; } + + protected ConfigValidator(Type supportedAttribute, IAssemblyConfig assemblyConfig) + { + this.supportedAttribute = supportedAttribute ?? throw new ArgumentNullException(nameof(supportedAttribute)); + this.assemblyConfig = assemblyConfig ?? throw new ArgumentNullException(nameof(assemblyConfig)); + } + + /// + /// Validates a given property, throws a if validation fails. + /// + /// The name of the property. + /// The value of the property. + /// The attributes assigned to this property. + public void Validate(string propertyName, object propertyValue, AttributeCollection attributeCollection) + { + if (string.IsNullOrEmpty(propertyName)) + { + throw new ArgumentException($"'{nameof(propertyName)}' cannot be null or empty", nameof(propertyName)); + } + + Attribute attribute = attributeCollection[supportedAttribute]; + if (attribute == null) + { + return; + } + + // If default value for namespace base uri is provided in the assembly info, skip value check requirements. + if (propertyName == Api.Utils.Constants.NamespaceUriBasePropertyName && !string.IsNullOrEmpty(assemblyConfig.DefaultSBOMNamespaceBaseUri)) + { + return; + } + + switch (propertyValue) + { + case null: + // If the value is null, let the implementing validator handle it. + ValidateInternal(propertyName, propertyValue, attribute); + break; + + case ConfigurationSetting configSetting: + ValidateInternal(propertyName, configSetting.Value, attribute); + break; + + case ConfigurationSetting configSettingInt: + ValidateInternal(propertyName, configSettingInt.Value, attribute); + break; + + case ConfigurationSetting configSettingString: + ValidateInternal(propertyName, configSettingString.Value, attribute); + break; + + default: + throw new ArgumentException($"'{propertyName}' must be of type '{typeof(ConfigurationSetting<>)}'"); + } + } + + public abstract void ValidateInternal(string paramName, object paramValue, Attribute attribute); + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/DirectoryExistsValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/DirectoryExistsValidator.cs new file mode 100644 index 00000000..6fe278e3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/DirectoryExistsValidator.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Validates if the directory exists with read permissions. + /// + public class DirectoryExistsValidator : ConfigValidator + { + private readonly IFileSystemUtils fileSystemUtils; + + public DirectoryExistsValidator(IFileSystemUtils fileSystemUtils, IAssemblyConfig assemblyConfig) + : base(typeof(DirectoryExistsAttribute), assemblyConfig) + { + this.fileSystemUtils = fileSystemUtils; + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (attribute is DirectoryExistsAttribute directoryExistsAttribute + && directoryExistsAttribute.ForAction.HasFlag(CurrentAction)) + { + if (paramValue != null + && paramValue is string value + && !string.IsNullOrEmpty(value)) + { + if (!fileSystemUtils.DirectoryExists(value)) + { + throw new ValidationArgException($"{paramName} directory not found for '{value}'"); + } + + if (directoryExistsAttribute.HasReadPermissions && !fileSystemUtils.DirectoryHasReadPermissions(value)) + { + throw new ValidationArgException($"{paramName} directory does not have read permissions '{value}'"); + } + + if (directoryExistsAttribute.HasWritePermissions && !fileSystemUtils.DirectoryHasWritePermissions(value)) + { + throw new ValidationArgException($"{paramName} directory does not have write permissions '{value}'"); + } + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/DirectoryPathIsWritableValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/DirectoryPathIsWritableValidator.cs new file mode 100644 index 00000000..5f44a6dc --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/DirectoryPathIsWritableValidator.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Verify that the directory path is writable. + /// + public class DirectoryPathIsWritableValidator : ConfigValidator + { + private readonly IFileSystemUtils fileSystemUtils; + + public DirectoryPathIsWritableValidator(IFileSystemUtils fileSystemUtils, IAssemblyConfig assemblyConfig) + : base(typeof(DirectoryPathIsWritableAttribute), assemblyConfig) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (paramValue != null && paramValue is string value && !string.IsNullOrEmpty(value)) + { + // check if directory exist + if (!fileSystemUtils.DirectoryExists(value)) + { + throw new ValidationArgException($"{paramName} directory not found for '{value}'"); + } + + // check directory for write permission + if (!fileSystemUtils.DirectoryHasWritePermissions(value)) + { + throw new AccessDeniedValidationArgException($"{paramName} directory does not have write permissions '{value}'"); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/FileExistsValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/FileExistsValidator.cs new file mode 100644 index 00000000..55fc8594 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/FileExistsValidator.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Validates if the file exists. + /// + public class FileExistsValidator : ConfigValidator + { + private readonly IFileSystemUtils fileSystemUtils; + + public FileExistsValidator(IFileSystemUtils fileSystemUtils, IAssemblyConfig assemblyConfig) + : base(typeof(FileExistsAttribute), assemblyConfig) + { + this.fileSystemUtils = fileSystemUtils; + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (paramValue != null && paramValue is string value && !string.IsNullOrEmpty(value)) + { + if (!fileSystemUtils.FileExists(value)) + { + throw new ValidationArgException($"{paramName} file not found for '{value}'"); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/FilePathIsWritableValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/FilePathIsWritableValidator.cs new file mode 100644 index 00000000..dd658ffa --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/FilePathIsWritableValidator.cs @@ -0,0 +1,54 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Verify that the filepath is writable. + /// + public class FilePathIsWritableValidator : ConfigValidator + { + private readonly IFileSystemUtils fileSystemUtils; + + public FilePathIsWritableValidator(IFileSystemUtils fileSystemUtils, IAssemblyConfig assemblyConfig) + : base(typeof(FilePathIsWritableAttribute), assemblyConfig) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (paramValue != null && paramValue is string value && !string.IsNullOrEmpty(value)) + { + string directoryPath; + try + { + directoryPath = fileSystemUtils.GetDirectoryName(value); + } + catch (Exception e) + { + throw new ValidationArgException($"Unable to get directory for '{value}': {e.Message}"); + } + + // check if directory exist + if (!fileSystemUtils.DirectoryExists(directoryPath)) + { + throw new ValidationArgException($"{paramName} directory not found for '{value}'"); + } + + // check directory for write permission + if (!fileSystemUtils.DirectoryHasWritePermissions(directoryPath)) + { + throw new ValidationArgException($"{paramName} directory does not have write permissions '{directoryPath}'"); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/IntRangeValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/IntRangeValidator.cs new file mode 100644 index 00000000..a797ee01 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/IntRangeValidator.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Validates if the integer property is in the provided inclusive range. + /// + public class IntRangeValidator : ConfigValidator + { + public IntRangeValidator(IAssemblyConfig assemblyConfig) + : base(typeof(IntRangeAttribute), assemblyConfig) + { + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (paramValue != null && paramValue is int value) + { + IntRangeAttribute intRangeAttribute = attribute as IntRangeAttribute; + + if (value < intRangeAttribute.MinRange || value > intRangeAttribute.MaxRange) + { + throw new ValidationArgException($"The value for {paramName} should be equal to or between {intRangeAttribute.MinRange} and {intRangeAttribute.MaxRange}"); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/UriValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/UriValidator.cs new file mode 100644 index 00000000..b2141e2d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/UriValidator.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Common.Config.Validators; +using PowerArgs; +using System; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Validates if a value is a valid URI. + /// + public class UriValidator : ConfigValidator + { + public UriValidator(IAssemblyConfig assemblyConfig) + : base(typeof(ValidUriAttribute), assemblyConfig) + { + } + + public UriValidator(Type supportedAttribute, IAssemblyConfig assemblyConfig) + : base(supportedAttribute, assemblyConfig) + { + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (attribute != null && attribute is ValidUriAttribute validUriAttribute && validUriAttribute.ForAction.HasFlag(CurrentAction)) + { + if (paramValue is string value && !string.IsNullOrEmpty(value) && !Uri.IsWellFormedUriString(value, validUriAttribute.UriKind)) + { + throw new ValidationArgException($"The value of {paramName} must be a valid URI."); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/Validators/ValueRequiredValidator.cs b/src/Microsoft.Sbom.Api/Config/Validators/ValueRequiredValidator.cs new file mode 100644 index 00000000..2d070ea3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/Validators/ValueRequiredValidator.cs @@ -0,0 +1,40 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Common.Config.Attributes; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Config.Validators +{ + /// + /// Verifies that the value is not null + /// + public class ValueRequiredValidator : ConfigValidator + { + public ValueRequiredValidator(IAssemblyConfig assemblyConfig) + : base(typeof(ValueRequiredAttribute), assemblyConfig) + { + } + + public ValueRequiredValidator(Type type, IAssemblyConfig assemblyConfig) + : base(type, assemblyConfig) + { + } + + public override void ValidateInternal(string paramName, object paramValue, Attribute attribute) + { + if (attribute is ValueRequiredAttribute valueRequiredAttribute && !valueRequiredAttribute.ForAction.HasFlag(CurrentAction)) + { + return; + } + + if (paramValue == null || (paramValue is string value && string.IsNullOrEmpty(value))) + { + throw new ValidationArgException($"The value of {paramName} can't be null or empty."); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/BoolConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/BoolConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..b89963ad --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/BoolConfigurationSettingAddingConverter.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts a nullable bool member to a ConfigurationSetting decorated string member + /// + internal class BoolConfigurationSettingAddingConverter : IValueConverter> + { + private readonly SettingSource settingSource; + + public BoolConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(bool sourceMember, ResolutionContext context) + { + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..3db5510d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/HashAlgorithmNameConfigurationSettingAddingConverter.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts.Enums; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts an LogEventLevel member to a ConfigurationSetting decorated string member + /// + internal class HashAlgorithmNameConfigurationSettingAddingConverter : IValueConverter> + { + private SettingSource settingSource; + + public HashAlgorithmNameConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(AlgorithmName sourceMember, ResolutionContext context) + { + if (sourceMember == null) + { + settingSource = SettingSource.Default; + } + + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember ?? Constants.DefaultHashAlgorithmName + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/IntConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/IntConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..20f5a762 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/IntConfigurationSettingAddingConverter.cs @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts the int property to a ConfigurationSetting decorated member + /// Int.MinValue is considered invalid. + /// + internal class IntConfigurationSettingAddingConverter : IValueConverter>, IValueConverter> + { + private readonly SettingSource settingSource; + + public IntConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(int? sourceMember, ResolutionContext context) + { + if (sourceMember == null) + { + return null; + } + + return Convert(sourceMember.Value, context); + } + + public ConfigurationSetting Convert(int sourceMember, ResolutionContext context) + { + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..93e62541 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/LogEventLevelConfigurationSettingAddingConverter.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Serilog.Events; +using Constants = Microsoft.Sbom.Common.Constants; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts an LogEventLevel member to a ConfigurationSetting decorated string member + /// + internal class LogEventLevelConfigurationSettingAddingConverter : IValueConverter> + { + private SettingSource settingSource; + + public LogEventLevelConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(LogEventLevel? sourceMember, ResolutionContext context) + { + if (sourceMember == null) + { + settingSource = SettingSource.Default; + } + + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember ?? Constants.DefaultLogLevel + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/ManifestInfoConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/ManifestInfoConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..50815100 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/ManifestInfoConfigurationSettingAddingConverter.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Extensions.Entities; +using System.Collections.Generic; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts an ManifestInfo member to a ConfigurationSetting decorated string member + /// + internal class ManifestInfoConfigurationSettingAddingConverter : IValueConverter, ConfigurationSetting>> + { + private SettingSource settingSource; + + public ManifestInfoConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting> Convert(IList sourceMember, ResolutionContext context) + { + if (sourceMember == null) + { + settingSource = SettingSource.Default; + sourceMember = null; + } + + return new ConfigurationSetting> + { + Source = settingSource, + Value = sourceMember + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/NullableBoolConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/NullableBoolConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..035eaef1 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/NullableBoolConfigurationSettingAddingConverter.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts a nullable bool member to a ConfigurationSetting decorated string member + /// + internal class NullableBoolConfigurationSettingAddingConverter : IValueConverter> + { + private readonly SettingSource settingSource; + + public NullableBoolConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(bool? sourceMember, ResolutionContext context) + { + if (sourceMember == null) + { + return null; + } + + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember ?? false + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Config/ValueConverters/StringConfigurationSettingAddingConverter.cs b/src/Microsoft.Sbom.Api/Config/ValueConverters/StringConfigurationSettingAddingConverter.cs new file mode 100644 index 00000000..5eaf22cb --- /dev/null +++ b/src/Microsoft.Sbom.Api/Config/ValueConverters/StringConfigurationSettingAddingConverter.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Config.ValueConverters +{ + /// + /// Converts a string member to a ConfigurationSetting decorated string member + /// + internal class StringConfigurationSettingAddingConverter : IValueConverter> + { + private readonly SettingSource settingSource; + + public StringConfigurationSettingAddingConverter(SettingSource settingSource) + { + this.settingSource = settingSource; + } + + public ConfigurationSetting Convert(string sourceMember, ResolutionContext context) + { + if (string.IsNullOrEmpty(sourceMember)) + { + return null; + } + + return new ConfigurationSetting + { + Source = settingSource, + Value = sourceMember + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/ConfigurationProfile.cs b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs new file mode 100644 index 00000000..2a5cc905 --- /dev/null +++ b/src/Microsoft.Sbom.Api/ConfigurationProfile.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Extensions.Entities; +using Serilog.Events; +using System; +using System.Collections.Generic; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Api.Config.ValueConverters; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Api.Config; + +namespace Microsoft.Sbom.Api +{ + /// + /// Provides a named profile for the automapper that + /// generates a mapping for all the classes that map to a configuration object. + /// + public class ConfigurationProfile : Profile + { + public ConfigurationProfile() + { + // Create config for the validation args, ignoring other action members + CreateMap() +#pragma warning disable CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' + .ForMember(nameof(Configuration.ManifestPath), o => o.Ignore()) +#pragma warning restore CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' + .ForMember(nameof(Configuration.PackageName), o => o.Ignore()) + .ForMember(nameof(Configuration.PackageVersion), o => o.Ignore()) + .ForMember(nameof(Configuration.BuildListFile), o => o.Ignore()) + .ForMember(nameof(Configuration.ExternalDocumentReferenceListFile), o => o.Ignore()) + .ForMember(nameof(Configuration.BuildComponentPath), o => o.Ignore()) + .ForMember(nameof(Configuration.PackagesList), o => o.Ignore()) + .ForMember(nameof(Configuration.FilesList), o => o.Ignore()) + .ForMember(nameof(Configuration.DockerImagesToScan), o => o.Ignore()) + .ForMember(nameof(Configuration.AdditionalComponentDetectorArgs), o => o.Ignore()) + .ForMember(nameof(Configuration.GenerationTimestamp), o => o.Ignore()) + .ForMember(nameof(Configuration.NamespaceUriUniquePart), o => o.Ignore()) + .ForMember(nameof(Configuration.NamespaceUriBase), o => o.Ignore()); + + // Create config for the generation args, ignoring other action members + CreateMap() +#pragma warning disable CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' + .ForMember(nameof(Configuration.ManifestPath), o => o.Ignore()) +#pragma warning restore CS0618 // 'Configuration.ManifestPath' is obsolete: 'This field is not provided by the user or configFile, set by system' + .ForMember(nameof(Configuration.OutputPath), o => o.Ignore()) + .ForMember(nameof(Configuration.HashAlgorithm), o => o.Ignore()) + .ForMember(nameof(Configuration.RootPathFilter), o => o.Ignore()) + .ForMember(nameof(Configuration.CatalogFilePath), o => o.Ignore()) + .ForMember(nameof(Configuration.ValidateSignature), o => o.Ignore()) + .ForMember(nameof(Configuration.PackagesList), o => o.Ignore()) + .ForMember(nameof(Configuration.FilesList), o => o.Ignore()) + .ForMember(nameof(Configuration.IgnoreMissing), o => o.Ignore()); + + // Create config for the config json file to configuration. + CreateMap() + .ForMember(nameof(Configuration.PackagesList), o => o.Ignore()) + .ForMember(nameof(Configuration.FilesList), o => o.Ignore()); + + // Add maps to combine both config json and argument args, + // validate each settings using the config validator. + CreateMap() + .AfterMap() + .ForAllMembers(dest => dest.Condition((src, dest, srcObj, dstObj) => + { + // If the property is set in both source and destination (config and cmdline), + // this is a failure case, unless one of the property is a default value, in which + // case the non default value wins. + if (srcObj != null && dstObj != null + && srcObj is ISettingSourceable srcWithSource + && dstObj is ISettingSourceable dstWithSource) + { + if (srcWithSource.Source != SettingSource.Default && dstWithSource.Source != SettingSource.Default) + { + throw new Exception($"Duplicate keys found in config file and command line parameters."); + } + + return dstWithSource.Source == SettingSource.Default; + } + + // If source property is not null, use source, or else use destination value. + return srcObj != null; + })); + + // Set value converters for each type of object. + ForAllPropertyMaps( + p => p.SourceType == typeof(string), + (c, memberOptions) => memberOptions.ConvertUsing(new StringConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(bool?), + (c, memberOptions) => memberOptions.ConvertUsing(new NullableBoolConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(bool), + (c, memberOptions) => memberOptions.ConvertUsing(new BoolConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(int), + (c, memberOptions) => memberOptions.ConvertUsing(new IntConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(int?), + (c, memberOptions) => memberOptions.ConvertUsing(new IntConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(LogEventLevel?), + (c, memberOptions) => memberOptions.ConvertUsing(new LogEventLevelConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(IList), + (c, memberOptions) => memberOptions.ConvertUsing(new ManifestInfoConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + ForAllPropertyMaps( + p => p.SourceType == typeof(AlgorithmName), + (c, memberOptions) => memberOptions.ConvertUsing(new HashAlgorithmNameConfigurationSettingAddingConverter(GetSettingSourceFor(c.SourceMember.ReflectedType)))); + } + + // Based on the type of source, return the settings type. + private SettingSource GetSettingSourceFor(Type sourceType) + { + switch (sourceType) + { + case Type _ when sourceType.IsSubclassOf(typeof(CommonArgs)): + case Type _ when sourceType == typeof(CommonArgs): + return SettingSource.CommandLine; + case Type _ when sourceType == typeof(ConfigFile): + return SettingSource.JsonConfig; + default: return SettingSource.Default; + } + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Converters/ComponentToExternalReferenceInfoConverter.cs b/src/Microsoft.Sbom.Api/Converters/ComponentToExternalReferenceInfoConverter.cs new file mode 100644 index 00000000..20560af3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Converters/ComponentToExternalReferenceInfoConverter.cs @@ -0,0 +1,86 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Entities; +using Serilog; +using System; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Converters +{ + /// + /// Converts ScannedComponent objects of SbomComponent type to ExternalDocumentReferenceInfo + /// + public class ComponentToExternalReferenceInfoConverter + { + private readonly ILogger log; + + public ComponentToExternalReferenceInfoConverter(ILogger log) + { + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public (ChannelReader output, ChannelReader errors) Convert(ChannelReader componentReader) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (ScannedComponent scannedComponent in componentReader.ReadAllAsync()) + { + try + { + var document = ConvertComponentToExternalReference(scannedComponent); + await output.Writer.WriteAsync(document); + } + catch (Exception e) + { + log.Debug($"Encountered an error while converting SBOM component {scannedComponent.Component.Id} to external reference: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = Entities.ErrorType.PackageError, + Path = scannedComponent.LocationsFoundAt?.FirstOrDefault() + }); + } + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private ExternalDocumentReferenceInfo ConvertComponentToExternalReference(ScannedComponent component) + { + if (!(component.Component is SpdxComponent)) + { + throw new ArgumentException($"{nameof(component.Component)} is not an SpdxComponent"); + } + + var sbomComponent = (SpdxComponent)component.Component; + + if (sbomComponent.DocumentNamespace is null) + { + throw new ArgumentException($"{nameof(sbomComponent)} should have {nameof(sbomComponent.DocumentNamespace)}"); + } + + return new ExternalDocumentReferenceInfo + { + ExternalDocumentName = sbomComponent.Name, + Checksum = new[] { new Checksum { Algorithm = AlgorithmName.SHA1, ChecksumValue = sbomComponent.Checksum } }, + Path = sbomComponent.Path, + DocumentNamespace = sbomComponent.DocumentNamespace.ToString(), + DescribedElementID = sbomComponent.RootElementId + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Converters/DropValidatorManifestPathConverter.cs b/src/Microsoft.Sbom.Api/Converters/DropValidatorManifestPathConverter.cs new file mode 100644 index 00000000..93c4fc02 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Converters/DropValidatorManifestPathConverter.cs @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using System; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Convertors +{ + /// + /// Converts a regular file path to a relative file path in the format the + /// DropValidator expects. The expected format looks like this: + /// + /// Root : C:\dropRoot + /// Absolute path : C:\dropRoot\folder1\file1.txt + /// Relative path : folder1\file1.txt + /// DropValidator Format : /folder1/file1.txt + /// + /// Throws a if the file is outside the root folder. + /// + public class DropValidatorManifestPathConverter : IManifestPathConverter + { + private readonly IConfiguration configuration; + private readonly IOSUtils osUtils; + private readonly IFileSystemUtils fileSystemUtils; + private readonly IFileSystemUtilsExtension fileSystemUtilsExtension; + + public DropValidatorManifestPathConverter(IConfiguration configuration, IOSUtils osUtils, IFileSystemUtils fileSystemUtils, IFileSystemUtilsExtension fileSystemUtilsExtension) + { + this.configuration = configuration; + this.osUtils = osUtils; + this.fileSystemUtils = fileSystemUtils; + this.fileSystemUtilsExtension = fileSystemUtilsExtension; + } + + public (string, bool) Convert(string path) + { + //relativeTo + string buildDropPath = configuration.BuildDropPath.Value; + bool isOutsideDropPath = false; + if (path == null) + { + throw new ArgumentNullException(nameof(path)); + } + + if (!fileSystemUtilsExtension.IsTargetPathInSource(path, buildDropPath)) + { + isOutsideDropPath = true; + + // Allow spdx files to be outside the root path, all externalDocumentReference must be in the file array regardless of where they are located. + // More details are in this spec: https://github.com/spdx/spdx-spec/issues/571 + if (!path.EndsWith(Constants.SPDXFileExtension, osUtils.GetFileSystemStringComparisonType())) + { + throw new InvalidPathException($"The file at {path} is outside the root path {buildDropPath}."); + } + } + + string relativePath = fileSystemUtils.GetRelativePath(buildDropPath, path); + string formattedRelativePath = $"/{relativePath.Replace("\\", "/")}"; + + return (formattedRelativePath, isOutsideDropPath); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Converters/ExternalReferenceInfoToPathConverter.cs b/src/Microsoft.Sbom.Api/Converters/ExternalReferenceInfoToPathConverter.cs new file mode 100644 index 00000000..58a15e13 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Converters/ExternalReferenceInfoToPathConverter.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Entities; +using Serilog; +using System; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Converters +{ + /// + /// Converts ExternalDocumentReferenceInfo objects to their path as string + /// + public class ExternalReferenceInfoToPathConverter + { + private readonly ILogger log; + + public ExternalReferenceInfoToPathConverter(ILogger log) + { + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public (ChannelReader output, ChannelReader errors) Convert(ChannelReader externalDocumentRefReader) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (ExternalDocumentReferenceInfo externalDocumentRef in externalDocumentRefReader.ReadAllAsync()) + { + try + { + var path = externalDocumentRef.Path; + + if (path == null) + { + log.Debug($"Encountered an error while converting external reference {externalDocumentRef.ExternalDocumentName} for null path."); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + + // on the exception that Path does not exist, use DocumentName for uniqueness + Path = externalDocumentRef.ExternalDocumentName + }); + } + else + { + await output.Writer.WriteAsync(path); + } + } + catch (Exception e) + { + log.Debug($"Encountered an error while converting external reference {externalDocumentRef.ExternalDocumentName} to path: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = externalDocumentRef.Path + }); + } + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Converters/IManifestPathConverter.cs b/src/Microsoft.Sbom.Api/Converters/IManifestPathConverter.cs new file mode 100644 index 00000000..88dccbfd --- /dev/null +++ b/src/Microsoft.Sbom.Api/Converters/IManifestPathConverter.cs @@ -0,0 +1,18 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Convertors +{ + public interface IManifestPathConverter + { + /// + /// Convert a file path from a relative path to a path format + /// that the manifest implements. + /// + /// The relative path of the file. + /// The file path in the manifest format and boolean for if the path is outside the BuildDropPath. + (string, bool) Convert(string path); + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/ErrorType.cs b/src/Microsoft.Sbom.Api/Entities/ErrorType.cs new file mode 100644 index 00000000..60b94919 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/ErrorType.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Entities +{ + /// + /// Type of validation error for a given file. + /// + public enum ErrorType + { + [EnumMember(Value = "None")] + None = 0, + + [EnumMember(Value = "Invalid Hash")] + InvalidHash = 1, + + [EnumMember(Value = "Additional File")] + AdditionalFile = 2, + + [EnumMember(Value = "Missing File")] + MissingFile = 3, + + [EnumMember(Value = "Filtered root path")] + FilteredRootPath = 4, + + [EnumMember(Value = "Manifest folder")] + ManifestFolder = 5, + + [EnumMember(Value = "Other")] + Other = 6, + + [EnumMember(Value = "Package error")] + PackageError = 7, + + [EnumMember(Value = "Json serialization error")] + JsonSerializationError = 8, + + [EnumMember(Value = "Unsupported hash algorithm")] + UnsupportedHashAlgorithm = 9 + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/ExitCode.cs b/src/Microsoft.Sbom.Api/Entities/ExitCode.cs new file mode 100644 index 00000000..268e57fd --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/ExitCode.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api +{ + /// + /// Defines the exit code returned by the ManifestTool executable + /// + public enum ExitCode + { + Success = 0, + GeneralError = 1, + WriteAccessError = 2 + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/FileValidationResult.cs b/src/Microsoft.Sbom.Api/Entities/FileValidationResult.cs new file mode 100644 index 00000000..3ed9573c --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/FileValidationResult.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Entities; +using Microsoft.Sbom.Contracts.Enums; +using Newtonsoft.Json; +using Newtonsoft.Json.Converters; +using EntityErrorType = Microsoft.Sbom.Contracts.Enums.ErrorType; + +namespace Microsoft.Sbom.Api.Entities +{ + public class FileValidationResult + { + /// + /// Gets or sets the relative path of the node. + /// + public string Path { get; set; } + + /// + /// Gets or sets the type of error if any. + /// + [JsonConverter(typeof(StringEnumConverter))] + public ErrorType ErrorType { get; set; } + + // TODO: Deprecate FileValidationResult to use EntityError + public EntityError ToEntityError() + { + EntityErrorType errorType = EntityErrorType.Other; + EntityType entityType = EntityType.Unknown; + Entity entity = null; + + switch (ErrorType) + { + case ErrorType.AdditionalFile: + case ErrorType.FilteredRootPath: + case ErrorType.ManifestFolder: + case ErrorType.MissingFile: + errorType = EntityErrorType.FileError; + entityType = EntityType.File; + break; + case ErrorType.InvalidHash: + case ErrorType.UnsupportedHashAlgorithm: + errorType = EntityErrorType.HashingError; + break; + case ErrorType.JsonSerializationError: + errorType = EntityErrorType.JsonSerializationError; + break; + case ErrorType.None: + errorType = EntityErrorType.None; + break; + case ErrorType.PackageError: + errorType = EntityErrorType.PackageError; + entityType = EntityType.Package; + break; + case ErrorType.Other: + errorType = EntityErrorType.Other; + break; + } + + switch (entityType) + { + case EntityType.Unknown: + case EntityType.File: + entity = new FileEntity(Path); + break; + case EntityType.Package: + entity = new PackageEntity(Path, null, Path, null); + break; + } + + return new EntityError() + { + ErrorType = errorType, + Entity = entity + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/JsonDocWithSerializer.cs b/src/Microsoft.Sbom.Api/Entities/JsonDocWithSerializer.cs new file mode 100644 index 00000000..afc1ee2a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/JsonDocWithSerializer.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Entities +{ + /// + /// Represents a JsonDocument that needs to be written to a serializer. This is struct as its passed along + /// multiple times in functions, and we need it to be passed by value. + /// + public struct JsonDocWithSerializer + { + private JsonDocument doc; + private IManifestToolJsonSerializer serializer; + + public JsonDocument Document { get => doc; set => doc = value; } + + public IManifestToolJsonSerializer Serializer { get => serializer; set => serializer = value; } + + public JsonDocWithSerializer(JsonDocument doc, IManifestToolJsonSerializer serializer) + { + this.doc = doc; + this.serializer = serializer; + } + + public override bool Equals(object obj) + { + return obj is JsonDocWithSerializer other && + EqualityComparer.Default.Equals(doc, other.doc) && + EqualityComparer.Default.Equals(serializer, other.serializer); + } + + public override int GetHashCode() + { + return HashCode.Combine(doc, serializer); + } + + public void Deconstruct(out JsonDocument doc, out IManifestToolJsonSerializer serializer) + { + doc = this.doc; + serializer = this.serializer; + } + + public static implicit operator (JsonDocument doc, IManifestToolJsonSerializer serializer)(JsonDocWithSerializer value) + { + return (value.doc, value.serializer); + } + + public static implicit operator JsonDocWithSerializer((JsonDocument doc, IManifestToolJsonSerializer serializer) value) + { + return new JsonDocWithSerializer(value.doc, value.serializer); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/ErrorContainer.cs b/src/Microsoft.Sbom.Api/Entities/output/ErrorContainer.cs new file mode 100644 index 00000000..1439da73 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/ErrorContainer.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Entities.Output +{ + /// + /// Error container for validation errors. + /// + /// + public class ErrorContainer + { + /// + /// Gets or sets the total count of errors. + /// + public int Count { get; set; } + + /// + /// Gets or sets the list of errors. + /// + public IList Errors { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/Result.cs b/src/Microsoft.Sbom.Api/Entities/output/Result.cs new file mode 100644 index 00000000..21de6229 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/Result.cs @@ -0,0 +1,19 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Entities.Output +{ + /// + /// The result of the validation. + /// + public enum Result + { + [EnumMember(Value = "Success")] + Success = 0, + + [EnumMember(Value = "Failure")] + Failure = 1 + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/Summary.cs b/src/Microsoft.Sbom.Api/Entities/output/Summary.cs new file mode 100644 index 00000000..40fd453b --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/Summary.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Entities.Output +{ + /// + /// The summary section specifies telemetry and other metadata about the validation. + /// + public class Summary + { + /// + /// Gets or sets the total time it took to run the validation. + /// + public double TotalExecutionTimeInSeconds { get; set; } + + /// + /// Gets or sets a representing the validation telemetry. + /// + public ValidationTelemetry ValidationTelemetery { get; set; } + + /// + /// Gets or sets a list of representing each input parameter used + /// in the validation. + /// + public IConfiguration Parameters { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/ValidationResult.cs b/src/Microsoft.Sbom.Api/Entities/output/ValidationResult.cs new file mode 100644 index 00000000..75d35a17 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/ValidationResult.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Entities.Output +{ + /// + /// The final result JSON that is serialized to the output location. + /// + public class ValidationResult + { + /// + /// Gets or sets the of the validation. + /// + public Result Result { get; set; } + + /// + /// Gets or sets a list of s. + /// + public ErrorContainer ValidationErrors { get; set; } + + /// + /// Gets or sets metadata and telemetry for this validation. + /// + public Summary Summary { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs b/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs new file mode 100644 index 00000000..044fc2bf --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/ValidationResultGenerator.cs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Entities.Output +{ + /// + /// Generates a object. + /// + public class ValidationResultGenerator + { + private int successCount; + private TimeSpan duration; + private readonly IConfiguration configuration; + private readonly ManifestData manifestData; + + public IList NodeValidationResults { get; set; } + + public ValidationResultGenerator(IConfiguration configuration, ManifestData manifestData) + { + this.configuration = configuration; + this.manifestData = manifestData; + } + + /// + /// Sets the count of successful results. + /// Retuns the for chaining. + /// + /// + /// + public ValidationResultGenerator WithSuccessCount(int successCount) + { + this.successCount = successCount; + return this; + } + + /// + /// Sets the total duration the validator ran.. + /// Retuns the for chaining. + /// + /// + /// + public ValidationResultGenerator WithTotalDuration(TimeSpan duration) + { + this.duration = duration; + return this; + } + + /// + /// Sets the failed validaion results. + /// Retuns the for chaining. + /// + /// + /// + public ValidationResultGenerator WithValidationResults(IList nodeValidationResults) + { + NodeValidationResults = nodeValidationResults ?? new List(); + return this; + } + + /// + /// Finalizes the validation generation and returns a new object. + /// + /// + public ValidationResult Build() + { + List validationErrors; + List skippedErrors; + if (configuration.IgnoreMissing.Value) + { + validationErrors = NodeValidationResults.Where(n => n.ErrorType != ErrorType.FilteredRootPath && n.ErrorType != ErrorType.ManifestFolder && n.ErrorType != ErrorType.MissingFile).ToList(); + skippedErrors = NodeValidationResults.Where(n => n.ErrorType == ErrorType.FilteredRootPath || n.ErrorType == ErrorType.ManifestFolder || n.ErrorType == ErrorType.MissingFile).ToList(); + } + else + { + validationErrors = NodeValidationResults.Where(n => n.ErrorType != ErrorType.FilteredRootPath && n.ErrorType != ErrorType.ManifestFolder).ToList(); + skippedErrors = NodeValidationResults.Where(n => n.ErrorType == ErrorType.FilteredRootPath || n.ErrorType == ErrorType.ManifestFolder).ToList(); + } + + return new ValidationResult + { + Result = validationErrors.Count == 0 ? Result.Success : Result.Failure, + ValidationErrors = new ErrorContainer + { + Count = validationErrors.Count, + Errors = validationErrors + }, + Summary = new Summary + { + TotalExecutionTimeInSeconds = duration.TotalSeconds, + ValidationTelemetery = new ValidationTelemetry + { + FilesSuccessfulCount = successCount, + FilesValidatedCount = NodeValidationResults.Count + successCount, + FilesFailedCount = validationErrors.Count, + FilesSkippedCount = skippedErrors.Count, + TotalFilesInManifest = manifestData.Count + }, + Parameters = configuration + } + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Entities/output/ValidationTelemetry.cs b/src/Microsoft.Sbom.Api/Entities/output/ValidationTelemetry.cs new file mode 100644 index 00000000..412ac3ca --- /dev/null +++ b/src/Microsoft.Sbom.Api/Entities/output/ValidationTelemetry.cs @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Entities.Output +{ + public class ValidationTelemetry + { + /// + /// Gets or sets count of files that were successful. + /// + public int FilesSuccessfulCount { get; set; } + + /// + /// Gets or sets total files in the manifest file. + /// + public int TotalFilesInManifest { get; set; } + + /// + /// Gets or sets count of files that were validated. + /// + public int FilesValidatedCount { get; set; } + + /// + /// Gets or sets count of files that were skipped. + /// + public int FilesSkippedCount { get; set; } + + /// + /// Gets or sets count of files that failed validation. + /// + public int FilesFailedCount { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/AccessDeniedValidationArgException.cs b/src/Microsoft.Sbom.Api/Exceptions/AccessDeniedValidationArgException.cs new file mode 100644 index 00000000..b1998211 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/AccessDeniedValidationArgException.cs @@ -0,0 +1,25 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using PowerArgs; +using System; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Exception during argument validation used to indicate when we don't have access to a path passed as argument + /// + [Serializable] + public class AccessDeniedValidationArgException : ValidationArgException + { + public AccessDeniedValidationArgException(string message) + : base(message) + { + } + + public AccessDeniedValidationArgException(string message, Exception innerException) + : base(message, innerException) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/ComponentDetectorException.cs b/src/Microsoft.Sbom.Api/Exceptions/ComponentDetectorException.cs new file mode 100644 index 00000000..ca0bf692 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/ComponentDetectorException.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when we encounter a problem while running the component detector. + /// + [Serializable] + public class ComponentDetectorException : Exception + { + public ComponentDetectorException() + { + } + + public ComponentDetectorException(string message) + : base(message) + { + } + + public ComponentDetectorException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected ComponentDetectorException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/ConfigurationException.cs b/src/Microsoft.Sbom.Api/Exceptions/ConfigurationException.cs new file mode 100644 index 00000000..89ea3f1d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/ConfigurationException.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when there is a problem in parsing the . + /// + [Serializable] + public class ConfigurationException : Exception + { + public ConfigurationException() + { + } + + public ConfigurationException(string message) + : base(message) + { + } + + public ConfigurationException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected ConfigurationException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Exceptions/HashGenerationException.cs b/src/Microsoft.Sbom.Api/Exceptions/HashGenerationException.cs new file mode 100644 index 00000000..cc76c83d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/HashGenerationException.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when the generated hash is invalid. + /// + [Serializable] + public class HashGenerationException : Exception + { + public HashGenerationException() + { + } + + public HashGenerationException(string message) + : base(message) + { + } + + public HashGenerationException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected HashGenerationException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/InvalidConverterException.cs b/src/Microsoft.Sbom.Api/Exceptions/InvalidConverterException.cs new file mode 100644 index 00000000..f5d7ea78 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/InvalidConverterException.cs @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when the instantiated + /// cannot convert the + /// + /// + /// Thrown out of public classes implementing IPackageInfoConverter so it must also be public. + /// + [Serializable] + public class InvalidConverterException : Exception + { + public InvalidConverterException() + { + } + + public InvalidConverterException(string message) + : base(message) + { + } + + public InvalidConverterException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected InvalidConverterException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/InvalidPathException.cs b/src/Microsoft.Sbom.Api/Exceptions/InvalidPathException.cs new file mode 100644 index 00000000..691e8c61 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/InvalidPathException.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when the file path is invalid or inaccessible. + /// + [Serializable] + public class InvalidPathException : Exception + { + public InvalidPathException() + { + } + + public InvalidPathException(string message) + : base(message) + { + } + + public InvalidPathException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected InvalidPathException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Exceptions/ManifestToolSerializerException.cs b/src/Microsoft.Sbom.Api/Exceptions/ManifestToolSerializerException.cs new file mode 100644 index 00000000..032d8019 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/ManifestToolSerializerException.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when the manifest tool is unable to serialize the SBOM component. + /// + [Serializable] + public class ManifestToolSerializerException : Exception + { + public ManifestToolSerializerException() { } + + public ManifestToolSerializerException(string message) + : base(message) { } + + public ManifestToolSerializerException(string message, Exception inner) + : base(message, inner) { } + + protected ManifestToolSerializerException( + System.Runtime.Serialization.SerializationInfo info, + System.Runtime.Serialization.StreamingContext context) + : base(info, context) { } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/MissingGeneratorException.cs b/src/Microsoft.Sbom.Api/Exceptions/MissingGeneratorException.cs new file mode 100644 index 00000000..bd961aae --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/MissingGeneratorException.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when we are unable to find a generator to serialize the SBOM + /// + [Serializable] + public class MissingGeneratorException : Exception + { + public MissingGeneratorException() + { + } + + public MissingGeneratorException(string message) + : base(message) + { + } + + public MissingGeneratorException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected MissingGeneratorException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/SignValidatorNotFoundException.cs b/src/Microsoft.Sbom.Api/Exceptions/SignValidatorNotFoundException.cs new file mode 100644 index 00000000..983e37fd --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/SignValidatorNotFoundException.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when the manifest tool cannot find a signature validator for the current + /// operating system. + /// + [Serializable] + public class SignValidatorNotFoundException : Exception + { + public SignValidatorNotFoundException() + { + } + + public SignValidatorNotFoundException(string message) + : base(message) + { + } + + public SignValidatorNotFoundException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected SignValidatorNotFoundException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Exceptions/UnsupportedHashAlgorithmException.cs b/src/Microsoft.Sbom.Api/Exceptions/UnsupportedHashAlgorithmException.cs new file mode 100644 index 00000000..73b7d027 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Exceptions/UnsupportedHashAlgorithmException.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Runtime.Serialization; + +namespace Microsoft.Sbom.Api.Exceptions +{ + /// + /// Thrown when we are provided a hash algorithm value that is currently not supported by our service. + /// + [Serializable] + public class UnsupportedHashAlgorithmException : Exception + { + public UnsupportedHashAlgorithmException() + { + } + + public UnsupportedHashAlgorithmException(string message) + : base(message) + { + } + + public UnsupportedHashAlgorithmException(string message, Exception innerException) + : base(message, innerException) + { + } + + protected UnsupportedHashAlgorithmException(SerializationInfo info, StreamingContext context) + : base(info, context) + { + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ChannelUtils.cs b/src/Microsoft.Sbom.Api/Executors/ChannelUtils.cs new file mode 100644 index 00000000..55036889 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ChannelUtils.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors +{ + public class ChannelUtils + { + /// + /// Merges a given array of input channels into a common input channel. + /// + /// Type of the channel. + /// The list of input channels. + /// A for all the combined inputs. + public ChannelReader Merge(params ChannelReader[] inputs) + { + var output = Channel.CreateUnbounded(); + + Task.Run(async () => + { + async Task Redirect(ChannelReader input) + { + await foreach (T item in input.ReadAllAsync()) + { + await output.Writer.WriteAsync(item); + } + } + + await Task.WhenAll(inputs.Select(i => Redirect(i)).ToArray()); + output.Writer.Complete(); + }); + + return output; + } + + /// + /// Splits a given input channel into 'n' seperate channels. + /// + /// The type of the channel. + /// The input channel. + /// The number of channels to create. + /// A of s. + public IList> Split(ChannelReader input, int n) + { + var outputs = new Channel[n]; + for (var i = 0; i < n; i++) + { + outputs[i] = Channel.CreateUnbounded(); + } + + Task.Run(async () => + { + var index = 0; + + await foreach (T item in input.ReadAllAsync()) + { + await outputs[index].Writer.WriteAsync(item); + index = (index + 1) % n; + } + + foreach (Channel ch in outputs) + { + ch.Writer.Complete(); + } + }); + + return outputs.Select(ch => ch.Reader).ToArray(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs new file mode 100644 index 00000000..b1bd90df --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ComponentDetectionBaseWalker.cs @@ -0,0 +1,131 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Castle.Core.Internal; +using DropValidator.Api.Config.Extensions; +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Common; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Serilog.Events; +using System; +using System.Collections.Generic; +using System.IO; +using System.Threading.Channels; +using System.Threading.Tasks; +using ILogger = Serilog.ILogger; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Abstract class that runs component detection tool in the given folder. + /// + public abstract class ComponentDetectionBaseWalker + { + private readonly ILogger log; + private readonly ComponentDetectorCachedExecutor componentDetector; + private readonly IConfiguration configuration; + private readonly VerbosityMode verbosity; + + private ComponentDetectionCliArgumentBuilder cliArgumentBuilder; + + public ComponentDetectionBaseWalker( + ILogger log, + ComponentDetectorCachedExecutor componentDetector, + IConfiguration configuration, + ISbomConfigProvider sbomConfigs) + { + if (sbomConfigs is null) + { + throw new ArgumentNullException(nameof(sbomConfigs)); + } + + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.componentDetector = componentDetector ?? throw new ArgumentNullException(nameof(componentDetector)); + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + + verbosity = configuration.Verbosity.Value switch + { + LogEventLevel.Verbose => VerbosityMode.Verbose, + _ => VerbosityMode.Normal, + }; + + cliArgumentBuilder = new ComponentDetectionCliArgumentBuilder().Scan().Verbosity(verbosity); + + // Enable SPDX22 detector which is disabled by default. + cliArgumentBuilder.AddDetectorArg("SPDX22SBOM", "EnableIfDefaultOff"); + + if (sbomConfigs.TryGet(Constants.SPDX22ManifestInfo, out ISbomConfig spdxSbomConfig)) + { + var directory = Path.GetDirectoryName(spdxSbomConfig.ManifestJsonFilePath); + if (!directory.IsNullOrEmpty()) + { + cliArgumentBuilder.AddArg("DirectoryExclusionList", directory); + } + } + } + + public (ChannelReader output, ChannelReader error) GetComponents(string buildComponentDirPath) + { + log.Debug($"Scanning for packages under the root path {buildComponentDirPath}."); + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + if (string.IsNullOrEmpty(buildComponentDirPath)) + { + output.Writer.Complete(); + errors.Writer.Complete(); + return (output, errors); + } + + async Task Scan(string path) + { + cliArgumentBuilder.SourceDirectory(buildComponentDirPath); + var cmdLineParams = configuration.ToComponentDetectorCommandLineParams(cliArgumentBuilder); + + var scanResult = await Task.Run(() => componentDetector.Scan(cmdLineParams)); + + if (scanResult.ResultCode != ProcessingResultCode.Success) + { + await errors.Writer.WriteAsync(new ComponentDetectorException($"Component detector failed. Result: {scanResult.ResultCode}.")); + return; + } + + var uniqueComponents = FilterScannedComponents(scanResult); + + foreach (var component in uniqueComponents) + { + await output.Writer.WriteAsync(component); + } + } + + Task.Run(async () => + { + try + { + await Scan(buildComponentDirPath); + } + catch (Exception e) + { + log.Error($"Unknown error while running CD scan: {e}"); + await errors.Writer.WriteAsync(new ComponentDetectorException("Unknown exception", e)); + return; + } + finally + { + output.Writer.Complete(); + errors.Writer.Complete(); + } + }); + + return (output, errors); + } + + protected abstract IEnumerable FilterScannedComponents(ScanResult result); + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs b/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs new file mode 100644 index 00000000..a7c5a807 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ComponentToPackageInfoConverter.cs @@ -0,0 +1,72 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Adapters.Adapters.ComponentDetection; +using Microsoft.Sbom.Adapters.Report; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Contracts; +using Serilog; +using System; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Takes a object and converts it to a + /// object using a . + /// + public class ComponentToPackageInfoConverter + { + private readonly ILogger log; + + // TODO: Remove and use interface + // For unit testing only + public ComponentToPackageInfoConverter() { } + + public ComponentToPackageInfoConverter(ILogger log) + { + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public virtual (ChannelReader output, ChannelReader errors) Convert(ChannelReader componentReader) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + var report = new AdapterReport(); + await foreach (ScannedComponent scannedComponent in componentReader.ReadAllAsync()) + { + await ConvertComponentToPackage(scannedComponent, output, errors); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + + async Task ConvertComponentToPackage(ScannedComponent scannedComponent, Channel output, Channel errors) + { + try + { + var sbom = scannedComponent.ToSbomPackage(report); + await output.Writer.WriteAsync(sbom); + } + catch (Exception e) + { + log.Debug($"Encountered an error while processing package {scannedComponent.Component.Id}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.PackageError, + Path = scannedComponent.LocationsFoundAt.FirstOrDefault() + }); + } + } + }); + + return (output, errors); + } + } +} \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Executors/DirectoryWalker.cs b/src/Microsoft.Sbom.Api/Executors/DirectoryWalker.cs new file mode 100644 index 00000000..80575d3c --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/DirectoryWalker.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Serilog; +using System; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Given a directory path, walks the subtree and returns all the + /// files in the directory. + /// + public class DirectoryWalker + { + private readonly IFileSystemUtils fileSystemUtils; + private readonly ILogger log; + private readonly bool followSymlinks; + + public DirectoryWalker(IFileSystemUtils fileSystemUtils, ILogger log, IConfiguration configuration) + { + if (configuration is null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + + followSymlinks = configuration.FollowSymlinks?.Value ?? true; + + if (!followSymlinks) + { + log.Information("FollowSymlinks parameter is set to false, we won't follow symbolic links while traversing the filesystem."); + } + } + + public (ChannelReader file, ChannelReader errors) GetFilesRecursively(string root) + { + log.Debug($"Enumerating files under the root path {root}."); + + if (!fileSystemUtils.DirectoryExists(root)) + { + throw new InvalidPathException($"The root path at {root} doesn't exist or is not accessible."); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + async Task WalkDir(string path) + { + try + { + foreach (var file in fileSystemUtils.GetFilesInDirectory(path, followSymlinks)) + { + await output.Writer.WriteAsync(file); + } + + var tasks = fileSystemUtils.GetDirectories(path, followSymlinks).Select(WalkDir); + await Task.WhenAll(tasks.ToArray()); + } + catch (Exception e) + { + log.Debug($"Encountered an unknown error for {path}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = path + }); + } + } + + Task.Run(async () => + { + await WalkDir(root); + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ExternalDocumentReferenceWriter.cs b/src/Microsoft.Sbom.Api/Executors/ExternalDocumentReferenceWriter.cs new file mode 100644 index 00000000..dd328f71 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ExternalDocumentReferenceWriter.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Uses the to write a json object that contains + /// a format specific representation of the . + /// + public class ExternalDocumentReferenceWriter + { + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + private readonly ILogger log; + + public ExternalDocumentReferenceWriter( + ManifestGeneratorProvider manifestGeneratorProvider, + ILogger log) + { + this.manifestGeneratorProvider = manifestGeneratorProvider ?? throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public (ChannelReader result, ChannelReader errors) Write(ChannelReader externalDocumentReferenceInfos, IList externalDocumentReferenceArraySupportingConfigs) + { + var errors = Channel.CreateUnbounded(); + var result = Channel.CreateUnbounded(); + + if (externalDocumentReferenceInfos is null) + { + throw new ArgumentNullException(nameof(externalDocumentReferenceInfos)); + } + + if (externalDocumentReferenceArraySupportingConfigs is null) + { + throw new ArgumentNullException(nameof(externalDocumentReferenceArraySupportingConfigs)); + } + + Task.Run(async () => + { + await foreach (ExternalDocumentReferenceInfo externalDocumentReferenceInfo in externalDocumentReferenceInfos.ReadAllAsync()) + { + foreach (var config in externalDocumentReferenceArraySupportingConfigs) + { + try + { + var generationResult = manifestGeneratorProvider + .Get(config.ManifestInfo) + .GenerateJsonDocument(externalDocumentReferenceInfo); + config.Recorder.RecordExternalDocumentReferenceIdAndRootElement(generationResult?.ResultMetadata?.EntityId, externalDocumentReferenceInfo.DescribedElementID); + await result.Writer.WriteAsync((generationResult?.Document, config.JsonSerializer)); + } + catch (Exception e) + { + log.Warning($"Encountered an error while generating json for external document reference {externalDocumentReferenceInfo.ExternalDocumentName}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.JsonSerializationError, + Path = externalDocumentReferenceInfo.ExternalDocumentName + }); + } + } + } + + errors.Writer.Complete(); + result.Writer.Complete(); + }); + + return (result, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/FileHasher.cs b/src/Microsoft.Sbom.Api/Executors/FileHasher.cs new file mode 100644 index 00000000..45930ae9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/FileHasher.cs @@ -0,0 +1,145 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Convertors; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Serilog; +using System; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Given a list of file paths, returns a object containing the + /// file's path in the manifest file format and its hash code. + /// + public class FileHasher + { + private readonly IHashCodeGenerator hashCodeGenerator; + private readonly IManifestPathConverter manifestPathConverter; + private readonly ILogger log; + private readonly ISbomConfigProvider sbomConfigs; + private readonly IFileTypeUtils fileTypeUtils; + private readonly AlgorithmName[] hashAlgorithmNames; + + public ManifestData ManifestData { get; set; } + + public FileHasher( + IHashCodeGenerator hashCodeGenerator, + IManifestPathConverter manifestPathConverter, + ILogger log, + IConfiguration configuration, + ISbomConfigProvider sbomConfigs, + ManifestGeneratorProvider manifestGeneratorProvider, + FileTypeUtils fileTypeUtils) + { + if (configuration is null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + if (manifestGeneratorProvider is null) + { + throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + this.hashCodeGenerator = hashCodeGenerator ?? throw new ArgumentNullException(nameof(hashCodeGenerator)); + this.manifestPathConverter = manifestPathConverter ?? throw new ArgumentNullException(nameof(manifestPathConverter)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); + this.fileTypeUtils = fileTypeUtils ?? throw new ArgumentNullException(nameof(fileTypeUtils)); + + // Set the hash algorithms to calculate based on the action. + switch (configuration.ManifestToolAction) + { + case ManifestToolActions.Validate: + hashAlgorithmNames = new AlgorithmName[] + { + configuration.HashAlgorithm.Value + }; + break; + case ManifestToolActions.Generate: + + hashAlgorithmNames = sbomConfigs.GetManifestInfos() + .Select(config => manifestGeneratorProvider + .Get(config) + .RequiredHashAlgorithms) + .SelectMany(h => h) + .Distinct() + .ToArray(); + break; + } + } + + public (ChannelReader, ChannelReader) Run(ChannelReader fileInfo) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (string file in fileInfo.ReadAllAsync()) + { + await GenerateHash(file, output, errors); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private async Task GenerateHash(string file, Channel output, Channel errors) + { + string relativeFilePath = null; + bool isOutsideDropPath = false; + try + { + (relativeFilePath, isOutsideDropPath) = manifestPathConverter.Convert(file); + Checksum[] fileHashes = hashCodeGenerator.GenerateHashes(file, hashAlgorithmNames); + if (fileHashes == null || fileHashes.Length == 0 || fileHashes.Any(f => string.IsNullOrEmpty(f.ChecksumValue))) + { + throw new HashGenerationException($"Failed to generate hashes for '{file}'."); + } + + // Record hashes + sbomConfigs.ApplyToEachConfig(config => config.Recorder.RecordChecksumForFile(fileHashes)); + + await output.Writer.WriteAsync( + new InternalSBOMFileInfo + { + Path = relativeFilePath, + IsOutsideDropPath = isOutsideDropPath, + Checksum = fileHashes, + FileTypes = fileTypeUtils.GetFileTypesBy(file), + }); + } + catch (Exception e) + { + if (ManifestData != null && !string.IsNullOrWhiteSpace(relativeFilePath)) + { + ManifestData.HashesMap.Remove(relativeFilePath); + } + + log.Error($"Encountered an error while generating hash for file {file}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = Entities.ErrorType.Other, + Path = relativeFilePath ?? file + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/FileInfoWriter.cs b/src/Microsoft.Sbom.Api/Executors/FileInfoWriter.cs new file mode 100644 index 00000000..a942a37e --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/FileInfoWriter.cs @@ -0,0 +1,102 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Uses the to write a json object that contains + /// a file path and its associated hashes. + /// + public class FileInfoWriter + { + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + private readonly ILogger log; + private readonly IFileSystemUtilsExtension fileSystemUtilsExtension; + private readonly IConfiguration configuration; + + public FileInfoWriter( + ManifestGeneratorProvider manifestGeneratorProvider, + ILogger log, + IFileSystemUtilsExtension fileSystemUtilsExtension, + IConfiguration configuration) + { + if (manifestGeneratorProvider is null) + { + throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + this.manifestGeneratorProvider = manifestGeneratorProvider; + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.fileSystemUtilsExtension = fileSystemUtilsExtension ?? throw new ArgumentNullException(nameof(fileSystemUtilsExtension)); + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + } + + public (ChannelReader result, ChannelReader errors) Write(ChannelReader fileInfos, IList filesArraySupportingSBOMs) + { + var errors = Channel.CreateUnbounded(); + var result = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (InternalSBOMFileInfo fileInfo in fileInfos.ReadAllAsync()) + { + await Generate(filesArraySupportingSBOMs, fileInfo, result, errors); + } + + errors.Writer.Complete(); + result.Writer.Complete(); + }); + + return (result, errors); + } + + private async Task Generate(IList filesArraySupportingSBOMs, InternalSBOMFileInfo sbomFile, Channel result, Channel errors) + { + try + { + foreach (var config in filesArraySupportingSBOMs) + { + var generationResult = manifestGeneratorProvider + .Get(config.ManifestInfo) + .GenerateJsonDocument(sbomFile); + + var fileId = generationResult?.ResultMetadata?.EntityId; + + if (!sbomFile.IsOutsideDropPath) + { + config.Recorder.RecordFileId(fileId); + } + + if (sbomFile.FileTypes != null && sbomFile.FileTypes.Contains(Contracts.Enums.FileType.SPDX)) + { + config.Recorder.RecordSPDXFileId(fileId); + } + + await result.Writer.WriteAsync((generationResult?.Document, config.JsonSerializer)); + } + } + catch (Exception e) + { + log.Debug($"Encountered an error while generating json for file {sbomFile.Path}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.JsonSerializationError, + Path = sbomFile.Path + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/FileListEnumerator.cs b/src/Microsoft.Sbom.Api/Executors/FileListEnumerator.cs new file mode 100644 index 00000000..1d8f1ddf --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/FileListEnumerator.cs @@ -0,0 +1,118 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Serilog; +using System; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using System.Collections.Generic; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Takes a data file containing a list of files and enumerates each file in the list. + /// The files should be present on the disk. + /// + public class FileListEnumerator + { + private readonly IFileSystemUtils fileSystemUtils; + private readonly ILogger log; + + /// + /// FileListEnumerator constructor for dependency injection. + /// + /// IFileSystemUtils interface used for this instance. + /// Ilogger interface used for this instance. + public FileListEnumerator(IFileSystemUtils fileSystemUtils, ILogger log) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + /// + /// Reads a list file which is a text file containing one full file path per line, validates the file + /// exists and adds the file to the output stream. + /// + /// Full file path to the list file to read. + /// + public (ChannelReader file, ChannelReader errors) GetFilesFromList(string listFile) + { + if (string.IsNullOrWhiteSpace(listFile)) + { + throw new ArgumentException($"'{nameof(listFile)}' cannot be null or whitespace.", nameof(listFile)); + } + + log.Debug($"Enumerating all files from {nameof(listFile)}."); + + if (!fileSystemUtils.FileExists(listFile)) + { + throw new InvalidPathException($"The list file {listFile} doesn't exist or is not accessible."); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + async Task ProcessLines(string file) + { + string allText = null; + try + { + allText = fileSystemUtils.ReadAllText(file); + } + catch (Exception) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = file + }); + } + + // Split on Environment.NewLine and discard blank lines. + string[] separator = new string[] { Environment.NewLine }; + IEnumerable files = allText.Split(separator, StringSplitOptions.None) + .Where(t => !string.IsNullOrEmpty(t)); + foreach (var oneFile in files) + { + try + { + string absoluteFileName = fileSystemUtils.AbsolutePath(oneFile); + if (!fileSystemUtils.FileExists(absoluteFileName)) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.MissingFile, + Path = absoluteFileName + }); + } + else + { + await output.Writer.WriteAsync(absoluteFileName); + } + } + catch (Exception) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = oneFile + }); + } + } + } + + Task.Run(async () => + { + await ProcessLines(listFile); + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/HashValidator.cs b/src/Microsoft.Sbom.Api/Executors/HashValidator.cs new file mode 100644 index 00000000..15dfb1fc --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/HashValidator.cs @@ -0,0 +1,87 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Given a list of objects, and a + /// object, validates if the file hash matches the hash provided in the manifest data. + /// + /// Used only in the Validation action. + /// + public class HashValidator + { + private readonly ManifestData manifestData; + private readonly IConfiguration configuration; + + public HashValidator(IConfiguration configuration, ManifestData manifestData) + { + this.configuration = configuration; + this.manifestData = manifestData; + } + + public (ChannelReader output, ChannelReader errors) + Validate(ChannelReader fileWithHash) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (var fileHash in fileWithHash.ReadAllAsync()) + { + await Validate(fileHash, output, errors); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private async Task Validate(InternalSBOMFileInfo fileHash, Channel output, Channel errors) + { + var result = new FileValidationResult + { + Path = fileHash.Path + }; + + if (manifestData.HashesMap.TryGetValue(fileHash.Path, out Checksum[] expectedHashes)) + { + manifestData.HashesMap.Remove(fileHash.Path); + + var expectedHash = expectedHashes + .Where(e => e.Algorithm == configuration.HashAlgorithm.Value) + .Select(e => e.ChecksumValue).First(); + var actualHash = fileHash.Checksum + .Where(e => e.Algorithm == configuration.HashAlgorithm.Value) + .Select(e => e.ChecksumValue).First(); + + if (expectedHash == actualHash) + { + result.ErrorType = ErrorType.None; + await output.Writer.WriteAsync(result); + } + else + { + result.ErrorType = ErrorType.InvalidHash; + await errors.Writer.WriteAsync(result); + } + } + else + { + result.ErrorType = ErrorType.AdditionalFile; + await errors.Writer.WriteAsync(result); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ISBOMReaderForExternalDocumentReference.cs b/src/Microsoft.Sbom.Api/Executors/ISBOMReaderForExternalDocumentReference.cs new file mode 100644 index 00000000..a1e67bfc --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ISBOMReaderForExternalDocumentReference.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Threading.Channels; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// interface to read read SBOM file. Implement this class for different formats of SBOM file + /// + public interface ISBOMReaderForExternalDocumentReference + { + (ChannelReader results, ChannelReader errors) ParseSBOMFile(ChannelReader sbomFileLocation); + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ListWalker.cs b/src/Microsoft.Sbom.Api/Executors/ListWalker.cs new file mode 100644 index 00000000..74c24b3e --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ListWalker.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Given a list of objects returns a stream of each of those individual objects in a channel stream. + /// + /// + public class ListWalker + { + public (ChannelReader output, ChannelReader error) GetComponents(IEnumerable components) + { + if (components is null) + { + throw new ArgumentNullException(nameof(components)); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + try + { + foreach (var component in components) + { + await output.Writer.WriteAsync(component); + } + } + catch (Exception ex) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.PackageError, + Path = ex.Message + }); + } + finally + { + output.Writer.Complete(); + errors.Writer.Complete(); + } + }); + + return (output, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ManifestFileFilterer.cs b/src/Microsoft.Sbom.Api/Executors/ManifestFileFilterer.cs new file mode 100644 index 00000000..cccc9a75 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ManifestFileFilterer.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Ninject; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Filters; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Filters out files in the manifest.json that are not present on the disk and shouldn't + /// be checked as specified by the root path filter parameter. + /// + public class ManifestFileFilterer + { + private readonly ManifestData manifestData; + private readonly IFilter rootPathFilter; + private readonly IConfiguration configuration; + private readonly ILogger log; + private readonly IFileSystemUtils fileSystemUtils; + + public ManifestFileFilterer( + ManifestData manifestData, + [Named(nameof(DownloadedRootPathFilter))] + IFilter rootPathFilter, + IConfiguration configuration, + ILogger log, + IFileSystemUtils fileSystemUtils) + { + this.manifestData = manifestData ?? throw new ArgumentNullException(nameof(manifestData)); + this.rootPathFilter = rootPathFilter ?? throw new ArgumentNullException(nameof(rootPathFilter)); + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.fileSystemUtils = fileSystemUtils; + } + + public ChannelReader FilterManifestFiles() + { + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + var manifestKeys = manifestData.HashesMap != null + ? new List(manifestData.HashesMap.Keys) + : new List(); + + foreach (var manifestFile in manifestKeys) + { + try + { + string file = fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, manifestFile); + if (!rootPathFilter.IsValid(file)) + { + // This path is filtered, remove from the manifest map. + manifestData.HashesMap.Remove(manifestFile); + + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.FilteredRootPath, + Path = manifestFile + }); + } + } + catch (Exception e) + { + log.Debug($"Encountered an error while filtering file {manifestFile} from the manifest: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = manifestFile + }); + } + } + + errors.Writer.Complete(); + }); + + return errors; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/ManifestFolderFilterer.cs b/src/Microsoft.Sbom.Api/Executors/ManifestFolderFilterer.cs new file mode 100644 index 00000000..1725d5f7 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/ManifestFolderFilterer.cs @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Ninject; +using Serilog; +using System; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Filters; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Filters out folders for which we don't generate hashes, such as anything under the _manifest folder. + /// + public class ManifestFolderFilterer + { + private readonly IFilter manifestFolderFilter; + private readonly ILogger log; + + public ManifestFolderFilterer( + [Named(nameof(ManifestFolderFilter))] IFilter manifestFolderFilter, + ILogger log) + { + this.manifestFolderFilter = manifestFolderFilter ?? throw new ArgumentNullException(nameof(manifestFolderFilter)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public (ChannelReader file, ChannelReader errors) FilterFiles(ChannelReader files) + { + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (string file in files.ReadAllAsync()) + { + await FilterFiles(file, errors, output); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private async Task FilterFiles(string file, Channel errors, Channel output) + { + try + { + if (!manifestFolderFilter.IsValid(file)) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.ManifestFolder, + Path = file + }); + } + else + { + await output.Writer.WriteAsync(file); + } + } + catch (Exception e) + { + log.Debug($"Encountered an error while filtering file {file}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = file + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/PackageInfoJsonWriter.cs b/src/Microsoft.Sbom.Api/Executors/PackageInfoJsonWriter.cs new file mode 100644 index 00000000..4e5b3f73 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/PackageInfoJsonWriter.cs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Contracts; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Uses the to write a json object that contains + /// a format specific representation of the . + /// + public class PackageInfoJsonWriter + { + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + private readonly ILogger log; + + public PackageInfoJsonWriter( + ManifestGeneratorProvider manifestGeneratorProvider, + ILogger log) + { + if (manifestGeneratorProvider is null) + { + throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + this.manifestGeneratorProvider = manifestGeneratorProvider; + this.log = log ?? throw new ArgumentNullException(nameof(log)); + } + + public (ChannelReader result, ChannelReader errors) Write(ChannelReader packageInfos, IList packagesArraySupportingConfigs) + { + var errors = Channel.CreateUnbounded(); + var result = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (SBOMPackage packageInfo in packageInfos.ReadAllAsync()) + { + await GenerateJson(packagesArraySupportingConfigs, packageInfo, result, errors); + } + + errors.Writer.Complete(); + result.Writer.Complete(); + }); + + return (result, errors); + } + + private async Task GenerateJson(IList packagesArraySupportingConfigs, SBOMPackage packageInfo, Channel result, + Channel errors) + { + try + { + foreach (ISbomConfig sbomConfig in packagesArraySupportingConfigs) + { + var generationResult = + manifestGeneratorProvider.Get(sbomConfig.ManifestInfo).GenerateJsonDocument(packageInfo); + sbomConfig.Recorder.RecordPackageId(generationResult?.ResultMetadata?.EntityId); + await result.Writer.WriteAsync((generationResult?.Document, sbomConfig.JsonSerializer)); + } + } + catch (Exception e) + { + log.Debug($"Encountered an error while generating json for packageInfo {packageInfo}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.JsonSerializationError, + Path = packageInfo.PackageName + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs b/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs new file mode 100644 index 00000000..3e2d8eac --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/PackagesWalker.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using System.Collections.Generic; +using System.Linq; +using ILogger = Serilog.ILogger; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Runs the component detection tool and returns a list of components scanned in the given folder. + /// + public class PackagesWalker : ComponentDetectionBaseWalker + { + public PackagesWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs) + : base(log, componentDetector, configuration, sbomConfigs) + { + } + + protected override IEnumerable FilterScannedComponents(ScanResult result) + { + return result + .ComponentsFound + .Where(component => !(component.Component is SpdxComponent)) // We exclude detected SBOMs from packages section and reference them as an ExternalReference + .Distinct(new ScannedComponentEqualityComparer()) + .ToList(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/RelationshipGenerator.cs b/src/Microsoft.Sbom.Api/Executors/RelationshipGenerator.cs new file mode 100644 index 00000000..6663a858 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/RelationshipGenerator.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Manifest; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Takes a list of relationships and generates the SBOM JsonDocuments for each of the + /// relationship. + /// + public class RelationshipGenerator + { + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + + public RelationshipGenerator(ManifestGeneratorProvider manifestGeneratorProvider) + { + this.manifestGeneratorProvider = manifestGeneratorProvider ?? throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + public virtual ChannelReader Run(IEnumerator relationships, ManifestInfo manifestInfo) + { + var output = Channel.CreateUnbounded(); + + Task.Run(async () => + { + using (relationships) + { + try + { + while (relationships.MoveNext()) + { + IManifestGenerator manifestGenerator = manifestGeneratorProvider.Get(manifestInfo); + GenerationResult generationResult = manifestGenerator.GenerateJsonDocument(relationships.Current); + await output.Writer.WriteAsync(generationResult?.Document); + } + } + finally + { + output.Writer.Complete(); + } + } + }); + + return output; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs b/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs new file mode 100644 index 00000000..7ad60a8f --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/SBOMComponentsWalker.cs @@ -0,0 +1,34 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using System.Collections.Generic; +using System.Linq; +using ILogger = Serilog.ILogger; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Runs the component detection tool and returns a list of SBOM components scanned in the given folder. + /// + public class SBOMComponentsWalker : ComponentDetectionBaseWalker + { + public SBOMComponentsWalker(ILogger log, ComponentDetectorCachedExecutor componentDetector, IConfiguration configuration, ISbomConfigProvider sbomConfigs) + : base(log, componentDetector, configuration, sbomConfigs) + { + } + + protected override IEnumerable FilterScannedComponents(ScanResult result) + { + return result + .ComponentsFound + .Where(component => component.Component is SpdxComponent) + .Distinct(new ScannedComponentEqualityComparer()) + .ToList(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs b/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs new file mode 100644 index 00000000..ad8e9d76 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/SBOMFileToFileInfoConverter.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Takes a SBOMFile and converts it to a FileInfo object. + /// + public class SBOMFileToFileInfoConverter + { + private readonly IFileTypeUtils fileTypeUtils; + + public SBOMFileToFileInfoConverter(FileTypeUtils fileTypeUtils) + { + this.fileTypeUtils = fileTypeUtils ?? throw new ArgumentNullException(nameof(fileTypeUtils)); + } + + public (ChannelReader output, ChannelReader error) Convert(ChannelReader componentReader) + { + if (componentReader is null) + { + throw new ArgumentNullException(nameof(componentReader)); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (SBOMFile component in componentReader.ReadAllAsync()) + { + await Convert(component, output, errors); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private async Task Convert(SBOMFile component, Channel output, Channel errors) + { + try + { + var checksums = new List(); + foreach (var checksum in component.Checksum) + { + checksums.Add(new Checksum + { + Algorithm = checksum.Algorithm, + ChecksumValue = checksum.ChecksumValue + }); + } + + var fileInfo = new InternalSBOMFileInfo + { + Path = component.Path, + Checksum = checksums.ToArray(), + FileCopyrightText = component.FileCopyrightText, + LicenseConcluded = component.LicenseConcluded, + LicenseInfoInFiles = component.LicenseInfoInFiles, + FileTypes = fileTypeUtils.GetFileTypesBy(component.Path), + IsOutsideDropPath = false, // assumption from SBOMApi is that Files are in dropPath + }; + + await output.Writer.WriteAsync(fileInfo); + } + catch (UnsupportedHashAlgorithmException) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.UnsupportedHashAlgorithm, + Path = component.Path + }); + } + catch (Exception) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = component.Path + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/SBOMPackageToPackageInfoConverter.cs b/src/Microsoft.Sbom.Api/Executors/SBOMPackageToPackageInfoConverter.cs new file mode 100644 index 00000000..9594a545 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/SBOMPackageToPackageInfoConverter.cs @@ -0,0 +1,92 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts; +using System; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Takes a SBOMPackage object and converts it into a PackageInfo object. + /// + public class SBOMPackageToPackageInfoConverter + { + public (ChannelReader ouput, ChannelReader errors) Convert(ChannelReader componentReader) + { + if (componentReader is null) + { + throw new ArgumentNullException(nameof(componentReader)); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (SBOMPackage component in componentReader.ReadAllAsync()) + { + await WritePackageInfo(component, output, errors); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private static async Task WritePackageInfo(SBOMPackage component, Channel output, Channel errors) + { + try + { + var checksums = new List(); + if (component.Checksum != null) + { + foreach (var checksum in component.Checksum) + { + checksums.Add(new Checksum + { + Algorithm = checksum.Algorithm, + ChecksumValue = checksum.ChecksumValue + }); + } + } + + var licenceInfo = new LicenseInfo + { + Concluded = component.LicenseInfo?.Concluded, + Declared = component.LicenseInfo?.Declared + }; + + var packageInfo = new SBOMPackage + { + Id = component.Id, + Checksum = checksums, + CopyrightText = component.CopyrightText, + FilesAnalyzed = component.FilesAnalyzed, + LicenseInfo = licenceInfo, + PackageName = component.PackageName, + PackageUrl = component.PackageUrl, + PackageSource = component.PackageSource, + Supplier = component.Supplier, + PackageVersion = component.PackageVersion + }; + + await output.Writer.WriteAsync(packageInfo); + } + catch (Exception) + { + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.PackageError, + Path = component.Id + }); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalReference.cs b/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalReference.cs new file mode 100644 index 00000000..8c525791 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Executors/SPDXSBOMReaderForExternalReference.cs @@ -0,0 +1,210 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Serilog; +using System; +using System.Buffers; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Threading.Channels; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; +using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; + +namespace Microsoft.Sbom.Api.Executors +{ + /// + /// Reads SPDX json format SBOM file + /// + public class SPDXSBOMReaderForExternalDocumentReference : ISBOMReaderForExternalDocumentReference + { + private readonly IHashCodeGenerator hashCodeGenerator; + private readonly ILogger log; + private readonly ISbomConfigProvider sbomConfigs; + private readonly AlgorithmName[] hashAlgorithmNames; + private readonly IFileSystemUtils fileSystemUtils; + + private readonly IEnumerable supportedSPDXVersions = new List { "SPDX-2.2" }; + + public SPDXSBOMReaderForExternalDocumentReference( + IHashCodeGenerator hashCodeGenerator, + ILogger log, + IConfiguration configuration, + ISbomConfigProvider sbomConfigs, + ManifestGeneratorProvider manifestGeneratorProvider, + IFileSystemUtils fileSystemUtils) + { + if (configuration is null) + { + throw new ArgumentNullException(nameof(configuration)); + } + + if (manifestGeneratorProvider is null) + { + throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + this.hashCodeGenerator = hashCodeGenerator ?? throw new ArgumentNullException(nameof(hashCodeGenerator)); + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.sbomConfigs = sbomConfigs ?? throw new ArgumentNullException(nameof(sbomConfigs)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + + hashAlgorithmNames = sbomConfigs.GetManifestInfos() + .Select(config => manifestGeneratorProvider + .Get(config) + .RequiredHashAlgorithms) + .SelectMany(h => h) + .Distinct() + .ToArray(); + } + + public virtual (ChannelReader results, ChannelReader errors) ParseSBOMFile(ChannelReader sbomFileLocation) + { + if (sbomFileLocation is null) + { + throw new ArgumentNullException(nameof(sbomFileLocation)); + } + + var output = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + Task.Run(async () => + { + IList externalDocumentReferenceInfos = new List(); + await foreach (string file in sbomFileLocation.ReadAllAsync()) + { + if (!file.EndsWith(Constants.SPDXFileExtension)) + { + log.Warning($"The file {file} is not an spdx document."); + } + else + { + try + { + var externalDocumentReference = ReadJson(file); + if (externalDocumentReference != null) + { + externalDocumentReferenceInfos.Add(externalDocumentReference); + } + } + catch (JsonException e) + { + log.Error($"Encountered an error while parsing the external SBOM file {file}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = file + }); + } + catch (HashGenerationException e) + { + log.Debug($"Encountered an error while generating hash for file {file}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = file + }); + } + catch (Exception e) + { + log.Debug($"Encountered an error while generating externalDocumentReferenceInfo from file {file}: {e.Message}"); + await errors.Writer.WriteAsync(new FileValidationResult + { + ErrorType = ErrorType.Other, + Path = file + }); + } + } + } + + foreach (var externalDocumentRefrence in externalDocumentReferenceInfos) + { + await output.Writer.WriteAsync(externalDocumentRefrence); + } + + output.Writer.Complete(); + errors.Writer.Complete(); + }); + + return (output, errors); + } + + private ExternalDocumentReferenceInfo ReadJson(string file) + { + Checksum[] checksums; + checksums = hashCodeGenerator.GenerateHashes(file, hashAlgorithmNames); + + using (var openStream = fileSystemUtils.OpenRead(file)) + using (JsonDocument doc = JsonDocument.Parse(openStream)) + { + JsonElement root = doc.RootElement; + string nameValue; + string documentNamespaceValue; + string versionValue; + string rootElementValue; + + if (root.TryGetProperty(Constants.SpdxVersionString, out JsonElement version)) + { + versionValue = version.GetString(); + } + else + { + throw new Exception($"{Constants.SpdxVersionString} property could not be parsed from referenced SPDX Document '{file}', this is not a valid SPDX-2.2 Document."); + } + + if (!IsSPDXVersionSupported(versionValue)) + { + throw new Exception($"The SPDX version ${versionValue} is not valid format in the referenced SBOM, we currently only support SPDX-2.2 SBOM format."); + } + + if (root.TryGetProperty(Constants.NameString, out JsonElement name)) + { + nameValue = name.GetString(); + } + else + { + throw new Exception($"{Constants.NameString} property could not be parsed from referenced SPDX Document '{file}'."); + } + + if (root.TryGetProperty(Constants.DocumentNamespaceString, out JsonElement documentNamespace)) + { + documentNamespaceValue = documentNamespace.GetString(); + } + else + { + throw new Exception($"{Constants.DocumentNamespaceString} property could not be parsed from referenced SPDX Document '{file}'."); + } + + if (root.TryGetProperty(Constants.DocumentDescribesString, out JsonElement rootElements)) + { + rootElementValue = rootElements.EnumerateArray().FirstOrDefault().ToString() ?? Constants.DefaultRootElement; + } + else + { + throw new Exception($"{Constants.DocumentDescribesString} property could not be parsed from referenced SPDX Document '{file}'."); + } + + return new ExternalDocumentReferenceInfo + { + DocumentNamespace = documentNamespaceValue, + ExternalDocumentName = nameValue, + Checksum = checksums, + DescribedElementID = rootElementValue + }; + } + } + + private bool IsSPDXVersionSupported(string version) => supportedSPDXVersions.Contains(version); + } +} diff --git a/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs new file mode 100644 index 00000000..b42c00b4 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Filters/DownloadedRootPathFilter.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Serilog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Filters +{ + /// + /// This filter checks if the path of a file matches the provided + /// root path filter, and returns true if it does. + /// + public class DownloadedRootPathFilter : IFilter + { + private readonly IConfiguration configuration; + private readonly IFileSystemUtils fileSystemUtils; + private readonly ILogger logger; + + private bool skipValidation; + private HashSet validPaths; + + public DownloadedRootPathFilter( + IConfiguration configuration, + IFileSystemUtils fileSystemUtils, + ILogger logger) + { + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + } + + /// + /// Returns true if filePath is present in root path filters. + /// + /// For example, say filePath is /root/parent1/parent2/child1/child2.txt, then if the root path + /// filters contains /root/parent1/ or /root/parent1/parent2/ in it, this filePath with return true, + /// but if the root path contains /root/parent3/, this filePath will return false. + /// + /// + /// The file path to validate + /// + public bool IsValid(string filePath) + { + if (skipValidation) + { + return true; + } + + if (string.IsNullOrEmpty(filePath)) + { + return false; + } + + bool isValid = false; + var normalizedPath = new FileInfo(filePath).FullName; + + foreach (var validPath in validPaths) + { + isValid |= normalizedPath.StartsWith(validPath, StringComparison.InvariantCultureIgnoreCase); + } + + return isValid; + } + + /// + /// Initializes the root path filters list. + /// + public void Init() + { + logger.Verbose("Adding root path filter valid paths"); + skipValidation = true; + + if (configuration.RootPathFilter != null) + { + skipValidation = false; + validPaths = new HashSet(); + string[] relativeRootPaths = configuration.RootPathFilter.Value.Split(';'); + + validPaths.UnionWith(relativeRootPaths.Select(r => + new FileInfo(fileSystemUtils.JoinPaths(configuration.BuildDropPath.Value, r)) + .FullName)); + + foreach (var validPath in validPaths) + { + logger.Verbose($"Added valid path {validPath}"); + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Filters/IFilter.cs b/src/Microsoft.Sbom.Api/Filters/IFilter.cs new file mode 100644 index 00000000..89cd9159 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Filters/IFilter.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Filters +{ + public interface IFilter + { + bool IsValid(string filePath); + + void Init(); + } +} diff --git a/src/Microsoft.Sbom.Api/Filters/ManifestFolderFilter.cs b/src/Microsoft.Sbom.Api/Filters/ManifestFolderFilter.cs new file mode 100644 index 00000000..ce3644e6 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Filters/ManifestFolderFilter.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Filters +{ + public class ManifestFolderFilter : IFilter + { + private readonly IConfiguration configuration; + private readonly IFileSystemUtils fileSystemUtils; + private readonly IOSUtils osUtils; + private string manifestFolderPath; + + public ManifestFolderFilter( + IConfiguration configuration, + IFileSystemUtils fileSystemUtils, + IOSUtils osUtils) + { + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + this.osUtils = osUtils ?? throw new ArgumentNullException(nameof(osUtils)); + } + + public bool IsValid(string filePath) + { + if (string.IsNullOrEmpty(filePath)) + { + return false; + } + + var normalizedPath = new FileInfo(filePath).FullName; + + return !normalizedPath.StartsWith(manifestFolderPath, osUtils.GetFileSystemStringComparisonType()); + } + + public void Init() + { + manifestFolderPath = new FileInfo(configuration.ManifestDirPath.Value).FullName; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/Algorithms/IHashAlgorithm.cs b/src/Microsoft.Sbom.Api/Hashing/Algorithms/IHashAlgorithm.cs new file mode 100644 index 00000000..a79fa2d2 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/Algorithms/IHashAlgorithm.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; + +namespace Microsoft.Sbom.Api.Hashing.Algorithms +{ + /// + /// Provides a hashing algorithm implementation that can be used + /// to generate the hash for a given string. + /// + internal interface IHashAlgorithm + { + /// + /// Returns a byte array of the content using the current hash algorithm. + /// + /// The read stream of the content to be hashed. + /// A byte array of the hash value. + byte[] ComputeHash(Stream inputStream); + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha1HashAlgorithm.cs b/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha1HashAlgorithm.cs new file mode 100644 index 00000000..f3e3a957 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha1HashAlgorithm.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; +using System.Security.Cryptography; + +namespace Microsoft.Sbom.Api.Hashing.Algorithms +{ + /// + /// The hash algorithm implementation of the hash type + /// +#pragma warning disable CA5350 // Suppress Do Not Use Weak Cryptographic Algorithms as we use SHA1 intentionally + public class Sha1HashAlgorithm : IHashAlgorithm + { + public byte[] ComputeHash(Stream stream) => SHA1.Create().ComputeHash(stream); + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha256HashAlgorithm.cs b/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha256HashAlgorithm.cs new file mode 100644 index 00000000..2d731c07 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/Algorithms/Sha256HashAlgorithm.cs @@ -0,0 +1,16 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; +using System.Security.Cryptography; + +namespace Microsoft.Sbom.Api.Hashing.Algorithms +{ + /// + /// The hash algorithm implementation of the hash type. + /// + public class Sha256HashAlgorithm : IHashAlgorithm + { + public byte[] ComputeHash(Stream stream) => SHA256.Create().ComputeHash(stream); + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/HashAlgorithmProvider.cs b/src/Microsoft.Sbom.Api/Hashing/HashAlgorithmProvider.cs new file mode 100644 index 00000000..dbe684b6 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/HashAlgorithmProvider.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Contracts.Interfaces; +using System; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Hashing +{ + public class HashAlgorithmProvider : IHashAlgorithmProvider + { + private readonly IAlgorithmNames[] algorithmNamesList; + private readonly Dictionary algorithmNameMap; + + public HashAlgorithmProvider(IAlgorithmNames[] algorithmNamesList) + { + this.algorithmNamesList = algorithmNamesList ?? throw new ArgumentNullException(nameof(algorithmNamesList)); + algorithmNameMap = new Dictionary(); + Init(); + } + + public void Init() + { + foreach (var algorithmNames in algorithmNamesList) + { + foreach (var algorithmName in algorithmNames.GetAlgorithmNames()) + { + algorithmNameMap[algorithmName.Name.ToLowerInvariant()] = algorithmName; + } + } + } + + public AlgorithmName Get(string algorithmName) + { + if (string.IsNullOrWhiteSpace(algorithmName)) + { + throw new ArgumentException($"'{nameof(algorithmName)}' cannot be null or whitespace.", nameof(algorithmName)); + } + + if (algorithmNameMap.TryGetValue(algorithmName.ToLowerInvariant(), out AlgorithmName value)) + { + return value; + } + + throw new UnsupportedHashAlgorithmException($"Unsupported hash algorithm {algorithmName}"); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/HashCodeGenerator.cs b/src/Microsoft.Sbom.Api/Hashing/HashCodeGenerator.cs new file mode 100644 index 00000000..aebe0b0a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/HashCodeGenerator.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.IO; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; + +namespace Microsoft.Sbom.Api.Hashing +{ + /// + /// Generates a list of for the given file. + /// + public class HashCodeGenerator : IHashCodeGenerator + { + private readonly IFileSystemUtils fileSystemUtils; + + public HashCodeGenerator(IFileSystemUtils fileSystemUtils) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + /// + /// Given a file path, returns a list of for the file + /// for each hash algorithm name provided in + /// + /// The path of the file + /// A list of the hash algorithms for which hashes will be generated. + /// A list of + public Checksum[] GenerateHashes(string filePath, AlgorithmName[] hashAlgorithmNames) + { + var fileHashes = new Checksum[hashAlgorithmNames.Length]; + int i = 0; + + using var bufferedStream = new BufferedStream(fileSystemUtils.OpenRead(filePath), 1024 * 32); + + foreach (var hashAlgorithmName in hashAlgorithmNames) + { + var checksum = hashAlgorithmName.ComputeHash(bufferedStream); + + fileHashes[i++] = new Checksum + { + Algorithm = hashAlgorithmName, + + // TODO make this be bytes instead of converting to string. + ChecksumValue = BitConverter.ToString(checksum).Replace("-", string.Empty) + }; + + // Seek to origin for the next hashing algorithm + // TODO check if using multiple streams is cheaper than resuing the same stream. + bufferedStream.Seek(0, SeekOrigin.Begin); + } + + return fileHashes; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/IHashAlgorithmProvider.cs b/src/Microsoft.Sbom.Api/Hashing/IHashAlgorithmProvider.cs new file mode 100644 index 00000000..905ab80b --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/IHashAlgorithmProvider.cs @@ -0,0 +1,12 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Contracts.Enums; + +namespace Microsoft.Sbom.Api.Hashing +{ + public interface IHashAlgorithmProvider + { + AlgorithmName Get(string algorithmName); + } +} diff --git a/src/Microsoft.Sbom.Api/Hashing/IHashCodeGenerator.cs b/src/Microsoft.Sbom.Api/Hashing/IHashCodeGenerator.cs new file mode 100644 index 00000000..d075f436 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Hashing/IHashCodeGenerator.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; + +namespace Microsoft.Sbom.Api.Hashing +{ + public interface IHashCodeGenerator + { + /// + /// Given a file path, returns a list of for the file + /// for each hash algorithm name provided in + /// + /// The path of the file + /// A list of the hash algorithms for which hashes will be generated. + /// A list of + Checksum[] GenerateHashes(string filePath, AlgorithmName[] hashAlgorithmNames); + } +} diff --git a/src/Microsoft.Sbom.Api/Logging/LoggerProvider.cs b/src/Microsoft.Sbom.Api/Logging/LoggerProvider.cs new file mode 100644 index 00000000..64fbe618 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Logging/LoggerProvider.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Common.Config; +using Ninject.Activation; +using Serilog; +using Serilog.Core; + +namespace Microsoft.Sbom.Api.Logging +{ + /// + /// Configures and returns a object. + /// + public class LoggerProvider : Provider + { + private readonly IConfiguration configuration; + + public LoggerProvider(IConfiguration configuration) + { + this.configuration = configuration; + } + + protected override ILogger CreateInstance(IContext context) + { + return new LoggerConfiguration() + .MinimumLevel.ControlledBy(new LoggingLevelSwitch { MinimumLevel = configuration.Verbosity.Value }) + .WriteTo.Console(outputTemplate: "##[{Level:w}]{Message}{NewLine}{Exception}") + .CreateLogger(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/Configuration/SBOMConfig.cs b/src/Microsoft.Sbom.Api/Manifest/Configuration/SBOMConfig.cs new file mode 100644 index 00000000..cc3280ff --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/Configuration/SBOMConfig.cs @@ -0,0 +1,129 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Common; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Manifest.Configuration +{ + /// + /// Represents a configuration object for a given SBOM Format. It holds all the + /// relevant serializers and generation data for the given SBOM format. + /// + public class SbomConfig : ISbomConfig, IDisposable, IAsyncDisposable + { + private Stream fileStream; + private readonly IFileSystemUtils fileSystemUtils; + + public SbomConfig(IFileSystemUtils fileSystemUtils) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + /// + /// Gets or sets absolute path of manifest json directory. + /// + public string ManifestJsonDirPath { get; set; } + + /// + /// Gets or sets absolute path of the manfest json file. + /// + public string ManifestJsonFilePath { get; set; } + + /// + /// Gets or sets derived manifestInfo or from configurations. + /// + public ManifestInfo ManifestInfo { get; set; } + + /// + /// Gets or sets the metadata builder for this manifest format. + /// + public IMetadataBuilder MetadataBuilder { get; set; } + + /// + /// Gets the generated manifest tool json serializer for this SBOM config. + /// + public IManifestToolJsonSerializer JsonSerializer { get; private set; } + + /// + /// Gets or sets records ids and generated package details for the current SBOM. + /// + public ISbomPackageDetailsRecorder Recorder { get; set; } + + public void StartJsonSerialization() + { + if (ManifestJsonDirPath == null) + { + throw new ArgumentNullException(nameof(ManifestJsonDirPath)); + } + + if (ManifestJsonFilePath == null) + { + throw new ArgumentNullException(nameof(ManifestJsonFilePath)); + } + + fileSystemUtils.CreateDirectory(ManifestJsonDirPath); + fileStream = fileSystemUtils.OpenWrite(ManifestJsonFilePath); + JsonSerializer = new ManifestToolJsonSerializer(fileStream); + } + + #region Disposable implementation + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsyncCore(); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + (JsonSerializer as IDisposable)?.Dispose(); + (fileStream as IDisposable)?.Dispose(); + } + + fileStream = null; + JsonSerializer = null; + } + + protected virtual async ValueTask DisposeAsyncCore() + { + if (JsonSerializer is IAsyncDisposable jsonSerializerDisposable) + { + await jsonSerializerDisposable.DisposeAsync().ConfigureAwait(false); + } + else + { + JsonSerializer?.Dispose(); + } + + if (fileStream is IAsyncDisposable fileStreamDisposable) + { + await fileStreamDisposable.DisposeAsync().ConfigureAwait(false); + } + else + { + fileStream?.Dispose(); + } + + fileStream = null; + JsonSerializer = null; + } + + #endregion + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigFactory.cs b/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigFactory.cs new file mode 100644 index 00000000..22a75bc8 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigFactory.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common; +using System; + +namespace Microsoft.Sbom.Api.Manifest.Configuration +{ + public class SbomConfigFactory : ISbomConfigFactory + { + private readonly IFileSystemUtils fileSystemUtils; + + public SbomConfigFactory(IFileSystemUtils fileSystemUtils) + { + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + public ISbomConfig Get( + ManifestInfo manifestInfo, + string manifestDirPath, + string manifestFilePath, + ISbomPackageDetailsRecorder recorder, + IMetadataBuilder metadataBuilder) + { + return new SbomConfig(fileSystemUtils) + { + ManifestInfo = manifestInfo, + ManifestJsonDirPath = manifestDirPath, + ManifestJsonFilePath = manifestFilePath, + MetadataBuilder = metadataBuilder, + Recorder = recorder + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigProvider.cs b/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigProvider.cs new file mode 100644 index 00000000..8ffb69ec --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/Configuration/SbomConfigProvider.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Metadata; +using Microsoft.Sbom.Api.Output.Telemetry; +using Serilog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Manifest.Configuration +{ + /// + /// Provides a list of configs for all the SBOM formats that need to be generated. + /// We might need to generate more than one SBOM for backward compatibility. + /// + public class SbomConfigProvider : ISbomConfigProvider + { + private readonly IDictionary configsDictionary = new Dictionary(); + private readonly IMetadataProvider[] metadataProviders; + private readonly ILogger logger; + private readonly IReadOnlyDictionary metadataDictionary; + + public SbomConfigProvider( + IManifestConfigHandler[] manifestConfigHandlers, + IMetadataProvider[] metadataProviders, + ILogger logger, + IRecorder recorder) + { + if (manifestConfigHandlers is null) + { + throw new ArgumentNullException(nameof(manifestConfigHandlers)); + } + + foreach (var configHandler in manifestConfigHandlers) + { + if (configHandler.TryGetManifestConfig(out ISbomConfig sbomConfig)) + { + configsDictionary.Add(sbomConfig.ManifestInfo, sbomConfig); + recorder.RecordSBOMFormat(sbomConfig.ManifestInfo, sbomConfig.ManifestJsonFilePath); + } + } + + this.metadataProviders = metadataProviders ?? throw new ArgumentNullException(nameof(metadataProviders)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + + metadataDictionary = InitializeDictionary(); + } + + /// + public ISbomConfig Get(ManifestInfo manifestInfo) + { + if (manifestInfo is null) + { + throw new ArgumentNullException(nameof(manifestInfo)); + } + + return configsDictionary[manifestInfo]; + } + + /// + public bool TryGet(ManifestInfo manifestInfo, out ISbomConfig sbomConfig) + { + if (manifestInfo is null) + { + throw new ArgumentNullException(nameof(manifestInfo)); + } + + return configsDictionary.TryGetValue(manifestInfo, out sbomConfig); + } + + public IEnumerable GetManifestInfos() + { + return configsDictionary.Keys; + } + + public IDisposable StartJsonSerialization() + { + ApplyToEachConfig(c => c.StartJsonSerialization()); + return this; + } + + public IAsyncDisposable StartJsonSerializationAsync() + { + ApplyToEachConfig(c => c.StartJsonSerialization()); + return this; + } + + /// + /// Helper method to operate an action on each included configs. + /// + /// The action to perform on the config. + public void ApplyToEachConfig(Action action) + { + foreach (var config in configsDictionary) + { + action(config.Value); + } + } + + #region IInternalMetadataProvider implementation + + private IReadOnlyDictionary InitializeDictionary() + { + try + { + return metadataProviders + .Select(md => md.MetadataDictionary) + .SelectMany(dict => dict) + .Where(kvp => kvp.Value != null) + .GroupBy(kvp => kvp.Key, kvp => kvp.Value) + .ToDictionary(g => g.Key, g => g.First()); + } + catch (ArgumentException e) + { + // Sanitize exceptions. + throw new Exception($"An error occured while creating metadata entries for the SBOM.", e); + } + } + + public object GetMetadata(MetadataKey key) + { + if (metadataDictionary.TryGetValue(key, out object value)) + { + logger.Debug($"Found value for header {key} in internal metadata."); + return value; + } + + throw new Exception($"Value for header {key} not found in internal metadata"); + } + + public bool TryGetMetadata(MetadataKey key, out object value) + { + if (metadataDictionary.ContainsKey(key)) + { + logger.Debug($"Found value for header {key} in internal metadata."); + value = metadataDictionary[key]; + return true; + } + + value = null; + return false; + } + + public GenerationData GetGenerationData(ManifestInfo manifestInfo) + { + if (configsDictionary.TryGetValue(manifestInfo, out ISbomConfig sbomConfig)) + { + return sbomConfig.Recorder.GetGenerationData(); + } + + throw new Exception($"Unable to get generation data for the {manifestInfo} SBOM."); + } + + public string GetSBOMNamespaceUri() + { + IMetadataProvider provider = null; + if (metadataDictionary.TryGetValue(MetadataKey.BuildEnvironmentName, out object buildEnvironmentName)) + { + provider = metadataProviders + .Where(p => p.BuildEnvironmentName != null && p.BuildEnvironmentName == buildEnvironmentName as string) + .FirstOrDefault(); + } + else + { + provider = metadataProviders.FirstOrDefault(p => p is IDefaultMetadataProvider); + } + + if (provider != null) + { + return provider.GetDocumentNamespaceUri(); + } + + logger.Error($"Unable to find any provider to generate the namespace."); + throw new Exception($"Unable to find any provider to generate the namespace."); + } + + #endregion + + #region Disposable implementation + + public void Dispose() + { + Dispose(disposing: true); + GC.SuppressFinalize(this); + } + + public async ValueTask DisposeAsync() + { + await DisposeAsyncCore(); + + Dispose(disposing: false); + GC.SuppressFinalize(this); + } + + protected virtual void Dispose(bool disposing) + { + if (disposing) + { + ApplyToEachConfig(c => c.Dispose()); + } + } + + protected virtual async ValueTask DisposeAsyncCore() + { + foreach (var config in configsDictionary) + { + await config.Value.DisposeAsync().ConfigureAwait(false); + } + } + + #endregion + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs new file mode 100644 index 00000000..206ccbae --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigHandlers/SPDX22ManifestConfigHandler.cs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using System; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Recorder; +using Microsoft.Sbom.Common; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers +{ + /// + /// Provides the ManifestConfig for the SPDX 2.2 format. + /// + public class SPDX22ManifestConfigHandler : IManifestConfigHandler + { + private readonly IConfiguration configuration; + private readonly IFileSystemUtils fileSystemUtils; + private readonly IMetadataBuilder metadataBuilder; + + private readonly string sbomDirPath; + private readonly string sbomFilePath; + + public SPDX22ManifestConfigHandler( + IConfiguration configuration, + IFileSystemUtils fileSystemUtils, + IMetadataBuilderFactory metadataBuilderFactory) + { + if (metadataBuilderFactory is null) + { + throw new ArgumentNullException(nameof(metadataBuilderFactory)); + } + + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + + string manifestDirPath = configuration.ManifestDirPath.Value; + + // directory path for SPDX 2.2 is + // root/_manifest/spdx_2.2/ + sbomDirPath = fileSystemUtils.JoinPaths(manifestDirPath, $"{Constants.SPDX22ManifestInfo.Name.ToLower()}_{Constants.SPDX22ManifestInfo.Version.ToLower()}"); + + // sbom file path is manifest.spdx.json in the sbom directory. + sbomFilePath = fileSystemUtils.JoinPaths(sbomDirPath, $"manifest.{Constants.SPDX22ManifestInfo.Name.ToLower()}.json"); + + metadataBuilder = metadataBuilderFactory.Get(Constants.SPDX22ManifestInfo); + } + + public bool TryGetManifestConfig(out ISbomConfig sbomConfig) + { + sbomConfig = new SbomConfig(fileSystemUtils) + { + ManifestInfo = Constants.SPDX22ManifestInfo, + ManifestJsonDirPath = sbomDirPath, + ManifestJsonFilePath = sbomFilePath, + MetadataBuilder = metadataBuilder, + Recorder = new SbomPackageDetailsRecorder() + }; + + // For generation the default behavior is to always return true + // as we generate all the current formats of SBOM. Only override if the -mi + // argument is specified. + if (configuration.ManifestToolAction == ManifestToolActions.Generate) + { + if (configuration.ManifestInfo?.Value != null + && !configuration.ManifestInfo.Value.Contains(Constants.SPDX22ManifestInfo)) + { + return false; + } + + return true; + } + + if (configuration.ManifestToolAction == ManifestToolActions.Validate + && fileSystemUtils.FileExists(sbomFilePath)) + { + // Even if we find a valid SPDX 2.2 SBOM, we should not return + // the SPDX validator as it is not implemented yet. + sbomConfig = null; + return false; + } + + sbomConfig = null; + return false; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestConfigProvider.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigProvider.cs new file mode 100644 index 00000000..3078589b --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestConfigProvider.cs @@ -0,0 +1,42 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject.Activation; +using PowerArgs; +using System; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Manifest +{ + /// + /// Provides the by deriving manifest configs for the operation. + /// + public class ManifestConfigProvider : Provider + { + private readonly IManifestConfigHandler[] manifestConfigHandlers; + + public ManifestConfigProvider(IManifestConfigHandler[] manifestConfigHandlers) + { + this.manifestConfigHandlers = manifestConfigHandlers ?? throw new ArgumentNullException(nameof(manifestConfigHandlers)); + } + + /// + /// Provides the object for the given SBOM format. + /// + /// + /// + protected override ISbomConfig CreateInstance(IContext context) + { + // Get the first usable handler. + foreach (var configHandler in manifestConfigHandlers) + { + if (configHandler.TryGetManifestConfig(out ISbomConfig sbomConfig)) + { + return sbomConfig; + } + } + + throw new ValidationArgException($"Unable to find a valid SBOM parser for the current SBOM format."); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestDataProvider.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestDataProvider.cs new file mode 100644 index 00000000..36a71031 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestDataProvider.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts; +using Ninject; +using Ninject.Activation; +using System.Collections.Concurrent; +using Microsoft.Sbom.Common.Config; +using System.Linq; + +namespace Microsoft.Sbom.Api.Manifest +{ + /// + /// Provides the from a given manifest file. + /// + public class ManifestDataProvider : Provider + { + private readonly IFileSystemUtils fileSystemUtils; + private readonly IOSUtils osUtils; + private readonly ISbomConfigProvider sbomConfigs; + private readonly IConfiguration configuration; + + public ManifestDataProvider(IFileSystemUtils fileSystemUtils, ISbomConfigProvider sbomConfigs, IOSUtils osUtils, IConfiguration configuration) + { + this.fileSystemUtils = fileSystemUtils ?? throw new System.ArgumentNullException(nameof(fileSystemUtils)); + this.sbomConfigs = sbomConfigs ?? throw new System.ArgumentNullException(nameof(sbomConfigs)); + this.osUtils = osUtils ?? throw new System.ArgumentNullException(nameof(osUtils)); + this.configuration = configuration ?? throw new System.ArgumentNullException(nameof(configuration)); + } + + /// + /// Uses the manifest parser provider to select the correct parser. + /// Converts the dictionary inside the into a case insensitive + /// concurrent dictionary + /// + /// + /// + /// + protected override ManifestData CreateInstance(IContext context) + { + var sbomConfig = sbomConfigs.Get(configuration.ManifestInfo?.Value?.FirstOrDefault()); + var parserProvider = context.Kernel.Get(); + var manifestValue = fileSystemUtils.ReadAllText(sbomConfig.ManifestJsonFilePath); + var manifestData = parserProvider.Get(sbomConfig.ManifestInfo).ParseManifest(manifestValue); + manifestData.HashesMap = new ConcurrentDictionary(manifestData.HashesMap, osUtils.GetFileSystemStringComparer()); + + return manifestData; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestGeneratorProvider.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestGeneratorProvider.cs new file mode 100644 index 00000000..941c4155 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestGeneratorProvider.cs @@ -0,0 +1,58 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using System; +using System.Collections.Generic; +using System.Linq; +using Microsoft.Sbom.Api.Exceptions; + +namespace Microsoft.Sbom.Api.Manifest +{ + /// + /// Factory class that returns the correct implementation of the + /// at runtime based on the 'ManifestInfo' parameter. + /// + public class ManifestGeneratorProvider + { + private readonly IManifestGenerator[] manifestGenerators; + private readonly IDictionary manifestMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + + public ManifestGeneratorProvider(IManifestGenerator[] manifestGenerators) + { + this.manifestGenerators = manifestGenerators ?? Array.Empty(); + } + + public void Init() + { + foreach (var manifestGenerator in manifestGenerators) + { + var manifestFormat = manifestGenerator.RegisterManifest(); + manifestMap[$"{manifestFormat.Name}:{manifestFormat.Version}"] = manifestGenerator; + } + } + + public IManifestGenerator Get(ManifestInfo manifestInfo) + { + var key = $"{manifestInfo.Name}:{manifestInfo.Version}"; + if (manifestMap.TryGetValue(key, out IManifestGenerator generator)) + { + return generator; + } + + throw new MissingGeneratorException($"The SBOM format '{key}' is not supported by the SBOM tool"); + } + + public IEnumerable GetSupportedManifestInfos() + { + var manifestInfoList = new List(); + foreach (var miString in manifestMap.Keys) + { + manifestInfoList.Add(ManifestInfo.Parse(miString)); + } + + return manifestInfoList; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Manifest/ManifestParserProvider.cs b/src/Microsoft.Sbom.Api/Manifest/ManifestParserProvider.cs new file mode 100644 index 00000000..2c62b107 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Manifest/ManifestParserProvider.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using System; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Manifest +{ + /// + /// Builds a map of s to the actual objects. + /// + public class ManifestParserProvider + { + private readonly IManifestInterface[] manifestInterfaces; + private readonly IDictionary manifestMap; + + public ManifestParserProvider(IManifestInterface[] manifestInterfaces) + { + this.manifestInterfaces = manifestInterfaces; + manifestMap = new Dictionary(StringComparer.OrdinalIgnoreCase); + } + + public void Init() + { + foreach (var manifestInterface in manifestInterfaces) + { + var supportedManifestFormats = manifestInterface.RegisterManifest(); + foreach (var manifestFormat in supportedManifestFormats) + { + // TODO implement getHashCode() in manifest interface. + manifestMap[$"{manifestFormat.Name}:{manifestFormat.Version}"] = manifestInterface; + } + } + } + + public IManifestInterface Get(ManifestInfo manifestInfo) + { + return manifestMap[$"{manifestInfo.Name}:{manifestInfo.Version}"]; + } + } +} diff --git a/src/Microsoft.Sbom.Api/MetaData/LocalMetadataProvider.cs b/src/Microsoft.Sbom.Api/MetaData/LocalMetadataProvider.cs new file mode 100644 index 00000000..9d855841 --- /dev/null +++ b/src/Microsoft.Sbom.Api/MetaData/LocalMetadataProvider.cs @@ -0,0 +1,77 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common.Extensions; +using Microsoft.Sbom.Common.Utils; +using System; +using System.Collections.Generic; +using System.Reflection; + +namespace Microsoft.Sbom.Api.Metadata +{ + /// + /// Provides metadata based on the local environment. + /// + public class LocalMetadataProvider : IMetadataProvider, IDefaultMetadataProvider + { + private const string ProductName = "Microsoft.SBOMTool"; + private const string buildEnvironmentName = "local"; + + private static readonly Lazy version = new Lazy(() => + { + return typeof(LocalMetadataProvider).GetTypeInfo().Assembly.GetCustomAttribute()?.InformationalVersion ?? ""; + }); + + public string BuildEnvironmentName => buildEnvironmentName; + + private readonly IConfiguration configuration; + + /// + /// Gets or sets stores the metadata that is generated by this metadata provider. + /// + public IDictionary MetadataDictionary { get; set; } = new Dictionary(); + + public LocalMetadataProvider(IConfiguration configuration) + { + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + PopulateMetadata(); + } + + private void PopulateMetadata() + { + MetadataDictionary.Add(MetadataKey.SBOMToolName, ProductName); + + // TODO get tool version from dll manifest. + MetadataDictionary.Add(MetadataKey.SBOMToolVersion, version.Value); + + // Add the package name if available. + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.PackageName, configuration.PackageName?.Value); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.PackageVersion, configuration.PackageVersion?.Value); + + // Add generation timestamp + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.GenerationTimestamp, configuration.GenerationTimestamp?.Value); + } + + public string GetDocumentNamespaceUri() + { + // This is used when we can't determine the build environment. So, use a guid and package information + // to generate the namespace. + var packageName = MetadataDictionary[MetadataKey.PackageName]; + var packageVersion = MetadataDictionary[MetadataKey.PackageVersion]; + var uniqueNsPart = configuration.NamespaceUriUniquePart?.Value ?? IdentifierUtils.GetShortGuid(Guid.NewGuid()); + + return Uri.EscapeUriString(string.Join("/", configuration.NamespaceUriBase.Value, packageName, packageVersion, uniqueNsPart)); + } + } + + /// + /// Marker interface to indicate that this metadata provider should be the provider of last resort when another cannot be located. + /// + /// Only one class should implement this interface, as it defines the one and only default provider. + internal interface IDefaultMetadataProvider + { + } +} diff --git a/src/Microsoft.Sbom.Api/MetaData/SBOMApiMetadataProvider.cs b/src/Microsoft.Sbom.Api/MetaData/SBOMApiMetadataProvider.cs new file mode 100644 index 00000000..151943d6 --- /dev/null +++ b/src/Microsoft.Sbom.Api/MetaData/SBOMApiMetadataProvider.cs @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common.Extensions; +using Microsoft.Sbom.Common.Utils; +using Microsoft.Sbom.Contracts; +using System; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Metadata +{ + /// + /// Provides metadata provided by the user from the SBOM Api. + /// + public class SBOMApiMetadataProvider : IMetadataProvider + { + private readonly SBOMMetadata metadata; + private readonly IConfiguration configuration; + + public SBOMApiMetadataProvider(SBOMMetadata metadata, IConfiguration configuration) + { + this.metadata = metadata ?? throw new ArgumentNullException(nameof(metadata)); + this.configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); + PopulateMetadata(); + } + + /// + /// Gets or sets stores the metadata that is generated by this metadata provider. + /// + public IDictionary MetadataDictionary { get; set; } = new Dictionary(); + + public string BuildEnvironmentName + { + get + { + if (MetadataDictionary.TryGetValue(MetadataKey.BuildEnvironmentName, out var name)) + { + return name as string; + } + + return null; + } + } + + public string GetDocumentNamespaceUri() + { + var nsUniquePart = configuration.NamespaceUriUniquePart?.Value ?? IdentifierUtils.GetShortGuid(Guid.NewGuid()); + var packageName = MetadataDictionary[MetadataKey.PackageName]; + var packageVersion = MetadataDictionary[MetadataKey.PackageVersion]; + + return Uri.EscapeUriString(string.Join("/", configuration.NamespaceUriBase.Value, packageName, packageVersion, nsUniquePart)); + } + + private void PopulateMetadata() + { + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.Build_BuildId, metadata.BuildId); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.PackageName, metadata.PackageName); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.PackageVersion, metadata.PackageVersion); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.Build_DefinitionName, metadata.BuildName); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.Build_Repository_Uri, metadata.RepositoryUri); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.Build_SourceBranchName, metadata.Branch); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.Build_SourceVersion, metadata.CommitId); + MetadataDictionary.AddIfKeyNotPresentAndValueNotNull(MetadataKey.BuildEnvironmentName, metadata.BuildEnvironmentName); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj new file mode 100644 index 00000000..5b98aaf2 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Microsoft.Sbom.Api.csproj @@ -0,0 +1,49 @@ + + + + netcoreapp3.1 + Microsoft.Sbom.Api + True + + + + + + + + + + + + + + + + + + + + + + + + + <_Parameter1>$(AssemblyName).Tests + + + + + + PreserveNewest + + + + + + + + + + + + \ No newline at end of file diff --git a/src/Microsoft.Sbom.Api/Output/FileOutputWriter.cs b/src/Microsoft.Sbom.Api/Output/FileOutputWriter.cs new file mode 100644 index 00000000..fe45e060 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/FileOutputWriter.cs @@ -0,0 +1,30 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; +using System.Threading.Tasks; +using Microsoft.Sbom.Common.Config; + +namespace Microsoft.Sbom.Api.Output +{ + /// + /// Writes a string to a file. + /// TODO Use serilog. + /// + public class FileOutputWriter : IOutputWriter + { + private readonly IConfiguration configuration; + + public FileOutputWriter(IConfiguration configuration) + { + this.configuration = configuration; + } + + public async Task WriteAsync(string output) + { + using FileStream fs = new FileStream(configuration.OutputPath.Value, FileMode.Create); + using StreamWriter outputFile = new StreamWriter(fs); + await outputFile.WriteAsync(output); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/IOutputWriter.cs b/src/Microsoft.Sbom.Api/Output/IOutputWriter.cs new file mode 100644 index 00000000..77105bf8 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/IOutputWriter.cs @@ -0,0 +1,17 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Output +{ + public interface IOutputWriter + { + /// + /// Writes a string to a file asynchronously + /// + /// + /// + Task WriteAsync(string output); + } +} diff --git a/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs b/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs new file mode 100644 index 00000000..fc52a513 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/ManifestToolJsonSerializer.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using System; +using System.IO; +using System.Text.Json; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Output +{ + /// + /// This implements the custom serializer for writing json output by the Manifest + /// tool. This serializer is optimized for writing a lot of array values, and some + /// additional metadata. + /// + /// It holds a object inside which is disposable. + /// + public sealed class ManifestToolJsonSerializer : IManifestToolJsonSerializer + { + private Utf8JsonWriter jsonWriter; + + public ManifestToolJsonSerializer(Stream stream) + { + if (stream is null) + { + throw new ArgumentNullException(nameof(stream)); + } + + jsonWriter = new Utf8JsonWriter(stream, new JsonWriterOptions { Indented = true }); + } + + public void Dispose() + { + jsonWriter?.Dispose(); + jsonWriter = null; + } + + public async ValueTask DisposeAsync() + { + if (jsonWriter != null) + { + await jsonWriter.DisposeAsync().ConfigureAwait(false); + } + + jsonWriter = null; + } + + /// + /// This writes a json document to the underlying stream. + /// We also call dispose on the JsonDocument once we finish writing. + /// + /// The json document + public void Write(JsonDocument jsonDocument) + { + if (jsonDocument == null) + { + return; + } + + using (jsonDocument) + { + jsonDocument.WriteTo(jsonWriter); + } + + // If the pending buffer size is greater than a megabyte, flush the stream. + if (jsonWriter.BytesPending > 1_000_000) + { + jsonWriter.Flush(); + } + } + + /// + /// Write a json string object. This usually is some metadata about the document + /// that is generated. + /// + public void WriteJsonString(string jsonString) + { + if (!string.IsNullOrEmpty(jsonString)) + { + using JsonDocument document = JsonDocument.Parse(jsonString); + foreach (JsonProperty property in document.RootElement.EnumerateObject()) + { + property.WriteTo(jsonWriter); + } + } + } + + /// + /// Writes the start JSON object. Must be called before writing to the serializer. + /// + public void StartJsonObject() => jsonWriter.WriteStartObject(); + + /// + /// Writes the end JSON object. Must be called after finishing writing to + /// the serializer to close the json object. + /// + public void FinalizeJsonObject() => jsonWriter.WriteEndObject(); + + /// + /// Start an array object with the header string as the key. + /// + /// They key to the array. + public void StartJsonArray(string arrayHeader) => jsonWriter.WriteStartArray(JsonEncodedText.Encode(arrayHeader)); + + /// + /// End the current array. Throws if there is currently no array object being written to. + /// + public void EndJsonArray() => jsonWriter.WriteEndArray(); + } +} diff --git a/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs new file mode 100644 index 00000000..4e6086ae --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/MetadataBuilder.cs @@ -0,0 +1,135 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Serilog; +using System; +using System.Text.Json; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Utils; + +namespace Microsoft.Sbom.Api.Output +{ + /// + /// Provides metadata that can be added to an SBOM. + /// as a header. + /// + public class MetadataBuilder : IMetadataBuilder + { + private readonly IManifestGenerator manifestGenerator; + private readonly ILogger logger; + private readonly ManifestInfo manifestInfo; + private readonly IRecorder recorder; + + public MetadataBuilder( + ILogger logger, + ManifestGeneratorProvider manifestGeneratorProvider, + ManifestInfo manifestInfo, + IRecorder recorder) + { + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.manifestInfo = manifestInfo ?? throw new ArgumentNullException(nameof(manifestInfo)); + manifestGenerator = manifestGeneratorProvider + .Get(manifestInfo); + this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + } + + /// + /// Gets the json string value of the header dictionary. + /// + /// + public string GetHeaderJsonString(IInternalMetadataProvider internalMetadataProvider) + { + using (recorder.TraceEvent(string.Format(Events.MetadataBuilder, manifestInfo))) + { + logger.Debug("Building the header object."); + var headerDictionary = manifestGenerator.GetMetadataDictionary(internalMetadataProvider); + return JsonSerializer.Serialize(headerDictionary); + } + } + + public bool TryGetFilesArrayHeaderName(out string headerName) + { + try + { + headerName = manifestGenerator.FilesArrayHeaderName; + return true; + } + catch (NotSupportedException) + { + headerName = null; + logger.Debug("Files array not suppored on this SBOM format."); + return false; + } + } + + public bool TryGetPackageArrayHeaderName(out string headerName) + { + try + { + headerName = manifestGenerator.PackagesArrayHeaderName; + return true; + } + catch (NotSupportedException) + { + headerName = null; + logger.Debug("Packages array not suppored on this SBOM format."); + return false; + } + } + + public bool TryGetExternalRefArrayHeaderName(out string headerName) + { + try + { + headerName = manifestGenerator.ExternalDocumentRefArrayHeaderName; + return true; + } + catch (NotSupportedException) + { + headerName = null; + logger.Debug("External Document Reference array not suppored on this SBOM format."); + return false; + } + } + + public bool TryGetRootPackageJson(IInternalMetadataProvider internalMetadataProvider, out GenerationResult generationResult) + { + try + { + generationResult = manifestGenerator + .GenerateRootPackage(internalMetadataProvider); + + if (generationResult == null) + { + return false; + } + + return true; + } + catch (NotSupportedException) + { + generationResult = null; + logger.Debug("Root package serialization not supported on this SBOM format."); + return false; + } + } + + public bool TryGetRelationshipsHeaderName(out string headerName) + { + try + { + headerName = manifestGenerator.RelationshipsArrayHeaderName; + return headerName != null; + } + catch (NotSupportedException) + { + headerName = null; + logger.Debug("Relationships array are not supported on this SBOM format."); + return false; + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/MetadataBuilderFactory.cs b/src/Microsoft.Sbom.Api/Output/MetadataBuilderFactory.cs new file mode 100644 index 00000000..706a5817 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/MetadataBuilderFactory.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Serilog; +using System; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Metadata; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Output +{ + /// + /// Builds a object for a given SBOM format. + /// + public class MetadataBuilderFactory : IMetadataBuilderFactory + { + private readonly IMetadataProvider[] metadataProviders; + private readonly ILogger logger; + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + private readonly IRecorder recorder; + + public MetadataBuilderFactory( + IMetadataProvider[] metadataProviders, + ILogger logger, + ManifestGeneratorProvider manifestGeneratorProvider, + IRecorder recorder) + { + if (logger is null) + { + throw new ArgumentNullException(nameof(logger)); + } + + if (manifestGeneratorProvider is null) + { + throw new ArgumentNullException(nameof(manifestGeneratorProvider)); + } + + this.metadataProviders = metadataProviders ?? throw new ArgumentNullException(nameof(metadataProviders)); + this.logger = logger; + this.manifestGeneratorProvider = manifestGeneratorProvider; + this.recorder = recorder ?? throw new ArgumentNullException(nameof(recorder)); + } + + public IMetadataBuilder Get(ManifestInfo manifestInfo) + { + return new MetadataBuilder(logger, manifestGeneratorProvider, manifestInfo, recorder); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMFile.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMFile.cs new file mode 100644 index 00000000..ff585851 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMFile.cs @@ -0,0 +1,29 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; + +namespace DropValidator.Api.Output.Telemetry.Entities +{ + /// + /// Represents a SBOM file object and contains additional properties + /// related to the file. + /// + public class SBOMFile + { + /// + /// Gets or sets the name and version of the format of the generated SBOM. + /// + public ManifestInfo SbomFormatName { get; set; } + + /// + /// Gets or sets the path where the final generated SBOM is placed. + /// + public string SbomFilePath { get; set; } + + /// + /// Gets or sets the size of the SBOM file in bytes. + /// + public long FileSizeInBytes { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs new file mode 100644 index 00000000..738d2d5a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/SBOMTelemetry.cs @@ -0,0 +1,59 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using DropValidator.Api.Output.Telemetry.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Entities.Output; + +namespace Microsoft.Sbom.Api.Output.Telemetry.Entities +{ + /// + /// The telemetry that is logged to a file/console for the given SBOM execution. + /// + [Serializable] + public class SBOMTelemetry + { + /// + /// Gets or sets the result of the execution + /// + public Result Result { get; set; } + + /// + /// Gets or sets a list of s that was encountered during the execution. + /// + public ErrorContainer Errors { get; set; } + + /// + /// Gets or sets a list of representing each input parameter used + /// in the validation. + /// + public IConfiguration Parameters { get; set; } + + /// + /// Gets or sets a list of the SBOM formats and related file properties that was used in the + /// generation/validation of the SBOM. + /// + public IList SBOMFormatsUsed { get; set; } + + /// + /// Gets or sets a list of event time durations. + /// + public IList Timings { get; set; } + + /// + /// Gets or sets any internal switches and their value that were used during the execution. + /// A switch can be something that was provided through a configuraiton or an environment + /// variable. + /// + public IDictionary Switches { get; set; } + + /// + /// Gets or sets if any exceptions were thrown, this shows the name of the exception and the error message + /// of the exception. + /// + public IDictionary Exceptions { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/Timing.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/Timing.cs new file mode 100644 index 00000000..1db02052 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/Entities/Timing.cs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; + +namespace Microsoft.Sbom.Api.Output.Telemetry.Entities +{ + /// + /// Records various time spans for a given event. + /// + [Serializable] + public class Timing + { + /// + /// Gets or sets the name of the event. + /// + public string EventName { get; set; } + + /// + /// Gets or sets the duration it took to execute the event. + /// + public string TimeSpan { get; set; } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs new file mode 100644 index 00000000..247f8256 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/IRecorder.cs @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Output.Telemetry +{ + /// + /// Records telemetry for the SBOM tool. + /// + public interface IRecorder + { + /// + /// Start recording the duration of exeuction of the given event. + /// + /// The event name + /// A disposable object. + public TimingRecorder TraceEvent(string eventName); + + /// + /// Record the total errors encountered during the execution of the SBOM tool. + /// + /// A list of errors. + /// If the errors object is null. + public void RecordTotalErrors(IList errors); + + /// + /// Records a SBOM format that we used during the execution of the SBOM tool. + /// + /// The SBOM format as a object + /// The path where the generated SBOM is stored. + /// If the manifestInfo object is null. + public void RecordSBOMFormat(ManifestInfo manifestInfo, string sbomFilePath); + + /// + /// Record a switch that was used during the execution of the SBOM tool. + /// + /// The name of the switch or environment variable. + /// The value of the variable. + /// If the switch name is null or whitespace. + /// If the value is empty. + public void RecordSwitch(string switchName, object value); + + /// + /// Record any exception that was encountered during the exection of the tool. + /// + /// The exception that was encountered. + /// If the exception is null + public void RecordException(Exception exception); + + /// + /// Finalize the recorder, and log the telemetry. + /// + public Task FinalizeAndLogTelemetryAsync(); + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs new file mode 100644 index 00000000..9ff9e16f --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/TelemetryRecorder.cs @@ -0,0 +1,200 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Ninject; +using Serilog; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Tasks; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Entities.Output; +using Microsoft.Sbom.Api.Output.Telemetry.Entities; +using DropValidator.Api.Output.Telemetry.Entities; +using PowerArgs; +using System.IO; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Output.Telemetry +{ + /// + /// Records telemetry for an execution of the SBOM tool. + /// + public class TelemetryRecorder : IRecorder + { + private readonly ConcurrentBag timingRecorders = new ConcurrentBag(); + private readonly IDictionary sbomFormats = new Dictionary(); + private readonly IDictionary switches = new Dictionary(); + private readonly IList exceptions = new List(); + + private IList errors = new List(); + private Result result = Result.Success; + + [Inject] + public IFileSystemUtils FileSystemUtils { get; set; } + + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ILogger Log { get; set; } + + public virtual IList Errors + { + get { return errors; } + } + + /// + /// Start recording the duration of exeuction of the given event. + /// + /// The event name + /// A disposable object. + public TimingRecorder TraceEvent(string eventName) + { + if (string.IsNullOrWhiteSpace(eventName)) + { + throw new ArgumentException($"'{nameof(eventName)}' cannot be null or whitespace.", nameof(eventName)); + } + + TimingRecorder timingRecorder = new TimingRecorder(eventName); + timingRecorders.Add(timingRecorder); + return timingRecorder; + } + + /// + /// Record the total errors encountered during the execution of the SBOM tool. + /// + /// A list of errors. + /// If the errors object is null. + public void RecordTotalErrors(IList errors) + { + this.errors = errors ?? throw new ArgumentNullException(nameof(errors)); + } + + /// + public void RecordSBOMFormat(ManifestInfo manifestInfo, string sbomFilePath) + { + if (manifestInfo is null) + { + throw new ArgumentNullException(nameof(manifestInfo)); + } + + if (string.IsNullOrWhiteSpace(sbomFilePath)) + { + throw new ArgumentException($"'{nameof(sbomFilePath)}' cannot be null or whitespace.", nameof(sbomFilePath)); + } + + this.sbomFormats[manifestInfo] = sbomFilePath; + } + + /// + /// Record any exception that was encountered during the exection of the tool. + /// + /// The exception that was encountered. + /// If the exception is null + public void RecordException(Exception exception) + { + if (exception is null) + { + throw new ArgumentNullException(nameof(exception)); + } + + this.exceptions.Add(exception); + } + + /// + /// Record a switch that was used during the execution of the SBOM tool. + /// + /// The name of the switch or environment variable. + /// The value of the variable. + /// If the switch name is null or whitespace. + /// If the value is empty. + public void RecordSwitch(string switchName, object value) + { + if (string.IsNullOrWhiteSpace(switchName)) + { + throw new ArgumentException($"'{nameof(switchName)}' cannot be null or whitespace.", nameof(switchName)); + } + + if (value is null) + { + throw new ArgumentNullException(nameof(value)); + } + + this.switches.Add(switchName, value); + } + + /// + /// Finalize the recorder, and log the telemetry. + /// + public async Task FinalizeAndLogTelemetryAsync() + { + try + { + // Calculate result + if (this.errors.Any() || this.exceptions.Any()) + { + this.result = Result.Failure; + } + + // Calculate SBOM file sizes. + var sbomFormatsUsed = sbomFormats + .Where(f => File.Exists(f.Value)) + .Select(f => new SBOMFile + { + SbomFilePath = f.Value, + SbomFormatName = f.Key, + FileSizeInBytes = new System.IO.FileInfo(f.Value).Length + }) + .ToList(); + + // Create the telemetry object. + var telemetry = new SBOMTelemetry + { + Result = this.result, + Errors = new ErrorContainer + { + Errors = this.errors, + Count = this.errors.Count + }, + Timings = timingRecorders.Select(t => t.ToTiming()).ToList(), + Parameters = Configuration, + SBOMFormatsUsed = sbomFormatsUsed, + Switches = this.switches, + Exceptions = this.exceptions.ToDictionary(k => k.GetType().ToString(), v => v.Message) + }; + + // Log to logger. + Log.Information("Finished execution of the {Action} workflow {@Telemetry}", Configuration.ManifestToolAction, telemetry); + + // Write to file. + if (!string.IsNullOrWhiteSpace(Configuration.TelemetryFilePath?.Value)) + { + using (var fileStream = FileSystemUtils.OpenWrite(Configuration.TelemetryFilePath.Value)) + { + var options = new JsonSerializerOptions + { + Converters = + { + new JsonStringEnumConverter() + } + }; + await JsonSerializer.SerializeAsync(fileStream, telemetry, options); + Log.Debug($"Wrote telemetry object to path {Configuration.TelemetryFilePath.Value}"); + } + } + } + catch (Exception ex) + { + // We should'nt fail the main workflow due to some failure on the telemetry generation. + // Just log the result and return silently. + Log.Warning($"Failed to log telemetry. Exception: {ex.Message}"); + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Output/Telemetry/TimingRecorder.cs b/src/Microsoft.Sbom.Api/Output/Telemetry/TimingRecorder.cs new file mode 100644 index 00000000..409d597a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Output/Telemetry/TimingRecorder.cs @@ -0,0 +1,61 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Diagnostics; +using Microsoft.Sbom.Api.Output.Telemetry.Entities; + +namespace Microsoft.Sbom.Api.Output.Telemetry +{ + /// + /// Records the elapsed time for a given event. + /// + public class TimingRecorder : IDisposable + { + private readonly string eventName; + private readonly Stopwatch stopWatch; + + /// + /// Record the duration of execution for a given event. + /// + /// The name of the event. + public TimingRecorder(string eventName) + { + if (string.IsNullOrWhiteSpace(eventName)) + { + throw new ArgumentException($"'{nameof(eventName)}' cannot be null or whitespace.", nameof(eventName)); + } + + this.eventName = eventName; + stopWatch = Stopwatch.StartNew(); + } + + public void Dispose() + { + stopWatch.Stop(); + } + + /// + /// Gets a value indicating whether returns true if the timings recorder is currently running. + /// + public bool IsRunning => stopWatch.IsRunning; + + /// + /// Returns a object representation of this event. + /// Make sure the timings recorder is not running before invoking this function. + /// + public Timing ToTiming() + { + if (stopWatch.IsRunning) + { + throw new Exception($"Tried to read event details for an executing event."); + } + + return new Timing + { + EventName = eventName, + TimeSpan = stopWatch.Elapsed.ToString() + }; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/EntityToJsonProviderBase.cs b/src/Microsoft.Sbom.Api/Providers/EntityToJsonProviderBase.cs new file mode 100644 index 00000000..8f81b4c6 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/EntityToJsonProviderBase.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using Serilog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers +{ + /// + /// The base class for all providers. Defines the basic workflow for a entity (file or package). + /// + /// + public abstract class EntityToJsonProviderBase : ISourcesProvider + { + /// + /// Gets or sets the configuration that is used to generate the SBOM. + /// + [Inject] + public IConfiguration Configuration { get; set; } + + /// + /// Gets or sets provides utilities for splitting and merging channel streams. + /// + [Inject] + public ChannelUtils ChannelUtils { get; set; } + + [Inject] + public ILogger Log { get; set; } + + /// + /// Generate a stream for all the entities for each of the required configuration. + /// + /// The configurations for which to generate serialized Json. + /// + public (ChannelReader results, ChannelReader errors) Get(IList requiredConfigs) + { + if (requiredConfigs is null) + { + throw new ArgumentNullException(nameof(requiredConfigs)); + } + + IList> errors = new List>(); + IList> jsonDocResults = + new List>(); + + var (sources, sourceErrors) = GetSourceChannel(); + errors.Add(sourceErrors); + + Log.Debug($"Splitting the workflow into {Configuration.Parallelism.Value} threads."); + var splitSourcesChannels = ChannelUtils.Split(sources, Configuration.Parallelism.Value); + + Log.Debug("Running the generation workflow ..."); + + foreach (var sourceChannel in splitSourcesChannels) + { + var (jsonResults, jsonErrors) = ConvertToJson(sourceChannel, requiredConfigs); + + jsonDocResults.Add(jsonResults); + errors.Add(jsonErrors); + } + + // Write out additional items if any. + var (additionalResults, additionalErrors) = WriteAdditionalItems(requiredConfigs); + if (additionalResults != null) + { + jsonDocResults.Add(additionalResults); + } + + if (additionalErrors != null) + { + errors.Add(additionalErrors); + } + + return (ChannelUtils.Merge(jsonDocResults.ToArray()), ChannelUtils.Merge(errors.ToArray())); + } + + /// + /// Should return true only if the provider type is supported. + /// + /// + /// + public abstract bool IsSupported(ProviderType providerType); + + /// + /// Get a channel reader for type that will give us a stream of objects to process. + /// + /// + protected abstract (ChannelReader entities, ChannelReader errors) GetSourceChannel(); + + /// + /// Given a channel of type return a channel of serialized SBOM Json for the objects. + /// + /// + /// + /// + protected abstract (ChannelReader results, ChannelReader errors) + ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs); + + /// + /// Return any additional Json objects for the given entity. + /// + /// + /// + protected abstract (ChannelReader results, ChannelReader errors) + WriteAdditionalItems(IList requiredConfigs); + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/CGExternalDocumentReferenceProvider.cs b/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/CGExternalDocumentReferenceProvider.cs new file mode 100644 index 00000000..96218ad4 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/CGExternalDocumentReferenceProvider.cs @@ -0,0 +1,81 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Converters; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Utils; +using Ninject; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; + +namespace Microsoft.Sbom.Api.Providers.ExternalDocumentReferenceProviders +{ + /// + /// Provider for external document reference which leverage component detection tool + /// to discover SBOM files. + /// + public class CGExternalDocumentReferenceProvider : EntityToJsonProviderBase + { + [Inject] + public ComponentToExternalReferenceInfoConverter ComponentToExternalReferenceInfoConverter { get; set; } + + [Inject] + public ExternalDocumentReferenceWriter ExternalDocumentReferenceWriter { get; set; } + + [Inject] + public SBOMComponentsWalker SBOMComponentsWalker { get; set; } + + [Inject] + public ExternalReferenceDeduplicator ExternalReferenceDeduplicator { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.ExternalDocumentReference) + { + Log.Debug($"Using the {nameof(CGExternalDocumentReferenceProvider)} provider for the external documents workflow."); + return true; + } + + return false; + } + + protected override (ChannelReader results, ChannelReader errors) ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs) + { + IList> errors = new List>(); + + var (output, convertErrors) = ComponentToExternalReferenceInfoConverter.Convert(sourceChannel); + errors.Add(convertErrors); + output = ExternalReferenceDeduplicator.Deduplicate(output); + + var (jsonDoc, jsonErrors) = ExternalDocumentReferenceWriter.Write(output, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonDoc, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + var (output, cdErrors) = SBOMComponentsWalker.GetComponents(Configuration.BuildComponentPath?.Value); + + if (cdErrors.TryRead(out ComponentDetectorException e)) + { + throw e; + } + + var errors = Channel.CreateUnbounded(); + errors.Writer.Complete(); + return (output, errors); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/ExternalDocumentReferenceProvider.cs b/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/ExternalDocumentReferenceProvider.cs new file mode 100644 index 00000000..cf5c62a8 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/ExternalDocumentReferenceProviders/ExternalDocumentReferenceProvider.cs @@ -0,0 +1,68 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Utils; +using Ninject; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; + +namespace Microsoft.Sbom.Api.Providers.ExternalDocumentReferenceProviders +{ + /// + /// Provider for external document reference. supported only when + /// ExternalDocumentReferenceListFile is provided in the generation arguments + /// + public class ExternalDocumentReferenceProvider : EntityToJsonProviderBase + { + [Inject] + public FileListEnumerator ListWalker { get; set; } + + [Inject] + public ISBOMReaderForExternalDocumentReference SPDXSBOMReaderForExternalDocumentReference { get; set; } + + [Inject] + public ExternalDocumentReferenceWriter ExternalDocumentReferenceWriter { get; set; } + + [Inject] + public ExternalReferenceDeduplicator ExternalReferenceDeduplicator { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.ExternalDocumentReference && !string.IsNullOrWhiteSpace(Configuration.ExternalDocumentReferenceListFile?.Value)) + { + Log.Debug($"Using the {nameof(ExternalDocumentReferenceProvider)} provider for the external documents workflow."); + return true; + } + + return false; + } + + protected override (ChannelReader results, ChannelReader errors) ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs) + { + IList> errors = new List>(); + var (results, parseErrors) = SPDXSBOMReaderForExternalDocumentReference.ParseSBOMFile(sourceChannel); + errors.Add(parseErrors); + results = ExternalReferenceDeduplicator.Deduplicate(results); + var (jsonDoc, jsonErrors) = ExternalDocumentReferenceWriter.Write(results, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonDoc, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + return ListWalker.GetFilesFromList(Configuration.ExternalDocumentReferenceListFile.Value); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/CGScannedExternalDocumentReferenceFileProvider.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/CGScannedExternalDocumentReferenceFileProvider.cs new file mode 100644 index 00000000..8477d3fd --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/CGScannedExternalDocumentReferenceFileProvider.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Api.Converters; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Ninject; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Provider for external document reference which leverage component detection tool + /// to discover SBOM files. + /// + public class CGScannedExternalDocumentReferenceFileProvider : PathBasedFileToJsonProviderBase + { + [Inject] + public ComponentToExternalReferenceInfoConverter ComponentToExternalReferenceInfoConverter { get; set; } + + [Inject] + public ExternalReferenceInfoToPathConverter ExternalReferenceInfoToPathConverter { get; set; } + + [Inject] + public ExternalDocumentReferenceWriter ExternalDocumentReferenceWriter { get; set; } + + [Inject] + public SBOMComponentsWalker SBOMComponentsWalker { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Files) + { + Log.Debug($"Using the {nameof(CGScannedExternalDocumentReferenceFileProvider)} provider for the files workflow."); + return true; + } + + return false; + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + var (sbomOutput, cdErrors) = SBOMComponentsWalker.GetComponents(Configuration.BuildComponentPath?.Value); + IList> errors = new List>(); + + if (cdErrors.TryRead(out ComponentDetectorException e)) + { + throw e; + } + + var (externalRefDocOutput, externalRefDocErrors) = ComponentToExternalReferenceInfoConverter.Convert(sbomOutput); + errors.Add(externalRefDocErrors); + + var (pathOutput, pathErrors) = ExternalReferenceInfoToPathConverter.Convert(externalRefDocOutput); + errors.Add(pathErrors); + + return (pathOutput, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/DirectoryTraversingFileToJsonProvider.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/DirectoryTraversingFileToJsonProvider.cs new file mode 100644 index 00000000..f82f24f0 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/DirectoryTraversingFileToJsonProvider.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using System.Collections.Generic; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Traverse a given folder recursively to generate a list of files to be serialized. + /// + public class DirectoryTraversingFileToJsonProvider : PathBasedFileToJsonProviderBase + { + [Inject] + public DirectoryWalker DirectoryWalker { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Files) + { + // This is the last sources provider we should use, if no other sources have been provided by the user. + // Thus, this condition should be to check that all the remaining configurations for file inputs are null. + if (string.IsNullOrWhiteSpace(Configuration.BuildListFile?.Value) && Configuration.FilesList?.Value == null) + { + Log.Debug($"Using the {nameof(DirectoryTraversingFileToJsonProvider)} provider for the files workflow."); + return true; + } + } + + return false; + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + return DirectoryWalker.GetFilesRecursively(Configuration.BuildDropPath?.Value); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/ExternalDocumentReferenceFileProvider.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/ExternalDocumentReferenceFileProvider.cs new file mode 100644 index 00000000..dd535b78 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/ExternalDocumentReferenceFileProvider.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using System.Collections.Generic; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Provider for external document reference file supported only when + /// ExternalDocumentReferenceListFile is provided in the generation arguments + /// + public class ExternalDocumentReferenceFileProvider : PathBasedFileToJsonProviderBase + { + [Inject] + public FileListEnumerator ListWalker { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Files && !string.IsNullOrWhiteSpace(Configuration.ExternalDocumentReferenceListFile?.Value)) + { + Log.Debug($"Using the {nameof(ExternalDocumentReferenceFileProvider)} provider for the files workflow."); + return true; + } + + return false; + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + if (Configuration.ExternalDocumentReferenceListFile?.Value == null) + { + var emptyList = Channel.CreateUnbounded(); + emptyList.Writer.Complete(); + var errors = Channel.CreateUnbounded(); + errors.Writer.Complete(); + return (emptyList, errors); + } + + return ListWalker.GetFilesFromList(Configuration.ExternalDocumentReferenceListFile.Value); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileListBasedFileToJsonProvider.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileListBasedFileToJsonProvider.cs new file mode 100644 index 00000000..bf689158 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileListBasedFileToJsonProvider.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using System.Collections.Generic; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Takes in a list of files provided in a text file and only serializes those files. + /// The files in the list should be present on the disk and should be inside the build drop folder. + /// + public class FileListBasedFileToJsonProvider : PathBasedFileToJsonProviderBase + { + [Inject] + public FileListEnumerator ListWalker { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Files) + { + // Return true only if the BuildListFile parameter is provided. + if (!string.IsNullOrWhiteSpace(Configuration.BuildListFile?.Value)) + { + Log.Debug($"Using the {nameof(FileListBasedFileToJsonProvider)} provider for the files workflow."); + return true; + } + } + + return false; + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + return ListWalker.GetFilesFromList(Configuration.BuildListFile.Value); + } + + protected override (ChannelReader results, ChannelReader errors) WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileToJsonProviderBase.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileToJsonProviderBase.cs new file mode 100644 index 00000000..9c72f891 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/FileToJsonProviderBase.cs @@ -0,0 +1,79 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// An abstract base class for all files providers. This class defines the main workflow for files generation, which all + /// inheriting classes can modify by overriding the abstract methods. + /// + /// The type of the files channel that is used by the provider. + public abstract class FileToJsonProviderBase : ISourcesProvider + { + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + public ChannelUtils ChannelUtils { get; set; } + + public (ChannelReader results, ChannelReader errors) Get(IList requiredConfigs) + { + IList> errors = new List>(); + IList> jsonDocResults = + new List>(); + + var (files, dirErrors) = GetFilesChannel(); + errors.Add(dirErrors); + + Log.Debug($"Splitting the workflow into {Configuration.Parallelism.Value} threads."); + var splitFilesChannels = ChannelUtils.Split(files, Configuration.Parallelism.Value); + + Log.Debug("Running the files generation workflow ..."); + foreach (var fileChannel in splitFilesChannels) + { + var (jsonDoc, convertErrors) = ConvertToJson(fileChannel, requiredConfigs); + + errors.Add(convertErrors); + jsonDocResults.Add(jsonDoc); + } + + return (ChannelUtils.Merge(jsonDocResults.ToArray()), ChannelUtils.Merge(errors.ToArray())); + } + + /// + /// Get a channel reader for type that will give us a stream of file objects to process. + /// + /// + protected abstract (ChannelReader files, ChannelReader errors) GetFilesChannel(); + + /// + /// Given the files channel of type return a channel of serialized SBOM Json for the file. + /// + /// + /// + /// + protected abstract (ChannelReader files, ChannelReader errors) + ConvertToJson(ChannelReader files, IList requiredConfigs); + + /// + /// Should return true only if the provider type is supported. + /// + /// + /// + public abstract bool IsSupported(ProviderType providerType); + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/PathBasedFileToJsonProviderBase.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/PathBasedFileToJsonProviderBase.cs new file mode 100644 index 00000000..71bf8108 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/PathBasedFileToJsonProviderBase.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Abstract base class for all file path based providers. This assumes that we are getting a list of file + /// paths to process as a string. + /// + public abstract class PathBasedFileToJsonProviderBase : EntityToJsonProviderBase + { + [Inject] + public FileHasher FileHasher { get; set; } + + [Inject] + public ManifestFolderFilterer FileFilterer { get; set; } + + [Inject] + public FileInfoWriter FileHashWriter { get; set; } + + [Inject] + public InternalSBOMFileInfoDeduplicator InternalSBOMFileInfoDeduplicator { get; set; } + + protected override (ChannelReader results, ChannelReader errors) + ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs) + { + IList> errors = new List>(); + + // Filter files + var (filteredFiles, filteringErrors) = FileFilterer.FilterFiles(sourceChannel); + errors.Add(filteringErrors); + + // Generate hash code for the files + var (fileInfos, hashingErrors) = FileHasher.Run(filteredFiles); + errors.Add(hashingErrors); + fileInfos = InternalSBOMFileInfoDeduplicator.Deduplicate(fileInfos); + + var (jsonDocCount, jsonErrors) = FileHashWriter.Write(fileInfos, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonDocCount, ChannelUtils.Merge(errors.ToArray())); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/FilesProviders/SBOMFileBasedFileToJsonProvider.cs b/src/Microsoft.Sbom.Api/Providers/FilesProviders/SBOMFileBasedFileToJsonProvider.cs new file mode 100644 index 00000000..307cbd5e --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/FilesProviders/SBOMFileBasedFileToJsonProvider.cs @@ -0,0 +1,85 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using Microsoft.Sbom.Contracts; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.FilesProviders +{ + /// + /// Serializes a list of objects provided through the API to SBOM Json objects. + /// + public class SBOMFileBasedFileToJsonProvider : EntityToJsonProviderBase + { + /// + /// Gets or sets serializes a object to Json + /// + [Inject] + public FileInfoWriter FileHashWriter { get; set; } + + /// + /// Gets or sets converts a object to a + /// + [Inject] + public SBOMFileToFileInfoConverter SBOMFileToFileInfoConverter { get; set; } + + /// + /// Gets or sets deduplicate FileInfo due to duplications of other providers. + /// + [Inject] + public InternalSBOMFileInfoDeduplicator FileInfoDeduplicator { get; set; } + + /// + /// Returns true only if the fileslist parameter is provided. + /// + /// + /// + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Files) + { + if (Configuration.FilesList?.Value != null && string.IsNullOrWhiteSpace(Configuration.BuildListFile?.Value)) + { + Log.Debug($"Using the {nameof(SBOMFileBasedFileToJsonProvider)} provider for the files workflow."); + return true; + } + } + + return false; + } + + protected override (ChannelReader results, ChannelReader errors) + ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs) + { + IList> errors = new List>(); + + var (fileInfos, hashErrors) = SBOMFileToFileInfoConverter.Convert(sourceChannel); + errors.Add(hashErrors); + fileInfos = FileInfoDeduplicator.Deduplicate(fileInfos); + + var (jsonDocCount, jsonErrors) = FileHashWriter.Write(fileInfos, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonDocCount, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + var listWalker = new ListWalker(); + return listWalker.GetComponents(Configuration.FilesList.Value); + } + + protected override (ChannelReader results, ChannelReader errors) + WriteAdditionalItems(IList requiredConfigs) + { + return (null, null); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/ISourcesProvider.cs b/src/Microsoft.Sbom.Api/Providers/ISourcesProvider.cs new file mode 100644 index 00000000..7235cf86 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/ISourcesProvider.cs @@ -0,0 +1,31 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using System.Threading.Channels; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest.Configuration; + +namespace Microsoft.Sbom.Api.Providers +{ + /// + /// Provides a stream of serialized Json for a given source, like packages or files. + /// + public interface ISourcesProvider + { + /// + /// Generate a stream for all the sources for each of the required configuration. + /// + /// The configurations for which to generate serialized Json. + /// + (ChannelReader results, ChannelReader errors) Get(IList requiredConfigs); + + /// + /// Returns true if this provider is suppored for the provided source. + /// + /// The type of the provider that is required. + /// + bool IsSupported(ProviderType providerType); + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs new file mode 100644 index 00000000..4758c9e1 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CGScannedPackagesProvider.cs @@ -0,0 +1,73 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Ninject; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; + +namespace Microsoft.Sbom.Api.Providers.PackagesProviders +{ + /// + /// Calls the component detector to get a list of packages in the current project and serializes them to Json. + /// + public class CGScannedPackagesProvider : CommonPackagesProvider + { + [Inject] + public ComponentToPackageInfoConverter PackageInfoConverter { get; set; } + + [Inject] + public PackagesWalker PackagesWalker { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Packages) + { + if (Configuration.PackagesList?.Value == null) + { + // If no other packages providers are present, use this one. + Log.Debug($"Using the {nameof(CGScannedPackagesProvider)} provider for the packages workflow."); + return true; + } + } + + return false; + } + + protected override (ChannelReader results, ChannelReader errors) + ConvertToJson( + ChannelReader sourceChannel, + IList requiredConfigs) + { + IList> errors = new List>(); + + var (packageInfos, packageErrors) = PackageInfoConverter.Convert(sourceChannel); + errors.Add(packageErrors); + + var (jsonResults, jsonErrors) = PackageInfoJsonWriter.Write(packageInfos, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonResults, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + var (output, cdErrors) = PackagesWalker.GetComponents(Configuration.BuildComponentPath?.Value); + + if (cdErrors.TryRead(out ComponentDetectorException e)) + { + throw e; + } + + var errors = Channel.CreateUnbounded(); + errors.Writer.Complete(); + return (output, errors); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs new file mode 100644 index 00000000..9aa527a6 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/CommonPackagesProvider.cs @@ -0,0 +1,71 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Providers.PackagesProviders +{ + /// + /// Abstract base class for all packages providers. Provides a list of common packages to be serialized + /// for every SBOM format. + /// + public abstract class CommonPackagesProvider : EntityToJsonProviderBase + { + [Inject] + public ISbomConfigProvider SBOMConfigs { get; set; } + + [Inject] + public PackageInfoJsonWriter PackageInfoJsonWriter { get; set; } + + /// + /// Get common packages that are provided by the build engine. + /// + /// + private Channel GetCommonPackages() + { + var packageInfos = Channel.CreateUnbounded(); + + Task.Run(async () => + { + try + { + if (SBOMConfigs.TryGetMetadata(MetadataKey.ImageOS, out object imageOsObj) && + SBOMConfigs.TryGetMetadata(MetadataKey.ImageVersion, out object imageVersionObj)) + { + Log.Debug($"Adding the image OS package to the packages list as a dependency."); + string name = $"Azure Pipelines Hosted Image {imageOsObj}"; + await packageInfos.Writer.WriteAsync(new SBOMPackage() + { + PackageName = name, + PackageVersion = (string)imageVersionObj, + PackageUrl = "https://github.com/actions/virtual-environments", + Id = $"{name} {(string)imageVersionObj}".Replace(' ', '-'), + Supplier = "Microsoft/GitHub" + }); + } + } + finally + { + packageInfos.Writer.Complete(); + } + }); + + return packageInfos; + } + + protected override (ChannelReader results, ChannelReader errors) + WriteAdditionalItems(IList requiredConfigs) + { + return PackageInfoJsonWriter.Write(GetCommonPackages(), requiredConfigs); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs new file mode 100644 index 00000000..b02d6634 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/PackagesProviders/SBOMPackagesProvider.cs @@ -0,0 +1,55 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Ninject; +using Microsoft.Sbom.Contracts; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.Providers.PackagesProviders +{ + /// + /// Provides a serialized list of packages given a list of + /// + public class SBOMPackagesProvider : CommonPackagesProvider + { + [Inject] + public SBOMPackageToPackageInfoConverter PackageInfoConverter { get; set; } + + public override bool IsSupported(ProviderType providerType) + { + if (providerType == ProviderType.Packages) + { + if (Configuration.PackagesList?.Value != null) + { + Log.Debug($"Using the {nameof(SBOMPackagesProvider)} provider for the packages workflow."); + return true; + } + } + + return false; + } + + protected override (ChannelReader results, ChannelReader errors) ConvertToJson(ChannelReader sourceChannel, IList requiredConfigs) + { + IList> errors = new List>(); + var (convertedSource, conversionErrors) = PackageInfoConverter.Convert(sourceChannel); + errors.Add(conversionErrors); + + var (jsonDocCount, jsonErrors) = PackageInfoJsonWriter.Write(convertedSource, requiredConfigs); + errors.Add(jsonErrors); + + return (jsonDocCount, ChannelUtils.Merge(errors.ToArray())); + } + + protected override (ChannelReader entities, ChannelReader errors) GetSourceChannel() + { + var listWalker = new ListWalker(); + return listWalker.GetComponents(Configuration.PackagesList.Value); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Providers/ProviderType.cs b/src/Microsoft.Sbom.Api/Providers/ProviderType.cs new file mode 100644 index 00000000..9cee9376 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Providers/ProviderType.cs @@ -0,0 +1,26 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +namespace Microsoft.Sbom.Api.Providers +{ + /// + /// The type of provider for a given source. + /// + public enum ProviderType + { + /// + /// Packages provider + /// + Packages, + + /// + /// Files provider. + /// + Files, + + /// + /// External Document Reference provider. + /// + ExternalDocumentReference + } +} diff --git a/src/Microsoft.Sbom.Api/README.md b/src/Microsoft.Sbom.Api/README.md new file mode 100644 index 00000000..dedc31e9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/README.md @@ -0,0 +1,43 @@ +Generates Software Bill of Materials (SBOM) + +#### Scan Sample +```C# + var generator = new SBOMGenerator(); + + string scanPath = @"D:\tmp\SBOM\"; + string outputPath = @"D:\tmp\SBOM\_manifest"; + + SBOMMetadata metadata = new SBOMMetadata() + { + PackageName = "MyVpack", + PackageVersion = "0.0.1" }; + + IList specifications = new List() + { + new SBOMSpecification ("SPDX", "2.2") + }; + + RuntimeConfiguration configuration = new RuntimeConfiguration() + { + DeleteManifestDirectoryIfPresent = true, + WorkflowParallelism = 4, + Verbosity = System.Diagnostics.Tracing.EventLevel.Verbose, + }; + + await generator.GenerateSBOMAsync(scanPath, scanPath, metadata, null, configuration, outputPath); +``` + +#### If you have files and don't need to scan for them + +```C# + SBOMGenerator generator = new(); + Task task = generator.GenerateSBOMAsync( + outputDirectory, + sbomFiles, + sbomPackages, + metadata, + new List { new("SPDX", "2.2") }, + new RuntimeConfiguration { DeleteManifestDirectoryIfPresent = true }); + + bool taskCompleted = task.Wait(TimeSpan.FromMinutes(timeoutMinutes)); +``` diff --git a/src/Microsoft.Sbom.Api/Recorder/SBOMPackageDetailsRecorder.cs b/src/Microsoft.Sbom.Api/Recorder/SBOMPackageDetailsRecorder.cs new file mode 100644 index 00000000..f8de8281 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Recorder/SBOMPackageDetailsRecorder.cs @@ -0,0 +1,128 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Sbom.Api.Recorder +{ + /// + /// A recorder class, injected as a singleton that records details about files and + /// packages encountered while traversing the drop. + /// + public class SbomPackageDetailsRecorder : ISbomPackageDetailsRecorder + { + private string rootPackageId; + private string documentId; + private readonly ConcurrentBag fileIds = new ConcurrentBag(); + private readonly ConcurrentBag spdxFileIds = new ConcurrentBag(); + private readonly ConcurrentBag packageIds = new ConcurrentBag(); + private readonly ConcurrentBag> externalDocumentRefIdRootElementPairs = new ConcurrentBag>(); + private readonly ConcurrentBag checksums = new ConcurrentBag(); + + /// + /// Record a fileId that is included in this SBOM. + /// + /// + public void RecordFileId(string fileId) + { + if (string.IsNullOrEmpty(fileId)) + { + throw new ArgumentException($"'{nameof(fileId)}' cannot be null or empty.", nameof(fileId)); + } + + fileIds.Add(fileId); + } + + public void RecordSPDXFileId(string spdxFileId) + { + if (string.IsNullOrEmpty(spdxFileId)) + { + throw new ArgumentException($"'{nameof(spdxFileId)}' cannot be null or empty.", nameof(spdxFileId)); + } + + spdxFileIds.Add(spdxFileId); + } + + /// + /// Record a packageId that is included in this SBOM. + /// + /// + public void RecordPackageId(string packageId) + { + if (string.IsNullOrEmpty(packageId)) + { + throw new ArgumentException($"'{nameof(packageId)}' cannot be null or empty.", nameof(packageId)); + } + + packageIds.Add(packageId); + } + + /// + /// Record a externalDocumentReference Id that is included in this SBOM. + /// + /// + public void RecordExternalDocumentReferenceIdAndRootElement(string externalDocumentReferenceId, string rootElement) + { + if (string.IsNullOrEmpty(externalDocumentReferenceId)) + { + throw new ArgumentException($"'{nameof(externalDocumentReferenceId)}' cannot be null or empty.", nameof(externalDocumentReferenceId)); + } + + externalDocumentRefIdRootElementPairs.Add(new KeyValuePair(externalDocumentReferenceId, rootElement)); + } + + public GenerationData GetGenerationData() + { + return new GenerationData + { + Checksums = checksums.ToList(), + FileIds = fileIds.ToList(), + SPDXFileIds = spdxFileIds.ToList(), + PackageIds = packageIds.ToList(), + ExternalDocumentReferenceIDs = externalDocumentRefIdRootElementPairs.ToList(), + RootPackageId = rootPackageId, + DocumentId = documentId + }; + } + + /// + /// Record the SHA1 hash for the file. + /// + /// + public void RecordChecksumForFile(Checksum[] checksums) + { + if (checksums is null) + { + throw new ArgumentNullException(nameof(checksums)); + } + + this.checksums.Add(checksums); + } + + public void RecordRootPackageId(string rootPackageId) + { + if (rootPackageId is null) + { + throw new ArgumentNullException(nameof(rootPackageId)); + } + + this.rootPackageId = rootPackageId; + } + + public void RecordDocumentId(string documentId) + { + if (documentId is null) + { + throw new ArgumentNullException(nameof(documentId)); + } + + this.documentId = documentId; + } + } +} diff --git a/src/Microsoft.Sbom.Api/SBOMGenerator.cs b/src/Microsoft.Sbom.Api/SBOMGenerator.cs new file mode 100644 index 00000000..714ee13f --- /dev/null +++ b/src/Microsoft.Sbom.Api/SBOMGenerator.cs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.ComponentModel; +using System.Linq; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Config; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Api.Workflows; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Ninject; +using PowerArgs; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api +{ + /// + /// Responsible for an API to generate SBOMs. + /// + public class SBOMGenerator : ISBOMGenerator + { + private readonly ApiConfigurationBuilder configurationBuilder; + private readonly StandardKernel kernel; + private readonly IFileSystemUtils fileSystemUtils; + + public SBOMGenerator() + { + kernel = new StandardKernel(new Bindings()); + configurationBuilder = new ApiConfigurationBuilder(); + fileSystemUtils = new FileSystemUtils(); + } + + public SBOMGenerator(StandardKernel kernel, IFileSystemUtils fileSystemUtils) + { + this.kernel = kernel ?? throw new ArgumentNullException(nameof(kernel)); + configurationBuilder = new ApiConfigurationBuilder(); + this.fileSystemUtils = fileSystemUtils ?? throw new ArgumentNullException(nameof(fileSystemUtils)); + } + + /// + public async Task GenerateSBOMAsync( + string rootPath, + string componentPath, + SBOMMetadata metadata, + IList specifications = null, + RuntimeConfiguration configuration = null, + string manifestDirPath = null, + string externalDocumentReferenceListFile = null) + { + if (string.IsNullOrWhiteSpace(manifestDirPath)) + { + manifestDirPath = fileSystemUtils.JoinPaths(rootPath, Constants.ManifestFolder); + } + + // Get scan configuration + var config = configurationBuilder.GetConfiguration( + rootPath, + manifestDirPath, null, null, metadata, specifications, + configuration, externalDocumentReferenceListFile, componentPath); + + // Initialize the IOC container. This varies depending on the configuration. + config = ValidateConfig(config); + kernel.Bind().ToConstant(config); + + // This is the generate workflow + IWorkflow workflow = kernel.Get(nameof(SBOMGenerationWorkflow)); + bool isSuccess = await workflow.RunAsync(); + + // TODO: Telemetry? + IRecorder recorder = kernel.Get(); + await recorder.FinalizeAndLogTelemetryAsync(); + + var entityErrors = ((TelemetryRecorder)recorder).Errors.Select(error => error.ToEntityError()).ToList(); + + return new SBOMGenerationResult(isSuccess, entityErrors); + } + + /// + public async Task GenerateSBOMAsync( + string rootPath, + IEnumerable files, + IEnumerable packages, + SBOMMetadata metadata, + IList specifications = null, + RuntimeConfiguration runtimeConfiguration = null, + string manifestDirPath = null, + string externalDocumentReferenceListFile = null) + { + if (string.IsNullOrWhiteSpace(rootPath)) + { + throw new ArgumentException($"'{nameof(rootPath)}' cannot be null or whitespace.", nameof(rootPath)); + } + + if (files is null) + { + throw new ArgumentNullException(nameof(files)); + } + + if (packages is null) + { + throw new ArgumentNullException(nameof(packages)); + } + + if (metadata is null) + { + throw new ArgumentNullException(nameof(metadata)); + } + + if (string.IsNullOrWhiteSpace(manifestDirPath)) + { + manifestDirPath = rootPath; + } + + var configuration = configurationBuilder.GetConfiguration( + rootPath, manifestDirPath, files, packages, metadata, specifications, + runtimeConfiguration, externalDocumentReferenceListFile); + configuration = ValidateConfig(configuration); + + kernel.Bind().ToConstant(configuration); + + kernel.Bind().ToConstant(metadata); + bool result = await kernel.Get(nameof(SBOMGenerationWorkflow)).RunAsync(); + return new SBOMGenerationResult(result, new List()); + } + + /// + public IEnumerable GetRequiredAlgorithms(SBOMSpecification specification) + { + if (specification is null) + { + throw new ArgumentNullException(nameof(specification)); + } + + var generatorProvider = kernel.Get(); + if (generatorProvider == null) + { + throw new MissingGeneratorException($"Unable to get a list of supported SBOM generators."); + } + + // The provider will throw if the generator is not found. + var generator = generatorProvider.Get(specification.ToManifestInfo()); + + return generator + .RequiredHashAlgorithms + .ToList(); + } + + public IEnumerable GetSupportedSBOMSpecifications() + { + var generatorProvider = kernel.Get(); + if (generatorProvider == null) + { + throw new Exception($"Unable to get a list of supported SBOM generators."); + } + + return generatorProvider + .GetSupportedManifestInfos() + .Select(g => g.ToSBOMSpecification()) + .ToList(); + } + + private Configuration ValidateConfig(Configuration config) + { + var configValidators = kernel.GetAll(); + var configSanitizer = kernel.Get(); + + foreach (PropertyDescriptor property in TypeDescriptor.GetProperties(config)) + { + configValidators.ForEach(v => + { + v.CurrentAction = config.ManifestToolAction; + v.Validate(property.DisplayName, property.GetValue(config), property.Attributes); + }); + } + + configSanitizer.SanitizeConfig(config); + return config; + } + } +} diff --git a/src/Microsoft.Sbom.Api/SignValidator/SignValidationProvider.cs b/src/Microsoft.Sbom.Api/SignValidator/SignValidationProvider.cs new file mode 100644 index 00000000..d68aa0c9 --- /dev/null +++ b/src/Microsoft.Sbom.Api/SignValidator/SignValidationProvider.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Serilog; +using System; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.SignValidator +{ + /// + /// Factory class that provides a implementation based on the + /// current operating system type. + /// + public class SignValidationProvider + { + private readonly ISignValidator[] signValidators; + private readonly Dictionary signValidatorsMap; + private readonly ILogger logger; + private readonly IOSUtils osUtils; + + public SignValidationProvider(ISignValidator[] signValidators, ILogger logger, IOSUtils osUtils) + { + this.signValidators = signValidators ?? throw new ArgumentNullException(nameof(signValidators)); + this.logger = logger ?? throw new ArgumentNullException(nameof(logger)); + this.osUtils = osUtils ?? throw new ArgumentNullException(nameof(osUtils)); + + signValidatorsMap = new Dictionary(); + } + + public void Init() + { + foreach (var signValidator in signValidators) + { + signValidatorsMap[signValidator.SupportedPlatform] = signValidator; + } + } + + public ISignValidator Get() + { + if (!signValidatorsMap.TryGetValue(osUtils.GetCurrentOSPlatform(), out ISignValidator signValidator)) + { + logger.Error($"No signature validator found for current OS, supported OS are {signValidatorsMap.Keys}"); + throw new SignValidatorNotFoundException("No signature validator found for current OS"); + } + + return signValidator; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/AssemblyConfig.cs b/src/Microsoft.Sbom.Api/Utils/AssemblyConfig.cs new file mode 100644 index 00000000..d4722b63 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/AssemblyConfig.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config.Attributes; +using System; +using System.IO; +using System.Reflection; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + public class AssemblyConfig : IAssemblyConfig + { + /// + public string DefaultSBOMNamespaceBaseUri => DefaultSBOMBaseNamespaceUri.Value; + + /// + public string DefaultSBOMNamespaceBaseUriWarningMessage => DefaultSBOMBaseNamespaceUriWarningMessage.Value; + + /// + public ManifestInfo DefaultManifestInfoForValidationAction => DefaultManifestInfoForValidationActionValue.Value; + + /// + public string AssemblyDirectory => AssemblyDirectoryValue.Value; + + private static readonly Lazy DefaultSBOMBaseNamespaceUri = new Lazy(() => + { + return Assembly.GetExecutingAssembly().GetCustomAttribute()?.DefaultBaseNamespaceUri; + }); + + private static readonly Lazy DefaultSBOMBaseNamespaceUriWarningMessage = new Lazy(() => + { + return Assembly.GetExecutingAssembly().GetCustomAttribute()?.WarningMessage; + }); + + private static readonly Lazy DefaultManifestInfoForValidationActionValue = new Lazy(() => + { + return Assembly.GetExecutingAssembly().GetCustomAttribute()?.ManifestInfo; + }); + + private static readonly Lazy AssemblyDirectoryValue = new Lazy(() => + { + string location = Assembly.GetExecutingAssembly().Location; + return Path.GetDirectoryName(location); + }); + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ChannelDeduplicator.cs b/src/Microsoft.Sbom.Api/Utils/ChannelDeduplicator.cs new file mode 100644 index 00000000..651e04ef --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ChannelDeduplicator.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Concurrent; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Provides deduplication of T objects inside a channel. + /// + public abstract class ChannelDeduplicator + { + protected ConcurrentDictionary uniqueObjects; + + protected ChannelDeduplicator() + { + uniqueObjects = new ConcurrentDictionary(); + } + + /// + /// Removes duplicate T objects from a channel. + /// + /// Input channel. + /// Output channel without duplicates. + public ChannelReader Deduplicate(ChannelReader input) + { + var output = Channel.CreateUnbounded(); + + Task.Run(async () => + { + await foreach (var obj in input.ReadAllAsync()) + { + if (uniqueObjects.TryAdd(GetKey(obj), true)) + { + await output.Writer.WriteAsync(obj); + } + } + + output.Writer.Complete(); + }); + + return output; + } + + public abstract string GetKey(T obj); + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ComponentDetectionCliArgumentBuilder.cs b/src/Microsoft.Sbom.Api/Utils/ComponentDetectionCliArgumentBuilder.cs new file mode 100644 index 00000000..edf6c9c3 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ComponentDetectionCliArgumentBuilder.cs @@ -0,0 +1,207 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Castle.Core.Internal; +using Microsoft.ComponentDetection.Common; +using PowerArgs; +using System; +using System.Collections.Generic; +using System.Linq; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// ComponentDetectionCliArgumentBuilder generates list of CLI params for Component Detection orchestrator. + /// + public class ComponentDetectionCliArgumentBuilder + { + private string action; + private VerbosityMode verbosity = VerbosityMode.Quiet; + private string sourceDirectory; + private Dictionary detectorArgs = new Dictionary(); + private Dictionary keyValueArgs = new Dictionary(); + private List keyArgs = new List(); + + private const string VerbosityParamName = "Verbosity"; + private const string SourceDirectoryParamName = "SourceDirectory"; + private const string DetectorArgsParamName = "DetectorArgs"; + + private const string ScanAction = "scan"; + + public ComponentDetectionCliArgumentBuilder() + { + } + + private void Validate() + { + if (action.IsNullOrEmpty()) + { + throw new ArgumentNullException("Action should be specified."); + } + + if (sourceDirectory.IsNullOrEmpty()) + { + throw new ArgumentNullException("Source directory should be specified."); + } + } + + public string[] Build() + { + Validate(); + + var command = $"{action} --{VerbosityParamName} {verbosity} --{SourceDirectoryParamName} {sourceDirectory}"; + + if (detectorArgs.Any()) + { + var args = string.Join(",", detectorArgs.Select(arg => $"{arg.Key}={arg.Value}")); + var detectorArgsCommand = $"--{DetectorArgsParamName} {args}"; + command += $" {detectorArgsCommand}"; + } + + if (keyValueArgs.Any()) + { + var argsList = keyValueArgs + .Select(x => new List() { $"--{x.Key}", x.Value.Contains(" ") ? $"\"{x.Value}\"" : x.Value }) + .SelectMany(x => x) + .ToList(); + var argsCommand = string.Join(" ", argsList); + command += $" {argsCommand}"; + } + + if (keyArgs.Any()) + { + var keyArgsCommand = string.Join(" ", keyArgs); + command += $" {keyArgsCommand}"; + } + + return Args.Convert(command.Trim()); + } + + public ComponentDetectionCliArgumentBuilder Scan() + { + action = ScanAction; + return this; + } + + public ComponentDetectionCliArgumentBuilder AddDetectorArg(string name, string value) + { + detectorArgs.Add(name, value); + return this; + } + + public ComponentDetectionCliArgumentBuilder Verbosity(VerbosityMode verbosity) + { + this.verbosity = verbosity; + return this; + } + + public ComponentDetectionCliArgumentBuilder SourceDirectory(string directory) + { + sourceDirectory = directory; + return this; + } + + public ComponentDetectionCliArgumentBuilder AddArg(string name, string value) + { + if (name.IsNullOrEmpty()) + { + throw new ArgumentNullException($"{nameof(name)} should not be null"); + } + + if (value.IsNullOrEmpty()) + { + throw new ArgumentNullException($"{nameof(value)} should not be null"); + } + + name = name.StartsWith("--") ? name.Substring(2) : name; + + if (name.IsNullOrEmpty()) + { + throw new ArgumentNullException($"{nameof(name)} should not be null or be empty"); + } + + if (name.Equals(SourceDirectoryParamName, StringComparison.OrdinalIgnoreCase)) + { + return SourceDirectory(value); + } + + if (name.Equals(VerbosityParamName, StringComparison.OrdinalIgnoreCase)) + { + if (!Enum.TryParse(value, out VerbosityMode verbosity)) + { + throw new ArgumentException($"Invalid verbosity value provided - {value}."); + } + + return Verbosity(verbosity); + } + + if (name.Equals(DetectorArgsParamName, StringComparison.OrdinalIgnoreCase)) + { + var detectorArgs = value.Split(",").Select(arg => arg.Trim()).Select(arg => arg.Split("=")); + if (detectorArgs.Any()) + { + foreach (var arg in detectorArgs) + { + if (arg.Length >= 2) + { + AddDetectorArg(arg[0], arg[1]); + } + } + } + + return this; + } + + keyValueArgs[name] = value; + return this; + } + + public ComponentDetectionCliArgumentBuilder AddArg(string value) + { + if (value.IsNullOrEmpty()) + { + throw new ArgumentNullException($"{nameof(value)} should not be null"); + } + + if (value.StartsWith("--") && !keyArgs.Exists(item => item == value)) + { + keyArgs.Add(value); + } + else + { + var argument = $"--{value}"; + if (!keyArgs.Exists(item => item == argument)) + { + keyArgs.Add(argument); + } + } + + return this; + } + + public ComponentDetectionCliArgumentBuilder ParseAndAddArgs(string args) + { + if (args.IsNullOrEmpty()) + { + throw new ArgumentNullException($"{nameof(args)} should not be null"); + } + + var argArray = Args.Convert(args); + for (int i = 0; i < argArray.Length; i++) + { + if (argArray[i].StartsWith("--") && i + 1 < argArray.Length && !argArray[i + 1].StartsWith("--")) + { + AddArg(argArray[i].Substring(2), argArray[i + 1]); + i++; + continue; + } + else if (argArray[i].StartsWith("--")) + { + AddArg(argArray[i].Substring(2)); + } + } + + return this; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ComponentDetector.cs b/src/Microsoft.Sbom.Api/Utils/ComponentDetector.cs new file mode 100644 index 00000000..f55108de --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ComponentDetector.cs @@ -0,0 +1,20 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Orchestrator; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// A component detector wrapper, used for unit testing. + /// + public class ComponentDetector + { + public virtual ScanResult Scan(string[] args) + { + var orchestrator = new Orchestrator(); + return orchestrator.Load(args); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ComponentDetectorCachedExecutor.cs b/src/Microsoft.Sbom.Api/Utils/ComponentDetectorCachedExecutor.cs new file mode 100644 index 00000000..f3950547 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ComponentDetectorCachedExecutor.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Serilog; +using System; +using System.Collections.Concurrent; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Wrapper class for a component detector that caches CD execution results with the same arguments. + /// The main use case for it is to reuse scanned component results across different providers (e.g packages, external document refs). + /// + public class ComponentDetectorCachedExecutor + { + private readonly ILogger log; + private readonly ComponentDetector detector; + private ConcurrentDictionary results; + + public ComponentDetectorCachedExecutor(ILogger log, ComponentDetector detector) + { + this.log = log ?? throw new ArgumentNullException(nameof(log)); + this.detector = detector ?? throw new ArgumentNullException(nameof(detector)); + + results = new ConcurrentDictionary(); + } + + /// + /// Performs component detection scan or gets results from cache based on provided arguments. + /// + /// CD arguments + /// Result of CD scan + public virtual ScanResult Scan(string[] args) + { + if (args is null) + { + throw new ArgumentNullException(nameof(args)); + } + + var argsHashCode = string.Join("", args).GetHashCode(); + if (results.ContainsKey(argsHashCode)) + { + log.Debug("Using cached CD scan result for the call with the same arguments"); + return results[argsHashCode]; + } + + var result = detector.Scan(args); + results.TryAdd(argsHashCode, result); + return result; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/Constants.cs b/src/Microsoft.Sbom.Api/Utils/Constants.cs new file mode 100644 index 00000000..a196f404 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/Constants.cs @@ -0,0 +1,45 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts.Enums; +using Serilog.Events; + +namespace Microsoft.Sbom.Api.Utils +{ + public static class Constants + { + public const string ManifestFolder = "_manifest"; + + public static ManifestInfo SPDX22ManifestInfo = new ManifestInfo + { + Name = "SPDX", + Version = "2.2" + }; + + // TODO: move to test csproj + public static ManifestInfo TestManifestInfo = new ManifestInfo + { + Name = "TestManifest", + Version = "1.0.0" + }; + + public static AlgorithmName DefaultHashAlgorithmName = AlgorithmName.SHA256; + + public const string ManifestBsiFileName = "bsi.json"; + + public const string SPDXFileExtension = ".spdx.json"; + public const string DocumentNamespaceString = "documentNamespace"; + public const string NameString = "name"; + public const string DocumentDescribesString = "documentDescribes"; + public const string SpdxVersionString = "spdxVersion"; + public const string DefaultRootElement = "SPDXRef-Document"; + public const string NamespaceUriBasePropertyName = "NamespaceUriBase"; + + #region Configuration switches + + public const string DeleteManifestDirBoolVariableName = "DeleteManifestDirIfPresent"; + + #endregion + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/Events.cs b/src/Microsoft.Sbom.Api/Utils/Events.cs new file mode 100644 index 00000000..4617af88 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/Events.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.Text; + +namespace Microsoft.Sbom.Api.Utils +{ + internal static class Events + { + #region Generation + internal const string SBOMGenerationWorkflow = "Total generation time"; + internal const string FilesGeneration = "Files generation time"; + internal const string PackagesGeneration = "Packages generation time"; + internal const string RelationshipsGeneration = "Relationships generation time"; + internal const string MetadataBuilder = "Metadata build time for {0} format"; + internal const string ExternalDocumentReferenceGeneration = "External document reference generation time"; + + #endregion + internal const string SBOMValidationWorkflow = "Total validation time"; + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ExternalDocumentReferenceEqualityComparer.cs b/src/Microsoft.Sbom.Api/Utils/ExternalDocumentReferenceEqualityComparer.cs new file mode 100644 index 00000000..83b94118 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ExternalDocumentReferenceEqualityComparer.cs @@ -0,0 +1,48 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; +using System.Text; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Compares two ExternalDocumentReferenceInfo objects to see if they represent the same underlying external document. + /// + public class ExternalDocumentReferenceEqualityComparer : IEqualityComparer + { + public bool Equals([AllowNull] ExternalDocumentReferenceInfo x, [AllowNull] ExternalDocumentReferenceInfo y) + { + if (x == null && y == null) + { + return true; + } + else if (x == null || y == null) + { + return false; + } + else if (string.Equals(x.DocumentNamespace, y.DocumentNamespace, StringComparison.OrdinalIgnoreCase)) + { + return true; + } + else + { + return false; + } + } + + public int GetHashCode([DisallowNull] ExternalDocumentReferenceInfo obj) + { + if (obj.DocumentNamespace is null) + { + throw new ArgumentNullException(nameof(obj.DocumentNamespace)); + } + + return obj.DocumentNamespace.GetHashCode(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ExternalReferenceDeduplicator.cs b/src/Microsoft.Sbom.Api/Utils/ExternalReferenceDeduplicator.cs new file mode 100644 index 00000000..fbe35953 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ExternalReferenceDeduplicator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Provides deduplication of ExternalDocumentReferenceInfo objects inside a channel. + /// + public class ExternalReferenceDeduplicator : ChannelDeduplicator + { + public ExternalReferenceDeduplicator() + : base() { } + + public override string GetKey(ExternalDocumentReferenceInfo obj) + { + return obj?.DocumentNamespace; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs b/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs new file mode 100644 index 00000000..deb6f296 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/FileTypeUtils.cs @@ -0,0 +1,27 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts.Enums; +using System; +using System.Collections.Generic; +using System.IO; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// FileTypeUtils is used to get the FileType for a given filename + /// + public class FileTypeUtils : IFileTypeUtils + { + public List GetFileTypesBy(string fileName) + { + if (!string.IsNullOrWhiteSpace(fileName) && fileName.EndsWith(Constants.SPDXFileExtension, StringComparison.OrdinalIgnoreCase)) + { + return new List { FileType.SPDX }; + } + + return null; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/IAssemblyConfig.cs b/src/Microsoft.Sbom.Api/Utils/IAssemblyConfig.cs new file mode 100644 index 00000000..89fde9ea --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/IAssemblyConfig.cs @@ -0,0 +1,35 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Contans configuration items defined in the assembly. + /// + public interface IAssemblyConfig + { + /// + /// Gets the namespace base URI as defined in the assembly. + /// + public string DefaultSBOMNamespaceBaseUri { get; } + + /// + /// Gets the warning message to show in case the assembly defined namespace base URI is used instead + /// of the user provided one. + /// + public string DefaultSBOMNamespaceBaseUriWarningMessage { get; } + + /// + /// Gets the default value to use for ManifestInfo for validation action in case the user doesn't provide a + /// value. + /// + public ManifestInfo DefaultManifestInfoForValidationAction { get; } + + /// + /// Gets the directory where the current executing assembly is located. + /// + public string AssemblyDirectory { get; } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/IFileTypeUtils.cs b/src/Microsoft.Sbom.Api/Utils/IFileTypeUtils.cs new file mode 100644 index 00000000..cd30c305 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/IFileTypeUtils.cs @@ -0,0 +1,13 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Contracts.Enums; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Utils +{ + public interface IFileTypeUtils + { + List GetFileTypesBy(string fileName); + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/InternalSBOMFileInfoDeduplicator.cs b/src/Microsoft.Sbom.Api/Utils/InternalSBOMFileInfoDeduplicator.cs new file mode 100644 index 00000000..71635d8a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/InternalSBOMFileInfoDeduplicator.cs @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Provides deduplication of InternalSBOMFileInfo objects inside a channel. + /// + public class InternalSBOMFileInfoDeduplicator : ChannelDeduplicator + { + public InternalSBOMFileInfoDeduplicator() + : base() { } + + public override string GetKey(InternalSBOMFileInfo obj) + { + return obj?.Path; + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs b/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs new file mode 100644 index 00000000..6324a9a0 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/SBOMFormatExtensions.cs @@ -0,0 +1,51 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts; +using System; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Extension methods to convert SBOM format specificaitons from multiple formats. + /// + public static class SBOMFormatExtensions + { + /// + /// Converts a to a object. + /// + /// + /// + /// + public static ManifestInfo ToManifestInfo(this SBOMSpecification specification) + { + if (specification is null) + { + throw new ArgumentNullException(nameof(specification)); + } + + return new ManifestInfo + { + Name = specification.Name, + Version = specification.Version + }; + } + + /// + /// Converts a to a object. + /// + /// + /// + /// + public static SBOMSpecification ToSBOMSpecification(this ManifestInfo manifestInfo) + { + if (manifestInfo is null) + { + throw new ArgumentNullException(nameof(manifestInfo)); + } + + return new SBOMSpecification(manifestInfo.Name, manifestInfo.Version); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Utils/ScannedComponentEqualityComparer.cs b/src/Microsoft.Sbom.Api/Utils/ScannedComponentEqualityComparer.cs new file mode 100644 index 00000000..dee3cbf8 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Utils/ScannedComponentEqualityComparer.cs @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using System; +using System.Collections.Generic; +using System.Diagnostics.CodeAnalysis; + +namespace Microsoft.Sbom.Api.Utils +{ + /// + /// Compares two objects to see if they represent the same underlying component. + /// + public class ScannedComponentEqualityComparer : IEqualityComparer + { + public bool Equals([AllowNull] ScannedComponent scannedComponent1, [AllowNull] ScannedComponent scannedComponent2) + { + if (scannedComponent2 == null && scannedComponent1 == null) + { + return true; + } + else if (scannedComponent1 == null || scannedComponent2 == null) + { + return false; + } + else if (string.Equals( + scannedComponent1.Component.Id, + scannedComponent2.Component.Id, + StringComparison.OrdinalIgnoreCase)) + { + return true; + } + else + { + return false; + } + } + + public int GetHashCode([DisallowNull] ScannedComponent scannedComponent) + { + return scannedComponent.Component.Id.ToLower().GetHashCode(); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/DropValidatorWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/DropValidatorWorkflow.cs new file mode 100644 index 00000000..7e4bb125 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/DropValidatorWorkflow.cs @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Entities.Output; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using PowerArgs; +using Serilog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; +using System.Text.Json.Serialization; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows +{ + /// + /// Defines the workflow steps for the drop validation action. + /// + public class DropValidatorWorkflow : IWorkflow + { + private readonly IConfiguration configuration; + private readonly DirectoryWalker directoryWalker; + private readonly ChannelUtils channelUtils; + private readonly FileHasher fileHasher; + private readonly HashValidator hashValidator; + private readonly ManifestData manifestData; + private readonly ManifestFolderFilterer fileFilterer; + private readonly ValidationResultGenerator validationResultGenerator; + private readonly IOutputWriter outputWriter; + private readonly ILogger log; + private readonly ISignValidator signValidator; + private readonly ManifestFileFilterer manifestFileFilterer; + private readonly IRecorder recorder; + + public DropValidatorWorkflow( + IConfiguration configuration, + DirectoryWalker directoryWalker, + ManifestFolderFilterer fileFilterer, + ChannelUtils channelUtils, + FileHasher fileHasher, + HashValidator hashValidator, + ManifestData manifestData, + ValidationResultGenerator validationResultGenerator, + IOutputWriter outputWriter, + ILogger log, + ISignValidator signValidator, + ManifestFileFilterer manifestFileFilterer, + IRecorder recorder) + { + this.configuration = configuration; + this.directoryWalker = directoryWalker; + this.channelUtils = channelUtils; + this.hashValidator = hashValidator; + this.manifestData = manifestData; + this.log = log; + this.fileFilterer = fileFilterer; + this.signValidator = signValidator; + this.validationResultGenerator = validationResultGenerator; + this.outputWriter = outputWriter; + this.manifestFileFilterer = manifestFileFilterer; + this.recorder = recorder; + this.fileHasher = fileHasher; + if (this.fileHasher != null) + { + this.fileHasher.ManifestData = manifestData; + } + } + + public async Task RunAsync() + { + ValidationResult validationResultOutput = null; + IEnumerable validFailures = null; + using (recorder.TraceEvent(Events.SBOMValidationWorkflow)) + { + try + { + log.Debug("Starting validation workflow."); + DateTime start = DateTime.Now; + + IList> errors = new List>(); + IList> results = new List>(); + + // Validate signature + if (!signValidator.Validate()) + { + log.Error("Sign validation failed."); + return false; + } + + // Workflow + // Read all files + var (files, dirErrors) = directoryWalker.GetFilesRecursively(configuration.BuildDropPath.Value); + errors.Add(dirErrors); + + // Filter root path matching files from the manifest map. + var manifestFilterErrors = manifestFileFilterer.FilterManifestFiles(); + errors.Add(manifestFilterErrors); + + log.Debug($"Splitting the workflow into {configuration.Parallelism.Value} threads."); + var splitFilesChannels = channelUtils.Split(files, configuration.Parallelism.Value); + + log.Debug("Waiting for the workflow to finish..."); + foreach (var fileChannel in splitFilesChannels) + { + // Filter files + var (filteredFiles, filteringErrors) = fileFilterer.FilterFiles(fileChannel); + errors.Add(filteringErrors); + + // Generate hash code for each file. + var (fileHashes, hashingErrors) = fileHasher.Run(filteredFiles); + errors.Add(hashingErrors); + + // Validate hashes for each file. + var (validationResults, validationErrors) = hashValidator.Validate(fileHashes); + errors.Add(validationErrors); + results.Add(validationResults); + } + + // 4. Wait for the pipeline to finish. + int successCount = 0; + var failures = new List(); + + ChannelReader resultChannel = channelUtils.Merge(results.ToArray()); + await foreach (FileValidationResult validationResult in resultChannel.ReadAllAsync()) + { + successCount++; + } + + ChannelReader workflowErrors = channelUtils.Merge(errors.ToArray()); + + await foreach (FileValidationResult error in workflowErrors.ReadAllAsync()) + { + failures.Add(error); + } + + // 5. Collect remaining entries in ManifestMap + failures.AddRange( + from manifestItem in manifestData.HashesMap + select new FileValidationResult + { + ErrorType = ErrorType.MissingFile, + Path = manifestItem.Key + }); + + // Failure + if (successCount < 0) + { + log.Error("Error running the workflow, failing without publishing results."); + return false; + } + + DateTime end = DateTime.Now; + log.Debug("Finished workflow, gathering results."); + + // 6. Generate JSON output + validationResultOutput = validationResultGenerator + .WithSuccessCount(successCount) + .WithTotalDuration(end - start) + .WithValidationResults(failures) + .Build(); + + // 7. Write JSON output to file. + var options = new JsonSerializerOptions + { + Converters = + { + new JsonStringEnumConverter() + } + }; + await outputWriter.WriteAsync(JsonSerializer.Serialize(validationResultOutput, options)); + validFailures = failures.Where(a => a.ErrorType != ErrorType.ManifestFolder + && a.ErrorType != ErrorType.FilteredRootPath); + + if (configuration.IgnoreMissing.Value) + { + log.Warning("Not including missing files on disk as -IgnoreMissing switch is on."); + validFailures = validFailures.Where(a => a.ErrorType != ErrorType.MissingFile); + } + + return !validFailures.Any(); + } + catch (Exception e) + { + recorder.RecordException(e); + log.Error("Encountered an error while validating the drop."); + log.Error($"Error details: {e.Message}"); + return false; + } + finally + { + if (validFailures != null) + { + recorder.RecordTotalErrors(validFailures.ToList()); + } + + // Log telemetry + LogResultsSummary(validationResultOutput, validFailures); + LogIndividualFileResults(validFailures); + } + } + } + + private void LogIndividualFileResults(IEnumerable validFailures) + { + if (validFailures == null) + { + // We failed to generate the output due to a workflow error. + return; + } + + log.Verbose(""); + log.Verbose("------------------------------------------------------------"); + log.Verbose("Individual file validation results"); + log.Verbose("------------------------------------------------------------"); + log.Verbose(""); + + log.Verbose("Additional files not in the manifest: "); + log.Verbose(""); + validFailures.Where(vf => vf.ErrorType == ErrorType.AdditionalFile).ForEach(f => log.Verbose(f.Path)); + log.Verbose("------------------------------------------------------------"); + + log.Verbose("Files with invalid hashes:"); + log.Verbose(""); + validFailures.Where(vf => vf.ErrorType == ErrorType.InvalidHash).ForEach(f => log.Verbose(f.Path)); + log.Verbose("------------------------------------------------------------"); + + log.Verbose("Files in the manifest missing from the disk:"); + log.Verbose(""); + validFailures.Where(vf => vf.ErrorType == ErrorType.MissingFile).ForEach(f => log.Verbose(f.Path)); + log.Verbose("------------------------------------------------------------"); + + log.Verbose("Unknown file failures:"); + log.Verbose(""); + validFailures.Where(vf => vf.ErrorType == ErrorType.Other).ForEach(f => log.Verbose(f.Path)); + log.Verbose("------------------------------------------------------------"); + } + + private void LogResultsSummary(ValidationResult validationResultOutput, IEnumerable validFailures) + { + if (validationResultOutput == null || validFailures == null) + { + // We failed to generate the output due to a workflow error. + return; + } + + log.Debug(""); + log.Debug("------------------------------------------------------------"); + log.Debug("Validation Summary"); + log.Debug("------------------------------------------------------------"); + log.Debug(""); + + log.Debug($"Validation Result . . . . . . . . . . . . . . . .{validationResultOutput.Result}"); + log.Debug($"Total execution time (sec) . . . . . . . . . . . {validationResultOutput.Summary.TotalExecutionTimeInSeconds}"); + log.Debug($"Files failed . . . . . . . . . . . . . . . . . . {validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount}"); + log.Debug($"Files successfully validated . . . . . . . . . . {validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount}"); + log.Debug($"Total files validated. . . . . . . . . . . . . . {validationResultOutput.Summary.ValidationTelemetery.FilesValidatedCount}"); + log.Debug($"Total files in manifest. . . . . . . . . . . . . {validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest}"); + log.Debug($""); + log.Debug($"Additional files not in the manifest . . . . . . {validFailures.Count(v => v.ErrorType == ErrorType.AdditionalFile)}"); + log.Debug($"Files with invalid hashes . . . . . . . . . . . .{validFailures.Count(v => v.ErrorType == ErrorType.InvalidHash)}"); + log.Debug($"Files in the manifest missing from the disk . . .{validFailures.Count(v => v.ErrorType == ErrorType.MissingFile)}"); + log.Debug($"Unknown file failures . . . . . . . . . . . . . {validFailures.Count(v => v.ErrorType == ErrorType.Other)}"); + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs new file mode 100644 index 00000000..7842391d --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/ExternalDocumentReferenceGenerator.cs @@ -0,0 +1,95 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Api.Config; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Providers; +using Microsoft.Sbom.Api.Utils; +using Ninject; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows.Helpers +{ + /// + /// This class generates an array of external document references + /// + public class ExternalDocumentReferenceGenerator : IJsonArrayGenerator + { + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + public ISbomConfigProvider SBOMConfigs { get; set; } + + [Inject] + public IList SourcesProviders { get; set; } + + [Inject] + public IRecorder Recorder { get; set; } + + public async Task> GenerateAsync() + { + using (Recorder.TraceEvent(Events.ExternalDocumentReferenceGeneration)) + { + IList totalErrors = new List(); + + IEnumerable sourcesProviders = SourcesProviders + .Where(s => s.IsSupported(ProviderType.ExternalDocumentReference)); + if (!sourcesProviders.Any()) + { + Log.Debug($"No source providers found for {ProviderType.ExternalDocumentReference}"); + return totalErrors; + } + + // Write the start of the array, if supported. + IList externalRefArraySupportingConfigs = new List(); + foreach (var manifestInfo in SBOMConfigs.GetManifestInfos()) + { + var config = SBOMConfigs.Get(manifestInfo); + if (config.MetadataBuilder.TryGetExternalRefArrayHeaderName(out string externalRefArrayHeaderName)) + { + externalRefArraySupportingConfigs.Add(config); + config.JsonSerializer.StartJsonArray(externalRefArrayHeaderName); + } + } + + foreach (var sourcesProvider in sourcesProviders) + { + var (jsonDocResults, errors) = sourcesProvider.Get(externalRefArraySupportingConfigs); + + // Collect all the json elements and write to the serializer. + int totalJsonDocumentsWritten = 0; + + await foreach (JsonDocWithSerializer jsonResults in jsonDocResults.ReadAllAsync()) + { + jsonResults.Serializer.Write(jsonResults.Document); + totalJsonDocumentsWritten++; + } + + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + totalErrors.Add(error); + } + } + + // Write the end of the array. + foreach (SbomConfig config in externalRefArraySupportingConfigs) + { + config.JsonSerializer.EndJsonArray(); + } + + return totalErrors; + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs new file mode 100644 index 00000000..62e9f35a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/FileArrayGenerator.cs @@ -0,0 +1,97 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Providers; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Ninject; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows.Helpers +{ + /// + /// This class generates an array of filenames and hashes based on the format of the SBOM. + /// + public class FileArrayGenerator : IJsonArrayGenerator + { + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + public ISbomConfigProvider SBOMConfigs { get; set; } + + [Inject] + public IList SourcesProviders { get; set; } + + [Inject] + public IRecorder Recorder { get; set; } + + /// + /// Traverses all the files inside the buildDropPath, and serializes the SBOM using the JSON serializer creating + /// an array object whose key is defined by . Upon failure, returns a list of + /// objects that can be used to trace the error. + /// + /// The serializer used to write the SBOM. + /// The header key for the file array object. + /// + public async Task> GenerateAsync() + { + using (Recorder.TraceEvent(Events.FilesGeneration)) + { + IList totalErrors = new List(); + + IEnumerable sourcesProviders = SourcesProviders + .Where(s => s.IsSupported(ProviderType.Files)); + + // Write the start of the array, if supported. + IList filesArraySupportingSBOMs = new List(); + foreach (var manifestInfo in SBOMConfigs.GetManifestInfos()) + { + var config = SBOMConfigs.Get(manifestInfo); + + if (config.MetadataBuilder.TryGetFilesArrayHeaderName(out string filesArrayHeaderName)) + { + config.JsonSerializer.StartJsonArray(filesArrayHeaderName); + filesArraySupportingSBOMs.Add(config); + } + } + + foreach (var sourcesProvider in sourcesProviders) + { + var (jsondDocResults, errors) = sourcesProvider.Get(filesArraySupportingSBOMs); + + await foreach (JsonDocWithSerializer jsonResults in jsondDocResults.ReadAllAsync()) + { + jsonResults.Serializer.Write(jsonResults.Document); + } + + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + // TODO fix errors. + if (error.ErrorType != ErrorType.ManifestFolder) + { + totalErrors.Add(error); + } + } + } + + // Write the end of the array. + foreach (ISbomConfig config in filesArraySupportingSBOMs) + { + config.JsonSerializer.EndJsonArray(); + } + + return totalErrors; + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs new file mode 100644 index 00000000..507be502 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/IJsonArrayGenerator.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Output; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Workflows.Helpers +{ + /// + /// Used to generate array objects in the JSON serializer. + /// + public interface IJsonArrayGenerator + { + /// + /// Generates an array in the json serializer with the headerName and writes all elements of the + /// specific type into the array. + /// + /// The list of failures. + Task> GenerateAsync(); + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs new file mode 100644 index 00000000..286c01ea --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/PackageArrayGenerator.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Providers; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Ninject; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows.Helpers +{ + /// + /// Generates a packages array that contains a list of all the packages that are referenced in this project. + /// + public class PackageArrayGenerator : IJsonArrayGenerator + { + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ChannelUtils ChannelUtils { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + public ISbomConfigProvider SBOMConfigs { get; set; } + + [Inject] + public IList SourcesProviders { get; set; } + + [Inject] + public IRecorder Recorder { get; set; } + + public async Task> GenerateAsync() + { + using (Recorder.TraceEvent(Events.PackagesGeneration)) + { + IList totalErrors = new List(); + + ISourcesProvider sourcesProvider = SourcesProviders + .Where(s => s.IsSupported(ProviderType.Packages)) + .FirstOrDefault(); + + // Write the start of the array, if supported. + IList packagesArraySupportingConfigs = new List(); + foreach (var manifestInfo in SBOMConfigs.GetManifestInfos()) + { + var config = SBOMConfigs.Get(manifestInfo); + if (config.MetadataBuilder.TryGetPackageArrayHeaderName(out string packagesArrayHeaderName)) + { + packagesArraySupportingConfigs.Add(config); + config.JsonSerializer.StartJsonArray(packagesArrayHeaderName); + } + } + + var (jsonDocResults, errors) = sourcesProvider.Get(packagesArraySupportingConfigs); + + // 6. Collect all the json elements and write to the serializer. + int totalJsonDocumentsWritten = 0; + + await foreach (JsonDocWithSerializer jsonDocResult in jsonDocResults.ReadAllAsync()) + { + jsonDocResult.Serializer.Write(jsonDocResult.Document); + totalJsonDocumentsWritten++; + } + + Log.Debug($"Wrote {totalJsonDocumentsWritten} package elements in the SBOM."); + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + totalErrors.Add(error); + } + + foreach (ISbomConfig sbomConfig in packagesArraySupportingConfigs) + { + // Write the root package information to the packages array. + if (sbomConfig.MetadataBuilder.TryGetRootPackageJson(SBOMConfigs, out GenerationResult generationResult)) + { + sbomConfig.JsonSerializer.Write(generationResult?.Document); + sbomConfig.Recorder.RecordRootPackageId(generationResult?.ResultMetadata?.EntityId); + sbomConfig.Recorder.RecordDocumentId(generationResult?.ResultMetadata?.DocumentId); + } + + // Write the end of the array. + sbomConfig.JsonSerializer.EndJsonArray(); + } + + return totalErrors; + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs new file mode 100644 index 00000000..cf1578ac --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/Helpers/RelationshipsArrayGenerator.cs @@ -0,0 +1,160 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Utils; +using Ninject; +using Serilog; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows.Helpers +{ + /// + /// Generates an array of relationships between different elements of the SBOM. + /// + public class RelationshipsArrayGenerator : IJsonArrayGenerator + { + [Inject] + public RelationshipGenerator Generator { get; set; } + + [Inject] + public ChannelUtils ChannelUtils { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + public ISbomConfigProvider SbomConfigs { get; set; } + + [Inject] + public IRecorder Recorder { get; set; } + + public async Task> GenerateAsync() + { + using (Recorder.TraceEvent(Events.RelationshipsGeneration)) + { + IList totalErrors = new List(); + + // Write the relationship array only if supported + foreach (var manifestInfo in SbomConfigs.GetManifestInfos()) + { + var sbomConfig = SbomConfigs.Get(manifestInfo); + if (sbomConfig.MetadataBuilder.TryGetRelationshipsHeaderName(out string relationshipArrayHeaderName)) + { + sbomConfig.JsonSerializer.StartJsonArray(relationshipArrayHeaderName); + + // Get generation data + var generationData = sbomConfig.Recorder.GetGenerationData(); + + var jsonChannelsArray = new ChannelReader[] + { + // Packages relationships + Generator.Run( + GetRelationships( + RelationshipType.DEPENDS_ON, + generationData.RootPackageId, + generationData.PackageIds), + sbomConfig.ManifestInfo), + + // Root package relationship + Generator.Run( + GetRelationships( + RelationshipType.DESCRIBES, + generationData.DocumentId, + new string[] { generationData.RootPackageId }), + sbomConfig.ManifestInfo), + + // External reference relationship + Generator.Run( + GetRelationships( + RelationshipType.PREREQUISITE_FOR, + generationData.RootPackageId, + generationData.ExternalDocumentReferenceIDs), + sbomConfig.ManifestInfo), + + // External reference file relationship + Generator.Run( + GetRelationships( + RelationshipType.DESCRIBED_BY, + generationData.SPDXFileIds, + generationData.DocumentId), + sbomConfig.ManifestInfo), + }; + + // Collect all the json elements and write to the serializer. + int count = 0; + + await foreach (JsonDocument jsonDoc in ChannelUtils.Merge(jsonChannelsArray).ReadAllAsync()) + { + count++; + sbomConfig.JsonSerializer.Write(jsonDoc); + } + + Log.Debug($"Wrote {count} relationship elements in the SBOM."); + + // Write the end of the array. + sbomConfig.JsonSerializer.EndJsonArray(); + } + } + + return totalErrors; + } + } + + private IEnumerator GetRelationships(RelationshipType relationshipType, string sourceElementId, IEnumerable targetElementIds) + { + foreach (var targetElementId in targetElementIds) + { + if (targetElementId != null || sourceElementId != null) + { + yield return new Relationship + { + RelationshipType = relationshipType, + TargetElementId = targetElementId, + SourceElementId = sourceElementId + }; + } + } + } + + private IEnumerator GetRelationships(RelationshipType relationshipType, IList sourceElementIds, string targetElementId) + { + foreach (var sourceElementId in sourceElementIds) + { + if (sourceElementId != null || targetElementId != null) + { + yield return new Relationship + { + RelationshipType = relationshipType, + SourceElementId = sourceElementId, + TargetElementId = targetElementId, + }; + } + } + } + + private IEnumerator GetRelationships(RelationshipType relationshipType, string sourceElementId, IEnumerable> targetElementIds) + { + foreach (var targetElementId in targetElementIds) + { + if (sourceElementId != null || targetElementId.Key != null || targetElementId.Value != null) + { + yield return new Relationship + { + RelationshipType = relationshipType, + TargetElementId = targetElementId.Value, + TargetElementExternalReferenceId = targetElementId.Key, + SourceElementId = sourceElementId + }; + } + } + } + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/IWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/IWorkflow.cs new file mode 100644 index 00000000..033fcf45 --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/IWorkflow.cs @@ -0,0 +1,15 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Workflows +{ + /// + /// Defines the workflow run for a given action. + /// + public interface IWorkflow + { + public Task RunAsync(); + } +} diff --git a/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs new file mode 100644 index 00000000..63945c7a --- /dev/null +++ b/src/Microsoft.Sbom.Api/Workflows/SBOMGenerationWorkflow.cs @@ -0,0 +1,217 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Hashing.Algorithms; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Api.Workflows.Helpers; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Ninject; +using PowerArgs; +using Serilog; +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Workflows +{ + /// + /// The SBOM tool workflow class that is used to generate a SBOM + /// file for a given build root path. + /// + public class SBOMGenerationWorkflow : IWorkflow + { + [Inject] + public IFileSystemUtils FileSystemUtils { get; set; } + + [Inject] + public IConfiguration Configuration { get; set; } + + [Inject] + public ILogger Log { get; set; } + + [Inject] + [Named(nameof(FileArrayGenerator))] + public IJsonArrayGenerator FileArrayGenerator { get; set; } + + [Inject] + [Named(nameof(PackageArrayGenerator))] + public IJsonArrayGenerator PackageArrayGenerator { get; set; } + + [Inject] + [Named(nameof(RelationshipsArrayGenerator))] + public IJsonArrayGenerator RelationshipsArrayGenerator { get; set; } + + [Inject] + [Named(nameof(ExternalDocumentReferenceGenerator))] + public IJsonArrayGenerator ExternalDocumentReferenceGenerator { get; set; } + + [Inject] + public ISbomConfigProvider SBOMConfigs { get; set; } + + [Inject] + public IOSUtils OSUtils { get; set; } + + [Inject] + public IRecorder Recorder { get; set; } + + public virtual async Task RunAsync() + { + IList validErrors = new List(); + string sbomDir = null; + bool deleteSBOMDir = false; + using (Recorder.TraceEvent(Events.SBOMGenerationWorkflow)) + { + try + { + Log.Debug("Starting SBOM generation workflow."); + + sbomDir = Configuration.ManifestDirPath.Value; + + // Don't remove directory if path is provided by user, there could be other files in that directory + if (Configuration.ManifestDirPath.IsDefaultSource) + { + RemoveExistingManifestDirectory(); + } + + await using (SBOMConfigs.StartJsonSerializationAsync()) + { + SBOMConfigs.ApplyToEachConfig(config => config.JsonSerializer.StartJsonObject()); + + // Files section + validErrors = await FileArrayGenerator.GenerateAsync(); + + // Packages section + validErrors.Concat(await PackageArrayGenerator.GenerateAsync()); + + // External Document Reference section + validErrors.Concat(await ExternalDocumentReferenceGenerator.GenerateAsync()); + + // Relationships section + validErrors.Concat(await RelationshipsArrayGenerator.GenerateAsync()); + + // Write headers + SBOMConfigs.ApplyToEachConfig(config => config.JsonSerializer.WriteJsonString(config.MetadataBuilder.GetHeaderJsonString(SBOMConfigs))); + + // Finalize JSON + SBOMConfigs.ApplyToEachConfig(config => config.JsonSerializer.FinalizeJsonObject()); + } + + // Generate SHA256 for manifest json + SBOMConfigs.ApplyToEachConfig(config => GenerateHashForManifestJson(config.ManifestJsonFilePath)); + + return !validErrors.Any(); + } + catch (Exception e) + { + Recorder.RecordException(e); + Log.Error("Encountered an error while generating the manifest."); + Log.Error($"Error details: {e.Message}"); + deleteSBOMDir = true; + + // TODO: Create EntityError with exception message and record to surface unexpected exceptions to client. + return false; + } + finally + { + if (validErrors != null) + { + Recorder.RecordTotalErrors(validErrors); + } + + // Delete the generated _manifest folder if generation failed. + if (deleteSBOMDir || validErrors.Any()) + { + DeleteManifestFolder(sbomDir); + } + } + } + + void DeleteManifestFolder(string sbomDir) + { + try + { + if (!string.IsNullOrEmpty(sbomDir) && FileSystemUtils.DirectoryExists(sbomDir)) + { + if (Configuration.ManifestDirPath.IsDefaultSource) + { + FileSystemUtils.DeleteDir(sbomDir, true); + } + else if (!FileSystemUtils.IsDirectoryEmpty(sbomDir)) + { + Log.Warning($"Manifest generation failed, however we were " + + $"unable to delete the partially generated manifest.json file and the {sbomDir} directory because the directory was not empty."); + } + } + } + catch (Exception e) + { + Log.Warning( + $"Manifest generation failed, however we were " + + $"unable to delete the partially generated manifest.json file and the {sbomDir} directory.", e); + } + } + } + + private void GenerateHashForManifestJson(string manifestJsonFilePath) + { + if (!FileSystemUtils.FileExists(manifestJsonFilePath)) + { + Log.Warning($"Failed to create manifest hash because the manifest json file does not exist."); + return; + } + + string hashFileName = $"{manifestJsonFilePath}.sha256"; + + using var readStream = FileSystemUtils.OpenRead(manifestJsonFilePath); + using var bufferedStream = new BufferedStream(readStream, 1024 * 32); + using var writeFileStream = FileSystemUtils.OpenWrite(hashFileName); + var hashValue = Encoding.Unicode.GetBytes(BitConverter.ToString(new Sha256HashAlgorithm().ComputeHash(bufferedStream)).Replace("-", string.Empty).ToLower()); + writeFileStream.Write(hashValue, 0, hashValue.Length); + } + + private void RemoveExistingManifestDirectory() + { + var rootManifestFolderPath = Configuration.ManifestDirPath.Value; + + try + { + // If the _manifest directory already exists, we must delete it first to avoid having + // multiple SBOMs for the same drop. However, the default behaviour is to fail with an + // Exception since we don't want to inadvertently delete someone else's data. This behaviour + // can be overridden by setting an environment variable. + if (FileSystemUtils.DirectoryExists(rootManifestFolderPath)) + { + bool.TryParse( + OSUtils.GetEnvironmentVariable(Constants.DeleteManifestDirBoolVariableName), + out bool deleteSbomDirSwitch); + + Recorder.RecordSwitch(Constants.DeleteManifestDirBoolVariableName, deleteSbomDirSwitch); + + if (!deleteSbomDirSwitch) + { + throw new Exception($"The BuildDropRoot folder already contains a _manifest folder. Please" + + $" delete this folder before running the generation or set the " + + $"{Constants.DeleteManifestDirBoolVariableName} environment variable to 'true' to " + + $"overwrite this folder."); + } + + Log.Warning($"Deleting pre-existing folder {rootManifestFolderPath} as {Constants.DeleteManifestDirBoolVariableName}" + + $" is 'true'."); + FileSystemUtils.DeleteDir(rootManifestFolderPath, true); + } + } + catch (Exception e) + { + throw new ValidationArgException($"Unable to create manifest directory at path {rootManifestFolderPath}. Error: {e.Message}"); + } + } + } +} diff --git a/src/Microsoft.Sbom.Common/FileSystemUtils.cs b/src/Microsoft.Sbom.Common/FileSystemUtils.cs index 89f9492f..5a6db824 100644 --- a/src/Microsoft.Sbom.Common/FileSystemUtils.cs +++ b/src/Microsoft.Sbom.Common/FileSystemUtils.cs @@ -14,25 +14,25 @@ namespace Microsoft.Sbom.Common /// public class FileSystemUtils : IFileSystemUtils { - private readonly EnumerationOptions _dontFollowSymlinks = new EnumerationOptions + private readonly EnumerationOptions dontFollowSymlinks = new EnumerationOptions { AttributesToSkip = FileAttributes.ReparsePoint }; - private const string _searchAllFilesAndFolders = "*"; + private const string SearchAllFilesAndFolders = "*"; public bool DirectoryExists(string path) => Directory.Exists(path); - public IEnumerable GetDirectories(string path, bool followSymlinks = true) => (followSymlinks) switch + public IEnumerable GetDirectories(string path, bool followSymlinks = true) => followSymlinks switch { true => Directory.GetDirectories(path), - false => Directory.GetDirectories(path, _searchAllFilesAndFolders, _dontFollowSymlinks) + false => Directory.GetDirectories(path, SearchAllFilesAndFolders, dontFollowSymlinks) }; - public IEnumerable GetFilesInDirectory(string path, bool followSymlinks = true) => (followSymlinks) switch + public IEnumerable GetFilesInDirectory(string path, bool followSymlinks = true) => followSymlinks switch { true => Directory.GetFiles(path), - false => Directory.GetFiles(path, _searchAllFilesAndFolders, _dontFollowSymlinks) + false => Directory.GetFiles(path, SearchAllFilesAndFolders, dontFollowSymlinks) }; public DirectorySecurity GetDirectorySecurity(string directoryPath) => new DirectoryInfo(directoryPath).GetAccessControl(); @@ -52,7 +52,8 @@ public class FileSystemUtils : IFileSystemUtils public bool FileExists(string path) => File.Exists(path); - public Stream OpenWrite(string filePath) => new FileStream(filePath, + public Stream OpenWrite(string filePath) => new FileStream( + filePath, FileMode.Create, FileAccess.Write, FileShare.Delete, diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Entities/FileEntity.cs b/src/Microsoft.Sbom.Contracts/Contracts/Entities/FileEntity.cs index e2829d90..526b15f8 100644 --- a/src/Microsoft.Sbom.Contracts/Contracts/Entities/FileEntity.cs +++ b/src/Microsoft.Sbom.Contracts/Contracts/Entities/FileEntity.cs @@ -31,7 +31,7 @@ public FileEntity(string path, string id = null) /// public override string ToString() { - return $"FileEntity (Path={Path}{(Id == null ? string.Empty : ", Id="+Id)})"; + return $"FileEntity (Path={Path}{(Id == null ? string.Empty : ", Id=" + Id)})"; } } } diff --git a/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs b/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs index c3426502..ef08a09e 100644 --- a/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs +++ b/src/Microsoft.Sbom.Contracts/Contracts/Enums/AlgorithmName.cs @@ -72,19 +72,18 @@ public override int GetHashCode() #pragma warning restore CA5350 /// - /// Gets equivalent to + /// Gets equivalent to . /// public static AlgorithmName SHA256 => new AlgorithmName(nameof(SHA256), stream => System.Security.Cryptography.SHA256.Create().ComputeHash(stream)); /// - /// Gets equivalent to + /// Gets equivalent to . /// - public static AlgorithmName SHA512 => new AlgorithmName(nameof(SHA512), stream => throw new ArgumentException($"Unsupported hash algorithm {nameof(SHA512)}")); + public static AlgorithmName SHA512 => new AlgorithmName(nameof(SHA512), stream => System.Security.Cryptography.SHA512.Create().ComputeHash(stream)); /// - /// Gets equivalent to + /// Gets equivalent to . /// - public static AlgorithmName MD5 => new AlgorithmName(nameof(MD5), stream => throw new ArgumentException($"Unsupported hash algorithm {nameof(MD5)}")); - + public static AlgorithmName MD5 => new AlgorithmName(nameof(MD5), stream => System.Security.Cryptography.MD5.Create().ComputeHash(stream)); } } diff --git a/src/Microsoft.Sbom.Contracts/ISBOMGenerator.cs b/src/Microsoft.Sbom.Contracts/ISBOMGenerator.cs index 4aaec277..1e438a0e 100644 --- a/src/Microsoft.Sbom.Contracts/ISBOMGenerator.cs +++ b/src/Microsoft.Sbom.Contracts/ISBOMGenerator.cs @@ -14,7 +14,7 @@ namespace Microsoft.Sbom.Contracts public interface ISBOMGenerator { /// - /// Generate a SBOM in the rootPath using the provided file and package lists + /// Generate a SBOM in the rootPath using the provided file and package lists. /// /// The root path of the drop where the generated SBOM will be placed. /// The list of files to include in this SBOM. @@ -45,10 +45,11 @@ Task GenerateSBOMAsync( /// Provide a list of that you want your SBOM to be generated /// for. If this is not provided, we will generate SBOMs for all the available formats. /// Configuration to tweak the SBOM generator workflow. - /// Output directory. If null defaults to rootPath joined to _manifest + /// Output directory. If null defaults to rootPath joined to _manifest. /// The result object that indicates if the generation succeeded, and a list of /// errors if it failed along with telemetry. - Task GenerateSBOMAsync(string rootPath, + Task GenerateSBOMAsync( + string rootPath, string componentPath, SBOMMetadata metadata, IList specifications = null, @@ -61,14 +62,14 @@ Task GenerateSBOMAsync(string rootPath, /// generated for them. Use this function to get a list of the required hash algorithms for your /// SBOM specification. The SBOM generator may throw an exception if a hash algorithm value is missing. /// - /// The SBOM specification - /// A list of + /// The SBOM specification. + /// A list of . IEnumerable GetRequiredAlgorithms(SBOMSpecification specification); /// /// Gets a list of this SBOM generator supports. /// - /// A list of + /// A list of . IEnumerable GetSupportedSBOMSpecifications(); } } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Checksum.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Checksum.cs index d1fcd712..bf291a3f 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Checksum.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Checksum.cs @@ -11,14 +11,14 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class Checksum { /// - /// The name of the hash algorithm. + /// Gets or sets the name of the hash algorithm. /// [JsonPropertyName("algorithm")] public string Algorithm { get; set; } /// - /// The string value of the computed hash. + /// Gets or sets the string value of the computed hash. /// [JsonPropertyName("checksumValue")] public string ChecksumValue { get; set; } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/CreationInfo.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/CreationInfo.cs index c271cc8c..297e2447 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/CreationInfo.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/CreationInfo.cs @@ -12,13 +12,13 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class CreationInfo { /// - /// A string that specifies the time the SBOM was created on. + /// Gets or sets a string that specifies the time the SBOM was created on. /// [JsonPropertyName("created")] public string Created { get; set; } /// - /// A list of strings that specify metadata about the creators of this + /// Gets or sets a list of strings that specify metadata about the creators of this /// SBOM. This could be the person or organization name, or tool name, etc. /// [JsonPropertyName("creators")] diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Enums/ExternalRepositoryType.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Enums/ExternalRepositoryType.cs index 57d68e75..7d4f3c6b 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Enums/ExternalRepositoryType.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/Enums/ExternalRepositoryType.cs @@ -5,7 +5,6 @@ namespace Microsoft.SPDX22SBOMParser.Entities.Enums { - #pragma warning disable SA1629 // Documentation text should end with a period /// /// Type of the external reference. These are definined in an appendix in the SPDX specification. @@ -23,23 +22,23 @@ public enum ExternalRepositoryType #region Persistent-Id - swh, + Swh, #endregion #region Package-Manager - maven_central, - npm, - nuget, - bower, - purl, + Maven_central, + Npm, + Nuget, + Bower, + Purl, #endregion #region Other - idstring + Idstring #endregion } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/ExternalReference.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/ExternalReference.cs index a77c9c29..a11e7b5c 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/ExternalReference.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/ExternalReference.cs @@ -14,20 +14,20 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class ExternalReference { /// - /// The category for the external reference. + /// Gets or sets the category for the external reference. /// [JsonPropertyName("referenceCategory")] public ReferenceCategory ReferenceCategory { get; set; } /// - /// Type of the external reference. These are definined in an appendix in the SPDX specification. + /// Gets or sets type of the external reference. These are definined in an appendix in the SPDX specification. /// https://spdx.github.io/spdx-spec/appendix-VI-external-repository-identifiers/ /// [JsonPropertyName("referenceType")] public ExternalRepositoryType Type { get; set; } /// - /// A unique string without any spaces that specifies a location where the package specific information + /// Gets or sets a unique string without any spaces that specifies a location where the package specific information /// can be located. The locator constraints are defined by the /// [JsonPropertyName("referenceLocator")] diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/PackageVerificationCode.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/PackageVerificationCode.cs index 40b8f78b..c1515ec0 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/PackageVerificationCode.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/PackageVerificationCode.cs @@ -13,13 +13,13 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class PackageVerificationCode { /// - /// The actual package verification code as a hex encoded value. + /// Gets or sets the actual package verification code as a hex encoded value. /// [JsonPropertyName("packageVerificationCodeValue")] public string PackageVerificationCodeValue { get; set; } /// - /// Files that were excluded when calculating the package verification code. + /// Gets or sets files that were excluded when calculating the package verification code. /// [JsonPropertyName("packageVerificationCodeExcludedFiles")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDX22Document.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDX22Document.cs index 9dcbc346..6be8d7be 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDX22Document.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDX22Document.cs @@ -9,48 +9,48 @@ namespace Microsoft.SPDX22SBOMParser.Entities internal class SPDX22Document { /// - /// Reference number for the version to understand how to parse and interpret the format. + /// Gets or sets reference number for the version to understand how to parse and interpret the format. /// [JsonPropertyName("spdxVersion")] public string SPDXVersion { get; set; } /// - /// License for compliance with the SPDX specification + /// Gets or sets license for compliance with the SPDX specification /// [JsonPropertyName("dataLicense")] public string DataLicense { get; set; } /// - /// Unique Identifier for elements in SPDX document. + /// Gets or sets unique Identifier for elements in SPDX document. /// public string SPDXID { get; set; } /// - /// Identify name of this document as designated by creator. + /// Gets or sets identify name of this document as designated by creator. /// [JsonPropertyName("name")] public string DocumentName { get; set; } /// - /// SPDX document specific namespace as a URI + /// Gets or sets sPDX document specific namespace as a URI /// [JsonPropertyName("documentNamespace")] public string DocumentNamespace { get; set; } /// - /// Provides the necessary information for forward and backward compatibility for processing tools. + /// Gets or sets provides the necessary information for forward and backward compatibility for processing tools. /// [JsonPropertyName("creationInfo")] public CreationInfo CreationInfo { get; set; } /// - /// Files referenced in the SPDX document. + /// Gets or sets files referenced in the SPDX document. /// [JsonPropertyName("files")] public List Files { get; set; } /// - /// Packages referenced in the SPDX document. + /// Gets or sets packages referenced in the SPDX document. /// [JsonPropertyName("packages")] public List Packages { get; set; } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXPackage.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXPackage.cs index 9bd6e8f1..aed55ba1 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXPackage.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXPackage.cs @@ -7,74 +7,74 @@ namespace Microsoft.SPDX22SBOMParser.Entities { /// - /// Represents a SPDX 2.2 Package + /// Represents a SPDX 2.2 Package. /// public class SPDXPackage { /// - /// Name of the package. + /// Gets or sets name of the package. /// [JsonPropertyName("name")] public string Name { get; set; } /// - /// Name of the package. + /// Gets or sets name of the package. /// [JsonPropertyName("packageFileName")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] public string PackageFileName { get; set; } /// - /// Unique Identifier for elements in SPDX document. + /// Gets or sets unique Identifier for elements in SPDX document. /// [JsonPropertyName("SPDXID")] public string SpdxId { get; set; } /// - /// The download URL for the exact package, NONE for no download location and NOASSERTION for no attempt. + /// Gets or sets the download URL for the exact package, NONE for no download location and NOASSERTION for no attempt. /// [JsonPropertyName("downloadLocation")] public string DownloadLocation { get; set; } /// - /// Used to identify specific contents of a package based on actual files that make up each package. + /// Gets or sets used to identify specific contents of a package based on actual files that make up each package. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("packageVerificationCode")] public PackageVerificationCode PackageVerificationCode { get; set; } /// - /// If set, specifies if the individual files inside this package were analyzed to capture more data. + /// Gets or sets a value indicating whether if set, specifies if the individual files inside this package were analyzed to capture more data. /// [JsonPropertyName("filesAnalyzed")] public bool FilesAnalyzed { get; set; } /// - /// Contain the license the SPDX file creator has concluded as the package or alternative values. + /// Gets or sets contain the license the SPDX file creator has concluded as the package or alternative values. /// [JsonPropertyName("licenseConcluded")] public string LicenseConcluded { get; set; } /// - /// Contains all license found in the package. + /// Gets or sets contains all license found in the package. /// [JsonPropertyName("licenseInfoFromFiles")] public List LicenseInfoFromFiles { get; set; } /// - /// Contains a list of licenses the have been declared by the authors of the package. + /// Gets or sets contains a list of licenses the have been declared by the authors of the package. /// [JsonPropertyName("licenseDeclared")] public string LicenseDeclared { get; set; } /// - /// Copyright holder of the package, as well as any dates present. + /// Gets or sets copyright holder of the package, as well as any dates present. /// [JsonPropertyName("copyrightText")] public string CopyrightText { get; set; } /// - /// Version of the package. + /// Gets or sets version of the package. /// Not Required /// [JsonPropertyName("versionInfo")] @@ -82,7 +82,7 @@ public class SPDXPackage public string VersionInfo { get; set; } /// - /// Provide an independently reproducible mechanism that permits unique identification of a specific + /// Gets or sets provide an independently reproducible mechanism that permits unique identification of a specific /// package that correlates to the data in this SPDX file /// [JsonPropertyName("checksums")] @@ -90,7 +90,7 @@ public class SPDXPackage public List Checksums { get; set; } /// - /// Provide a list of that provide additional information or metadata + /// Gets or sets provide a list of that provide additional information or metadata /// about this package. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] @@ -98,13 +98,13 @@ public class SPDXPackage public IList ExternalReferences { get; set; } /// - /// The name and optional contact information of the person or organization that built this package. + /// Gets or sets the name and optional contact information of the person or organization that built this package. /// [JsonPropertyName("supplier")] public string Supplier { get; set; } /// - /// The list of file ids that are contained in this package. + /// Gets or sets the list of file ids that are contained in this package. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("hasFiles")] diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXRelationship.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXRelationship.cs index 3fef2f2e..b10ad1d4 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXRelationship.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SPDXRelationship.cs @@ -12,19 +12,19 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class SPDXRelationship { /// - /// Defines the type of the relationship between the source and the target element. + /// Gets or sets defines the type of the relationship between the source and the target element. /// [JsonPropertyName("relationshipType")] public SPDXRelationshipType RelationshipType { get; set; } /// - /// The id of the target element with whom the source element has a relationship. + /// Gets or sets the id of the target element with whom the source element has a relationship. /// [JsonPropertyName("relatedSpdxElement")] public string TargetElementId { get; set; } /// - /// The id of the target element with whom the source element has a relationship. + /// Gets or sets the id of the target element with whom the source element has a relationship. /// [JsonPropertyName("spdxElementId")] public string SourceElementId { get; set; } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxExternalDocumentReference.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxExternalDocumentReference.cs index 76be8f6c..e9b99b5f 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxExternalDocumentReference.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxExternalDocumentReference.cs @@ -11,19 +11,19 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class SpdxExternalDocumentReference { /// - /// Unique Identifier for ExternalDocumentReference in SPDX document. + /// Gets or sets unique Identifier for ExternalDocumentReference in SPDX document. /// [JsonPropertyName("externalDocumentId")] public string ExternalDocumentId { get; set; } /// - /// Document namespace of the input SBOM + /// Gets or sets document namespace of the input SBOM /// [JsonPropertyName("spdxDocument")] public string SpdxDocument { get; set; } /// - /// Checksum values for External SBOM file + /// Gets or sets checksum values for External SBOM file /// [JsonPropertyName("checksum")] public Checksum Checksum { get; set; } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxFile.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxFile.cs index 75106b9d..a14c4b36 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxFile.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Entities/SpdxFile.cs @@ -10,44 +10,44 @@ namespace Microsoft.SPDX22SBOMParser.Entities public class SPDXFile { /// - /// Identify the full path and filename that corresponds to the file information. + /// Gets or sets identify the full path and filename that corresponds to the file information. /// [JsonPropertyName("fileName")] public string FileName { get; set; } /// - /// Unique Identifier for elements in SPDX document. + /// Gets or sets unique Identifier for elements in SPDX document. /// [JsonPropertyName("SPDXID")] public string SPDXId { get; set; } /// - /// Provide a unique identifier to match analysis information on each specific file in a package. + /// Gets or sets provide a unique identifier to match analysis information on each specific file in a package. /// [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] [JsonPropertyName("checksums")] public List FileChecksums { get; set; } /// - /// Contain the license the SPDX file creator has concluded as the package or alternative values. + /// Gets or sets contain the license the SPDX file creator has concluded as the package or alternative values. /// [JsonPropertyName("licenseConcluded")] public string LicenseConcluded { get; set; } /// - /// Contains the license information actually found in the file, if any. + /// Gets or sets contains the license information actually found in the file, if any. /// [JsonPropertyName("licenseInfoInFiles")] public List LicenseInfoInFiles { get; set; } /// - /// Copyright holder of the package, as well as any dates present. + /// Gets or sets copyright holder of the package, as well as any dates present. /// [JsonPropertyName("copyrightText")] public string FileCopyrightText { get; set; } /// - /// Provides a reasonable estimation of the file type. + /// Gets or sets provides a reasonable estimation of the file type. /// [JsonPropertyName("fileTypes")] [JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Generator.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Generator.cs index 8ff19b83..2be3ab6c 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Generator.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Generator.cs @@ -42,7 +42,10 @@ public Generator() public GenerationResult GenerateJsonDocument(InternalSBOMFileInfo fileInfo) { - if (fileInfo is null) throw new ArgumentNullException(nameof(fileInfo)); + if (fileInfo is null) + { + throw new ArgumentNullException(nameof(fileInfo)); + } var spdxFileElement = ConvertSbomFileToSpdxFile(fileInfo); return new GenerationResult @@ -135,11 +138,11 @@ public IDictionary GetMetadataDictionary(IInternalMetadataProvid { { Constants.SPDXVersionHeaderName, Version }, { Constants.DataLicenseHeaderName, Constants.DataLicenceValue }, - { Constants.SPDXIDHeaderName, Constants.SPDXDocumentIdValue}, + { Constants.SPDXIDHeaderName, Constants.SPDXDocumentIdValue }, { Constants.DocumentNameHeaderName, documentName }, - { Constants.DocumentNamespaceHeaderName, identityUtils.GetDocumentNamespace(internalMetadataProvider)}, + { Constants.DocumentNamespaceHeaderName, identityUtils.GetDocumentNamespace(internalMetadataProvider) }, { Constants.CreationInfoHeaderName, creationInfo }, - { Constants.DocumentDescribesHeaderName, new string [] { generationData.RootPackageId } } + { Constants.DocumentDescribesHeaderName, new string[] { generationData.RootPackageId } } }; } diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Microsoft.Sbom.SPDX22SBOMParser.csproj b/src/Microsoft.Sbom.SPDX22SBOMParser/Microsoft.Sbom.SPDX22SBOMParser.csproj index deefb7ab..babc43c5 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Microsoft.Sbom.SPDX22SBOMParser.csproj +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Microsoft.Sbom.SPDX22SBOMParser.csproj @@ -1,4 +1,4 @@ - + netstandard2.0 @@ -10,10 +10,6 @@ $(SBOMPackageVersion) - - - - diff --git a/src/Microsoft.Sbom.SPDX22SBOMParser/Utils/SPDXExtensions.cs b/src/Microsoft.Sbom.SPDX22SBOMParser/Utils/SPDXExtensions.cs index 1bdefb3f..c5c89a6e 100644 --- a/src/Microsoft.Sbom.SPDX22SBOMParser/Utils/SPDXExtensions.cs +++ b/src/Microsoft.Sbom.SPDX22SBOMParser/Utils/SPDXExtensions.cs @@ -82,7 +82,7 @@ public static void AddPackageUrls(this SPDXPackage spdxPackage, SBOMPackage pack var extRef = new ExternalReference { ReferenceCategory = ReferenceCategory.PACKAGE_MANAGER, - Type = ExternalRepositoryType.purl, + Type = ExternalRepositoryType.Purl, Locator = FormatPackageUrl(packageInfo.PackageUrl) }; diff --git a/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs b/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs index 6cf077d1..284297fe 100644 --- a/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs +++ b/test/Microsoft.Sbom.Adapters.Tests/ComponentDetectionToSBOMPackageAdapterTests.cs @@ -1,4 +1,7 @@ -using Microsoft.VisualStudio.TestTools.UnitTesting; +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; using System; using System.Collections.Generic; using System.IO; diff --git a/test/Microsoft.Sbom.Api.Tests/ApiConfigurationBuilderTests.cs b/test/Microsoft.Sbom.Api.Tests/ApiConfigurationBuilderTests.cs new file mode 100644 index 00000000..f775d991 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/ApiConfigurationBuilderTests.cs @@ -0,0 +1,191 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Config; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Serilog.Events; +using System; +using System.Collections.Generic; +using System.Diagnostics.Tracing; +using System.Linq; + +namespace Microsoft.Sbom.Api.Tests +{ + /// + /// Responsible for testing + /// + [TestClass] + public class ApiConfigurationBuilderTests + { + private const string rootPath = @"D:\TMP"; + private const int MinParallelism = 2; + private const int DefaultParallelism = 8; + private const int MaxParallelism = 48; + private const string packageName = "packageName"; + private const string packageVersion = "packageVersion"; + + ApiConfigurationBuilder builder = new ApiConfigurationBuilder(); + SBOMMetadata metadata = new SBOMMetadata() + { + PackageName = packageName, + PackageVersion = packageVersion, + }; + RuntimeConfiguration runtime = new RuntimeConfiguration() + { + Verbosity = EventLevel.Verbose, + WorkflowParallelism = DefaultParallelism, + DeleteManifestDirectoryIfPresent = true + }; + string manifestDirPath = "manifestDirPath"; + List files = new List(); + List packages = new List(); + string externalDocumentRefListFile = "externalDocRef"; + string componentPath = @"D:\COMPONENT"; + + [TestMethod] + public void GetConfiguration_PopulateAll() + { + List specs = new List(); + specs.Add(new SBOMSpecification("spdx", "2.2")); + + var expectedManifestInfo = new ManifestInfo() + { + Name = "spdx", + Version = "2.2" + }; + + var config = builder.GetConfiguration(rootPath, manifestDirPath, files, packages, metadata, specs, runtime, externalDocumentRefListFile, componentPath); + + Assert.AreEqual(rootPath, config.BuildDropPath.Value); + Assert.AreEqual(componentPath, config.BuildComponentPath.Value); + Assert.AreEqual(manifestDirPath, config.ManifestDirPath.Value); + Assert.AreEqual(ManifestToolActions.Generate, config.ManifestToolAction); + Assert.AreEqual(packageName, config.PackageName.Value); + Assert.AreEqual(packageVersion, config.PackageVersion.Value); + Assert.AreEqual(DefaultParallelism, config.Parallelism.Value); + Assert.AreEqual(LogEventLevel.Verbose, config.Verbosity.Value); + Assert.AreEqual(0, config.PackagesList.Value.ToList().Count); + Assert.AreEqual(0, config.FilesList.Value.ToList().Count); + Assert.AreEqual(externalDocumentRefListFile, config.ExternalDocumentReferenceListFile.Value); + Assert.AreEqual(1, config.ManifestInfo.Value.Count); + Assert.IsTrue(config.ManifestInfo.Value[0].Equals(expectedManifestInfo)); + + Assert.AreEqual(SettingSource.SBOMApi, config.BuildDropPath.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.BuildComponentPath.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.ManifestDirPath.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.PackageName.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.PackageVersion.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.Parallelism.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.Verbosity.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.PackagesList.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.FilesList.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.ExternalDocumentReferenceListFile.Source); + Assert.AreEqual(SettingSource.SBOMApi, config.ManifestInfo.Source); + } + + [TestMethod] + public void GetConfiguration_NullProperties() + { + var config = builder.GetConfiguration(rootPath, manifestDirPath, null, null, metadata, null, runtime, null, componentPath); + + Assert.IsNull(config.PackagesList); + Assert.IsNull(config.FilesList); + Assert.IsNull(config.ExternalDocumentReferenceListFile); + Assert.IsNull(config.ManifestInfo); + } + + [TestMethod] + [DataRow(null)] + [DataRow(" ")] + public void GetConfiguration_NullComponentPath(string componentPath) + { + var config = builder.GetConfiguration(rootPath, manifestDirPath, null, null, metadata, null, runtime, null, componentPath); + + Assert.IsNull(config.BuildComponentPath); + } + + [TestMethod] + [DataRow(EventLevel.Informational, LogEventLevel.Information)] + [DataRow(EventLevel.Critical, LogEventLevel.Fatal)] + [DataRow(EventLevel.Error, LogEventLevel.Error)] + [DataRow(EventLevel.LogAlways, LogEventLevel.Verbose)] + [DataRow(EventLevel.Verbose, LogEventLevel.Verbose)] + [DataRow(EventLevel.Warning, LogEventLevel.Warning)] + public void GetConfiguration_ShouldMapVerbosity(EventLevel input, LogEventLevel output) + { + // This uses EventLevel to avoid exposing the serilog implementation to the caller + var runtime = new RuntimeConfiguration() + { + Verbosity = input + }; + + IConfiguration config = builder.GetConfiguration( + rootPath, string.Empty, null, null, + metadata, null, runtime); + + Assert.AreEqual(output, config.Verbosity.Value); + } + + [TestMethod] + [DataRow(MinParallelism - 1, DefaultParallelism)] + [DataRow(MaxParallelism + 1, DefaultParallelism)] + [DataRow(10, 10)] + [DataRow(null, DefaultParallelism)] + public void GetConfiguration_SantizeRuntimeConfig_Parallelism(int? input, int output) + { + var runtime = new RuntimeConfiguration() + { + Verbosity = EventLevel.Verbose, + }; + + if (input != null) + { + runtime.WorkflowParallelism = (int)input; + } + + var config = builder.GetConfiguration("random", null, null, null, metadata, null, runtime); + Assert.AreEqual(output, config.Parallelism.Value); + } + + [TestMethod] + public void GetConfiguration_DefaultRuntime() + { + var defaultRuntime = new RuntimeConfiguration + { + WorkflowParallelism = DefaultParallelism, + Verbosity = EventLevel.Warning, + DeleteManifestDirectoryIfPresent = false + }; + + var config = builder.GetConfiguration("random", null, null, null, metadata, null, null); + Assert.AreEqual(defaultRuntime.WorkflowParallelism, config.Parallelism.Value); + Assert.AreEqual(LogEventLevel.Warning, config.Verbosity.Value); + } + + [TestMethod] + [DataRow(" ")] + [DataRow(null)] + [ExpectedException(typeof(ArgumentException))] + public void ThrowArguementExceptionOnRootPathValues(string input) + { + builder.GetConfiguration(input, null, null, null, null); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public void ThrowArguementNulExceptionOnNullMetadata() + { + builder.GetConfiguration("random", null, null, null, null); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public void ThrowArguementExceptionOnSpecificationZero() + { + builder.GetConfiguration("random", null, null, null, metadata, new List(), runtime); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs new file mode 100644 index 00000000..bb3a1633 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigSanitizerTests.cs @@ -0,0 +1,250 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Config; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using PowerArgs; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Security.Cryptography; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Tests.Config +{ + [TestClass] + public class ConfigSanitizerTests + { + private Mock mockFileSystemUtils; + private Mock mockHashAlgorithmProvider; + private Mock mockAssemblyConfig; + private ConfigSanitizer configSanitizer; + + [TestInitialize] + public void Initialize() + { + mockFileSystemUtils = new Mock(); + mockFileSystemUtils + .Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())) + .Returns((string p1, string p2) => Path.Join(p1, p2)); + + mockHashAlgorithmProvider = new Mock(); + mockHashAlgorithmProvider + .Setup(h => h.Get(It.IsAny())) + .Returns((string a) => + { + if (a == "SHA256") + { + return new AlgorithmName(a, stream => SHA256.Create().ComputeHash(stream)); + } + + throw new UnsupportedHashAlgorithmException("Unsupported"); + }); + + mockAssemblyConfig = new Mock(); + + configSanitizer = new ConfigSanitizer(mockHashAlgorithmProvider.Object, mockFileSystemUtils.Object, mockAssemblyConfig.Object); + } + + /// + /// This method returns a configuration object with all the properties set to standard values, + /// which won't make the test fail. Change one value that you are testing in order to ensure you + /// are testing the correct config. + /// + /// + private Configuration GetConfigurationBaseObject() + { + return new Configuration + { + HashAlgorithm = new ConfigurationSetting + { + Source = SettingSource.CommandLine, + Value = new AlgorithmName("SHA256", null) + }, + BuildDropPath = new ConfigurationSetting + { + Source = SettingSource.Default, + Value = "dropPath" + }, + ManifestInfo = new ConfigurationSetting> + { + Source = SettingSource.Default, + Value = new List + { Constants.TestManifestInfo } + } + }; + } + + + [TestMethod] + public void SetValueForManifestInfoForValidation_Succeeds() + { + var config = GetConfigurationBaseObject(); + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + + mockAssemblyConfig.Verify(); + } + + [TestMethod] + [ExpectedException(typeof(ValidationArgException))] + public void NoValueForManifestInfoForValidation_Throws() + { + var config = GetConfigurationBaseObject(); + config.ManifestToolAction = ManifestToolActions.Validate; + config.ManifestInfo.Value.Clear(); + + configSanitizer.SanitizeConfig(config); + } + + [TestMethod] + public void NoValueForManifestInfoForValidation_SetsDefaultValue() + { + var config = GetConfigurationBaseObject(); + config.ManifestToolAction = ManifestToolActions.Validate; + config.ManifestInfo.Value.Clear(); + mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(Constants.TestManifestInfo); + + var sanitizedConfig = configSanitizer.SanitizeConfig(config); + + Assert.IsNotNull(sanitizedConfig.ManifestInfo.Value); + Assert.AreEqual(1, sanitizedConfig.ManifestInfo.Value.Count); + Assert.AreEqual(Constants.TestManifestInfo, sanitizedConfig.ManifestInfo.Value.First()); + + mockAssemblyConfig.VerifyGet(a => a.DefaultManifestInfoForValidationAction); + } + + [TestMethod] + public void ForGenerateActionIgnoresEmptyAlgorithmName_Succeeds() + { + var config = GetConfigurationBaseObject(); + config.HashAlgorithm = null; + config.ManifestToolAction = ManifestToolActions.Generate; + var sanitizedConfig = configSanitizer.SanitizeConfig(config); + + Assert.IsNull(sanitizedConfig.HashAlgorithm); + } + + [TestMethod] + public void ForValidateGetsRealAlgorithmName_Succeeds_DoesNotThrow() + { + var config = GetConfigurationBaseObject(); + config.ManifestToolAction = ManifestToolActions.Validate; + var sanitizedConfig = configSanitizer.SanitizeConfig(config); + + Assert.IsNotNull(sanitizedConfig.HashAlgorithm); + + var result = config.HashAlgorithm.Value.ComputeHash(TestUtils.GenerateStreamFromString("Hekki")); + Assert.IsNotNull(result); + } + + [TestMethod] + [ExpectedException(typeof(UnsupportedHashAlgorithmException))] + public void ForValidateBadAlgorithmNameGetsRealAlgorithmName_Throws() + { + var config = GetConfigurationBaseObject(); + config.HashAlgorithm.Value = new AlgorithmName("a", null); + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + } + + [TestMethod] + public void NullManifestDirShouldUseDropPath_Succeeds() + { + var config = GetConfigurationBaseObject(); + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + + Assert.IsNotNull(config.ManifestDirPath); + Assert.IsNotNull(config.ManifestDirPath.Value); + Assert.AreEqual(Path.Join("dropPath", "_manifest"), config.ManifestDirPath.Value); + } + + [TestMethod] + public void ManifestDirShouldEndWithManifestDirForGenerate_Succeeds() + { + var config = GetConfigurationBaseObject(); + config.ManifestDirPath = new ConfigurationSetting + { + Source = SettingSource.Default, + Value = "manifestDirPath" + }; + + config.ManifestToolAction = ManifestToolActions.Generate; + configSanitizer.SanitizeConfig(config); + + Assert.IsNotNull(config.ManifestDirPath); + Assert.IsNotNull(config.ManifestDirPath.Value); + Assert.AreEqual(Path.Join("manifestDirPath", "_manifest"), config.ManifestDirPath.Value); + } + + [TestMethod] + public void ManifestDirShouldNotAddManifestDirForValidate_Succeeds() + { + var config = GetConfigurationBaseObject(); + config.ManifestDirPath = new ConfigurationSetting + { + Source = SettingSource.Default, + Value = "manifestDirPath" + }; + + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + + Assert.IsNotNull(config.ManifestDirPath); + Assert.IsNotNull(config.ManifestDirPath.Value); + Assert.AreEqual("manifestDirPath", config.ManifestDirPath.Value); + } + + [TestMethod] + public void NullDefaultNamespaceUriBaseShouldReturnExistingValue_Succeeds() + { + mockAssemblyConfig.SetupGet(a => a.DefaultSBOMNamespaceBaseUri).Returns(""); + var config = GetConfigurationBaseObject(); + config.NamespaceUriBase = new ConfigurationSetting + { + Source = SettingSource.Default, + Value = "http://base.uri" + }; + + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + + Assert.AreEqual("http://base.uri", config.NamespaceUriBase.Value); + + mockAssemblyConfig.VerifyGet(a => a.DefaultSBOMNamespaceBaseUri); + mockAssemblyConfig.VerifyNoOtherCalls(); + } + + [TestMethod] + public void UserProviderNamespaceUriBaseShouldReturnDefaultValue_Succeeds() + { + mockAssemblyConfig.SetupGet(a => a.DefaultSBOMNamespaceBaseUri).Returns("http://internal.base.uri"); + mockAssemblyConfig.SetupGet(a => a.DefaultSBOMNamespaceBaseUriWarningMessage).Returns("test"); + var config = GetConfigurationBaseObject(); + config.NamespaceUriBase = new ConfigurationSetting + { + Source = SettingSource.CommandLine, + Value = "http://base.uri" + }; + + config.ManifestToolAction = ManifestToolActions.Validate; + configSanitizer.SanitizeConfig(config); + + Assert.AreEqual("http://internal.base.uri", config.NamespaceUriBase.Value); + Assert.AreEqual(SettingSource.Default, config.NamespaceUriBase.Source); + + mockAssemblyConfig.VerifyGet(a => a.DefaultSBOMNamespaceBaseUri); + mockAssemblyConfig.VerifyGet(a => a.DefaultSBOMNamespaceBaseUriWarningMessage); + mockAssemblyConfig.VerifyNoOtherCalls(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs new file mode 100644 index 00000000..c0b2229a --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsBase.cs @@ -0,0 +1,74 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Api.Config.Validators; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config.Validators; +using Microsoft.Sbom.Contracts.Contracts.Entities; +using Microsoft.Sbom.Contracts.Interfaces; +using Moq; +using System; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Config.Tests +{ + public class ConfigurationBuilderTestsBase + { + protected Mock fileSystemUtilsMock; + protected private IMapper mapper; + protected ConfigValidator[] configValidators; + protected Mock mockAssemblyConfig; + + protected void Init() + { + fileSystemUtilsMock = new Mock(); + mockAssemblyConfig = new Mock(); + mockAssemblyConfig.SetupGet(a => a.DefaultManifestInfoForValidationAction).Returns(Constants.TestManifestInfo); + + configValidators = new ConfigValidator[] + { + new ValueRequiredValidator(mockAssemblyConfig.Object), + new FilePathIsWritableValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object), + new IntRangeValidator(mockAssemblyConfig.Object), + new FileExistsValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object), + new DirectoryExistsValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object), + new DirectoryPathIsWritableValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object), + new UriValidator(mockAssemblyConfig.Object) + }; + + var bindingsMock = new Mock + { + CallBase = true + }; + bindingsMock.Setup(b => b.Load()).Verifiable(); + + var hashAlgorithmProvider = new HashAlgorithmProvider(new IAlgorithmNames[] { new AlgorithmNames() }); + hashAlgorithmProvider.Init(); + + var configSanitizer = new ConfigSanitizer(hashAlgorithmProvider, fileSystemUtilsMock.Object, mockAssemblyConfig.Object); + object ctor(Type type) + { + if (type == typeof(ConfigPostProcessor)) + { + return new ConfigPostProcessor(configValidators, configSanitizer); + } + + return Activator.CreateInstance(type); + } + + var mapperConfiguration = new MapperConfiguration(cfg => + { + cfg.ConstructServicesUsing(ctor); + cfg.AddProfile(); + }); + + mapper = mapperConfiguration.CreateMapper(); + } + + protected const string JSONConfigWithManifestPath = "{ \"ManifestDirPath\": \"manifestDirPath\"}"; + protected const string JSONConfigGoodWithManifestInfo = "{ \"ManifestInfo\": [{ \"Name\":\"manifest\", \"Version\":\"1\"}]}"; + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs new file mode 100644 index 00000000..aa553b91 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForGeneration.cs @@ -0,0 +1,276 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using DropValidator.Api.Utils; +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using PowerArgs; +using System; +using System.IO; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Config.Tests +{ + [TestClass] + public class ConfigurationBuilderTestsForGeneration : ConfigurationBuilderTestsBase + { + [TestInitialize] + public void Setup() + { + Init(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_ForGenerator_CombinesConfigs() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigGoodWithManifestInfo)); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + NamespaceUriBase = "https://base.uri" + }; + + var configuration = await cb.GetConfiguration(args); + + Assert.AreEqual(configuration.BuildDropPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ConfigFilePath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ManifestInfo.Source, SettingSource.JsonConfig); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_ForGenerator_CombinesConfigs_CmdLineSucceeds() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigGoodWithManifestInfo)); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + NamespaceUriBase = "https://base.uri" + }; + + var configuration = await cb.GetConfiguration(args); + + Assert.AreEqual(configuration.BuildDropPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ConfigFilePath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ManifestInfo.Source, SettingSource.JsonConfig); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(ValidationArgException))] + public async Task ConfigurationBuilderTest_Generation_BuildDropPathDoNotExist_Throws() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(false); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + NamespaceUriBase = "https://base.uri" + }; + + var configuration = await cb.GetConfiguration(args); + } + + [TestMethod] + [ExpectedException(typeof(AccessDeniedValidationArgException))] + public async Task ConfigurationBuilderTest_Generation_BuildDropPathNotWriteAccess_Throws() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(false); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + NamespaceUriBase = "https://base.uri" + }; + + var configuration = await cb.GetConfiguration(args); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Generation_DefaultManifestDirPath_AddsManifestDir() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + NamespaceUriBase = "https://base.uri" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual(Path.Join(args.BuildDropPath, Constants.ManifestFolder), config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Generation_UserManifestDirPath_AddsManifestDir() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ManifestDirPath = "ManifestDirPath", + NamespaceUriBase = "https://base.uri" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual(Path.Join("ManifestDirPath", Constants.ManifestFolder), config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Generation_NSBaseUri_Validated() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ManifestDirPath = "ManifestDirPath", + NamespaceUriBase = "https://base.uri" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual(Path.Join("ManifestDirPath", Constants.ManifestFolder), config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Generation_BadNSBaseUriWithDefaultValu_Succeds() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + mockAssemblyConfig.SetupGet(a => a.DefaultSBOMNamespaceBaseUri).Returns("https://uri"); + mockAssemblyConfig.SetupGet(a => a.DefaultSBOMNamespaceBaseUriWarningMessage).Returns("Test"); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ManifestDirPath = "ManifestDirPath", + NamespaceUriBase = "baduri" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual(Path.Join("ManifestDirPath", Constants.ManifestFolder), config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + mockAssemblyConfig.VerifyGet(a => a.DefaultSBOMNamespaceBaseUri); + mockAssemblyConfig.VerifyGet(a => a.DefaultSBOMNamespaceBaseUriWarningMessage); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Generation_BadNSBaseUri_Fails() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var badNsUris = new string[] + { + "baduri", + "https://", + "ww.com", + "https//test.com", + }; + int failedCount = 0; + + foreach (var badNsUri in badNsUris) + { + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + ManifestDirPath = "ManifestDirPath", + NamespaceUriBase = badNsUri + }; + + try + { + var config = await cb.GetConfiguration(args); + Assert.Fail($"NamespaceUriBase test should fail. nsUri: {badNsUri}"); + } + catch (ValidationArgException e) + { + ++failedCount; + Assert.AreEqual("The value of NamespaceUriBase must be a valid URI.", e.Message); + } + } + + Assert.AreEqual(badNsUris.Length, failedCount); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs new file mode 100644 index 00000000..4fc826bb --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationBuilderTestsForValidation.cs @@ -0,0 +1,186 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using AutoMapper; +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using PowerArgs; +using System.IO; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Config.Tests +{ + [TestClass] + public class ConfigurationBuilderTestsForValidation : ConfigurationBuilderTestsBase + { + [TestInitialize] + public void Setup() + { + Init(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_CombinesConfigs() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigWithManifestPath)).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.GetDirectoryName(It.IsAny())).Returns("test").Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var args = new ValidationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + OutputPath = "Test", + HashAlgorithm = AlgorithmName.SHA512 + }; + + var configuration = await cb.GetConfiguration(args); + + Assert.AreEqual(configuration.BuildDropPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ConfigFilePath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.OutputPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.Parallelism.Source, SettingSource.Default); + Assert.AreEqual(configuration.Parallelism.Value, Common.Constants.DefaultParallelism); + Assert.AreEqual(configuration.HashAlgorithm.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.HashAlgorithm.Value, AlgorithmName.SHA512); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_CombinesConfigs_DuplicateConfig_DefaultLoses() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigWithManifestPath)).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.GetDirectoryName(It.IsAny())).Returns("test").Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var args = new ValidationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + OutputPath = "Test", + Parallelism = 4, + Verbosity = Serilog.Events.LogEventLevel.Fatal + }; + + var configuration = await cb.GetConfiguration(args); + + Assert.AreEqual(configuration.BuildDropPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.ConfigFilePath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.OutputPath.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.Parallelism.Source, SettingSource.CommandLine); + Assert.AreEqual(configuration.Verbosity.Value, Serilog.Events.LogEventLevel.Fatal); + Assert.AreEqual(configuration.Verbosity.Source, SettingSource.CommandLine); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(AutoMapperMappingException))] + public async Task ConfigurationBuilderTest_CombinesConfigs_DuplicateConfig_Throws() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigWithManifestPath)); + + var args = new ValidationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + OutputPath = "Test", + ManifestDirPath = "ManifestPath" + }; + + var configuration = await cb.GetConfiguration(args); + } + + [TestMethod] + [ExpectedException(typeof(ValidationArgException))] + public async Task ConfigurationBuilderTest_CombinesConfigs_NegativeParallism_Throws() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(JSONConfigWithManifestPath)); + + var args = new ValidationArgs + { + BuildDropPath = "BuildDropPath", + ConfigFilePath = "config.json", + OutputPath = "Test", + Parallelism = -1 + }; + + var configuration = await cb.GetConfiguration(args); + } + + + [TestMethod] + public async Task ConfigurationBuilderTest_Validation_DefaultManifestDirPath_AddsManifestDir() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + var args = new ValidationArgs + { + OutputPath = "Test", + BuildDropPath = "BuildDropPath" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual(Path.Join("BuildDropPath", Constants.ManifestFolder), config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + } + + [TestMethod] + public async Task ConfigurationBuilderTest_Validation_UserManifestDirPath_DoesntManifestDir() + { + var configFileParser = new ConfigFileParser(fileSystemUtilsMock.Object); + var cb = new ConfigurationBuilder(mapper, configFileParser); + + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var args = new ValidationArgs + { + OutputPath = "Test", + BuildDropPath = "BuildDropPath", + ManifestDirPath = "ManifestDirPath" + }; + + var config = await cb.GetConfiguration(args); + + Assert.IsNotNull(config); + Assert.IsNotNull(config.ManifestDirPath); + Assert.AreEqual("ManifestDirPath", config.ManifestDirPath.Value); + + fileSystemUtilsMock.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationCLITests.cs b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationCLITests.cs new file mode 100644 index 00000000..38e90ce5 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ConfigurationCLITests.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using DropValidator.Api.Config.Extensions; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace Microsoft.Sbom.Api.Tests.Config +{ + [TestClass] + public class ConfigurationCLITests + { + private Mock mockConfiguration; + + [TestInitialize] + public void Setup() + { + mockConfiguration = new Mock(); + } + + [TestMethod] + public void Configuration_CommandLineParams() + { + // A property that is not a ComponentDetectorArgument + mockConfiguration.SetupProperty(c => c.BuildComponentPath, new ConfigurationSetting { Value = "build_component_path" }); + + // A named ComponentDetectorArgument + mockConfiguration.SetupProperty(c => c.DockerImagesToScan, new ConfigurationSetting { Value = "the_docker_image" }); + + // An unnamed ComponentDetectorArgument + mockConfiguration.SetupProperty(c => c.AdditionalComponentDetectorArgs, new ConfigurationSetting { Value = "--arg1 val1 --arg2 val2" }); + + var config = mockConfiguration.Object; + + var argBuilder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("defaultArg1", "val1") + .AddArg("defaultArg2", "val2"); + + var commandLineParams = config.ToComponentDetectorCommandLineParams(argBuilder); + + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --defaultArg1 val1 --defaultArg2 val2 --DockerImagesToScan the_docker_image --arg1 val1 --arg2 val2", string.Join(" ", commandLineParams)); + } + + [TestMethod] + public void Configuration_CommandLineParams_DefaultArgsOnly() + { + var config = mockConfiguration.Object; + + var argBuilder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("defaultArg1", "val1") + .AddArg("defaultArg2", "val2"); + + var commandLineParams = config.ToComponentDetectorCommandLineParams(argBuilder); + + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --defaultArg1 val1 --defaultArg2 val2", string.Join(" ", commandLineParams)); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Config/ManifestToolCmdRunnerTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/ManifestToolCmdRunnerTests.cs new file mode 100644 index 00000000..782df9a7 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/ManifestToolCmdRunnerTests.cs @@ -0,0 +1,75 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Config.Args; +using Microsoft.Sbom.Api.Workflows; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Ninject; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Config; + +namespace Microsoft.Sbom.Api.Tests.Config +{ + [TestClass] + public class ManifestToolCmdRunnerTests + { + [TestMethod] + public async Task ManifestToolCmdRunner_Generate_BuildPathNoWritePermissions_AccessDenied() + { + var bindings = new Bindings(); + + var runner = new ManifestToolCmdRunner(new StandardKernel(bindings)); + + var fileSystemUtilsMock = new Mock(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(false).Verifiable(); + bindings.Rebind().ToConstant(fileSystemUtilsMock.Object).InSingletonScope(); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath" + }; + + await runner.Generate(args); + + Assert.IsTrue(runner.IsFailed); + Assert.IsTrue(runner.IsAccessError); + } + + [TestMethod] + public async Task ManifestToolCmdRunner_Generate_Success() + { + var bindings = new Bindings(); + + var runner = new ManifestToolCmdRunner(new StandardKernel(bindings)); + + var fileSystemUtilsMock = new Mock(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + fileSystemUtilsMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var workflowMock = new Mock(); + workflowMock.Setup(f => f.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); + + bindings.Rebind().ToConstant(fileSystemUtilsMock.Object).InSingletonScope(); + bindings.Rebind().ToConstant(workflowMock.Object).Named(nameof(SBOMGenerationWorkflow)); + + var args = new GenerationArgs + { + BuildDropPath = "BuildDropPath", + NamespaceUriBase = "https://base.uri" + }; + + await runner.Generate(args); + + Assert.IsFalse(runner.IsFailed); + Assert.IsFalse(runner.IsAccessError); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/SBOMConfigTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/SBOMConfigTests.cs new file mode 100644 index 00000000..b251fdf2 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/SBOMConfigTests.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Metadata; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; + +namespace Microsoft.Sbom.Api.Tests.Config +{ + [TestClass] + public class SBOMConfigTests + { + private readonly Mock configHandler; + private readonly Configuration config; + private readonly Mock logger; + private readonly Mock recorder; + private readonly LocalMetadataProvider localMetadataProvider; + + public SBOMConfigTests() + { + configHandler = new Mock(); + config = new Configuration + { + PackageName = new ConfigurationSetting("the-package-name"), + PackageVersion = new ConfigurationSetting("the-package-version"), + NamespaceUriUniquePart = new ConfigurationSetting("some-custom-value-here"), + NamespaceUriBase = new ConfigurationSetting("http://sbom.microsoft") + }; + + logger = new Mock(); + recorder = new Mock(); + localMetadataProvider = new LocalMetadataProvider(config); + } + + [TestMethod] + public void SBOMConfig_DefaultMetadataProvider_Returned() + { + var metadataProviders = new IMetadataProvider[] { localMetadataProvider }; + var sbomConfigs = CreateSbomConfigs(metadataProviders); + + var uri = sbomConfigs.GetSBOMNamespaceUri(); + + Assert.AreEqual(localMetadataProvider.GetDocumentNamespaceUri(), uri); + } + + [TestMethod] + public void SBOMConfig_BuildEnvironmentMetadataProvider_Returned() + { + var sbomMetadata = new SBOMMetadata + { + PackageName = "sbom-package-name", + PackageVersion = "sbom-package-version", + BuildEnvironmentName = "the-build-envsdfgsdg" + }; + + var sbomApiMetadataProvider = new SBOMApiMetadataProvider(sbomMetadata, config); + var metadataProviders = new IMetadataProvider[] { localMetadataProvider, sbomApiMetadataProvider }; + var sbomConfigs = CreateSbomConfigs(metadataProviders); + + var uri = sbomConfigs.GetSBOMNamespaceUri(); + + Assert.AreEqual(sbomApiMetadataProvider.GetDocumentNamespaceUri(), uri); + } + + [TestMethod] + public void SBOMConfig_NoBuildEnvironmentName_DefaultMetadataProvider_Returned() + { + var sbomMetadata = new SBOMMetadata + { + PackageName = "sbom-package-name", + PackageVersion = "sbom-package-version", + BuildEnvironmentName = null + }; + + var sbomApiMetadataProvider = new SBOMApiMetadataProvider(sbomMetadata, config); + var metadataProviders = new IMetadataProvider[] { localMetadataProvider, sbomApiMetadataProvider }; + var sbomConfigs = CreateSbomConfigs(metadataProviders); + + var uri = sbomConfigs.GetSBOMNamespaceUri(); + + Assert.AreEqual(localMetadataProvider.GetDocumentNamespaceUri(), uri); + } + + private ISbomConfigProvider CreateSbomConfigs(IMetadataProvider[] metadataProviders) => + new SbomConfigProvider(manifestConfigHandlers: new IManifestConfigHandler[] { configHandler.Object }, + metadataProviders: metadataProviders, + logger: logger.Object, + recorder: recorder.Object); + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Config/Validators/DirectoryPathIsWritableValidatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Config/Validators/DirectoryPathIsWritableValidatorTests.cs new file mode 100644 index 00000000..43fd18c1 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Config/Validators/DirectoryPathIsWritableValidatorTests.cs @@ -0,0 +1,53 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Config.Validators; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using PowerArgs; + +namespace Microsoft.Sbom.Api.Tests.Config.Validators +{ + [TestClass] + public class DirectoryPathIsWritableValidatorTests + { + private readonly Mock mockAssemblyConfig = new Mock(); + + [TestMethod] + [ExpectedException(typeof(ValidationArgException))] + public void WhenDirectoryDoesNotExistsThrows() + { + var fileSystemUtilsMock = new Mock(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(false).Verifiable(); + + var validator = new DirectoryPathIsWritableValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object); + validator.ValidateInternal("property", "value", null); + } + + [TestMethod] + [ExpectedException(typeof(AccessDeniedValidationArgException))] + public void WhenDirectoryDoesNotHaveWriteAccessThrows() + { + var fileSystemUtilsMock = new Mock(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(false).Verifiable(); + + var validator = new DirectoryPathIsWritableValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object); + validator.ValidateInternal("property", "value", null); + } + + [TestMethod] + public void WhenDirectoryHasWriteAccess() + { + var fileSystemUtilsMock = new Mock(); + fileSystemUtilsMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemUtilsMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + + var validator = new DirectoryPathIsWritableValidator(fileSystemUtilsMock.Object, mockAssemblyConfig.Object); + validator.ValidateInternal("property", "value", null); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Converters/ComponentToExternalReferenceInfoConverterTests.cs b/test/Microsoft.Sbom.Api.Tests/Converters/ComponentToExternalReferenceInfoConverterTests.cs new file mode 100644 index 00000000..1d64da3b --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Converters/ComponentToExternalReferenceInfoConverterTests.cs @@ -0,0 +1,116 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Api.Converters; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; + +namespace Microsoft.Sbom.Api.Tests.Converters +{ + [TestClass] + public class ComponentToExternalReferenceInfoConverterTests + { + private readonly Mock mockLogger = new Mock(); + + [TestMethod] + public async Task When_ConvertingComponentToExternalDocRefInfo_WithCommonCase_ThenTestPass() + { + var scannedComponents = from i in Enumerable.Range(1, 5) + select new ScannedComponent + { + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), $"sbom{i}", "123", $"elementId{i}", $"path{i}") + }; + + var componentsChannel = Channel.CreateUnbounded(); + foreach (var component in scannedComponents) + { + await componentsChannel.Writer.WriteAsync(component); + } + + componentsChannel.Writer.Complete(); + + var converter = new ComponentToExternalReferenceInfoConverter(mockLogger.Object); + var (results, errors) = converter.Convert(componentsChannel); + + var refs = await results.ReadAllAsync().ToListAsync(); + + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.ErrorType}"); + } + + var index = 1; + foreach (var reference in refs) + { + Assert.AreEqual($"sbom{index}", reference.ExternalDocumentName); + Assert.AreEqual(new Uri("http://test.uri").ToString(), reference.DocumentNamespace); + Assert.AreEqual($"elementId{index}", reference.DescribedElementID); + Assert.AreEqual($"path{index}", reference.Path); + Assert.AreEqual("123", reference.Checksum.First().ChecksumValue); + + index++; + } + + Assert.AreEqual(scannedComponents.ToList().Count, index - 1); + } + + [TestMethod] + public async Task When_ConvertingComponentToExternalDocRefInfo_WithWrongComponentType_ThenTestPass() + { + var scannnedComponent1 = new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "sbom1", "123", "abcdef", "path1") + }; + var scannnedComponent2 = new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "sbom2", "123", "abcdef", "path2") + }; + var scannnedComponent3 = new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "sbom3", "123", "abcdef", "path3") + }; + var scannnedComponent4 = new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new NpmComponent("npmpackage", "1.0.0") + }; + + var scannedComponents = new List() + { + scannnedComponent1, + scannnedComponent2, + scannnedComponent3, + scannnedComponent4 + }; + + var componentsChannel = Channel.CreateUnbounded(); + foreach (var component in scannedComponents) + { + await componentsChannel.Writer.WriteAsync(component); + } + + componentsChannel.Writer.Complete(); + + var converter = new ComponentToExternalReferenceInfoConverter(mockLogger.Object); + var (results, errors) = converter.Convert(componentsChannel); + + var refs = await results.ReadAllAsync().ToListAsync(); + var errorList = await errors.ReadAllAsync().ToListAsync(); + + Assert.IsTrue(errorList.Count == scannedComponents.Where(c => !(c.Component is SpdxComponent)).ToList().Count); + Assert.IsTrue(refs.Count == scannedComponents.Where(c => c.Component is SpdxComponent).ToList().Count); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Converters/DropValidatorManifestPathConverterTests.cs b/test/Microsoft.Sbom.Api.Tests/Converters/DropValidatorManifestPathConverterTests.cs new file mode 100644 index 00000000..e9f32b43 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Converters/DropValidatorManifestPathConverterTests.cs @@ -0,0 +1,184 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System.Collections.Generic; +using System.Runtime.InteropServices; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Convertors.Tests +{ + [TestClass] + public class DropValidatorManifestPathConverterTests + { + private Mock osUtils; + private Mock fileSystemUtils; + private Mock fileSystemExtensionUtils; + private Mock configurationMock; + private DropValidatorManifestPathConverter converter; + + [TestInitialize] + public void Setup() + { + osUtils = new Mock(); + fileSystemUtils = new Mock(); + fileSystemExtensionUtils = new Mock(); + configurationMock = new Mock(); + + converter = new DropValidatorManifestPathConverter(configurationMock.Object, osUtils.Object, fileSystemUtils.Object, fileSystemExtensionUtils.Object); + + fileSystemUtils.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + fileSystemExtensionUtils.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(true); + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_ValidPath_Succeeds() + { + var rootPath = @"C:\Sample\Root"; + var operatingSystems = new List() { + OSPlatform.Windows, + OSPlatform.Linux, + OSPlatform.OSX, +#if !NETFRAMEWORK + OSPlatform.FreeBSD +#endif + }; + + foreach (var os in operatingSystems) + { + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(os); + var (path, isOutsideDropPath) = converter.Convert(rootPath + @"\hello\World"); + Assert.AreEqual("/hello/World", path); + } + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_ValidPathWithDot_Succeeds() + { + var rootPath = @"C:\Sample\Root\."; + var operatingSystems = new List() { + OSPlatform.Windows, + OSPlatform.Linux, + OSPlatform.OSX, + OSPlatform.FreeBSD + }; + + foreach (var os in operatingSystems) + { + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(os); + var (path, isOutsideDropPath) = converter.Convert(rootPath + @"\hello\.\World"); + Assert.AreEqual("/hello/World", path); + } + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_BuildDropPathRelative_Succeeds() + { + var rootPath = @"Sample\.\Root\"; + var operatingSystems = new List() { + OSPlatform.Windows, + OSPlatform.Linux, + OSPlatform.OSX, + OSPlatform.FreeBSD + }; + + foreach (var os in operatingSystems) + { + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(os); + var (path, isOutsideDropPath) = converter.Convert(rootPath + @"\hello\.\World"); + Assert.AreEqual("/hello/World", path); + } + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_CaseSensitive_Windows_FreeBSD_Succeeds() + { + var rootPath = @"C:\Sample\Root"; + var operatingSystems = new List() { + OSPlatform.Windows, +#if !NETFRAMEWORK + OSPlatform.FreeBSD +#endif + }; + + foreach (var os in operatingSystems) + { + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(os); + var (path, isOutsideDropPath) = converter.Convert(@"C:\sample\Root" + @"\hello\World"); + Assert.AreEqual("/hello/World", path); + } + } + + [TestMethod] + [ExpectedException(typeof(InvalidPathException))] + public void DropValidatorManifestPathConverterTests_CaseSensitive_OSX_Fails() + { + var rootPath = @"C:\Sample\Root"; + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.OSX); + fileSystemExtensionUtils.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(false); + var (path, isOutsideDropPath) = converter.Convert(@"C:\sample\Root" + @"\hello\World"); + Assert.AreEqual("/hello/World", path); + } + + [TestMethod] + [ExpectedException(typeof(InvalidPathException))] + public void DropValidatorManifestPathConverterTests_CaseSensitive_Linux_Fails() + { + var rootPath = @"C:\Sample\Root"; + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Linux); + fileSystemExtensionUtils.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(false); + var (path, isOutsideDropPath) = converter.Convert(@"C:\sample\Root" + @"\hello\World"); + Assert.AreEqual("/hello/World", path); + } + + [TestMethod] + [ExpectedException(typeof(InvalidPathException))] + public void DropValidatorManifestPathConverterTests_RootPathOutside_Fails() + { + var rootPath = @"C:\Sample\Root"; + + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Windows); + fileSystemExtensionUtils.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(false); + + converter.Convert(@"d:\Root\hello\World"); + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_RootPathOutside_SbomOnDifferentDrive_Succeeds() + { + var rootPath = @"C:\Sample\Root"; + var filePath = @"d:\Root\hello\World.spdx.json"; + var expectedPath = @"/d:/Root/hello/World.spdx.json"; + + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Windows); + var (path, isOutsideDropPath) = converter.Convert(filePath); + Assert.AreEqual(expectedPath, path); + } + + [TestMethod] + public void DropValidatorManifestPathConverterTests_RootPathOutside_SbomOnSameDrive_Succeeds() + { + var rootPath = @"C:\Sample\Root"; + var filePath = @"C:\Sample\hello\World.spdx.json"; + var expectedPath = @"/../hello/World.spdx.json"; + + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = rootPath }); + osUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Windows); + var (path, isOutsideDropPath) = converter.Convert(filePath); + Assert.AreEqual(expectedPath, path); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Converters/ExternalReferenceInfoToPathConverterTest.cs b/test/Microsoft.Sbom.Api.Tests/Converters/ExternalReferenceInfoToPathConverterTest.cs new file mode 100644 index 00000000..f674c869 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Converters/ExternalReferenceInfoToPathConverterTest.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Converters; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Extensions.Entities; + +namespace Microsoft.Sbom.Api.Tests.Converters +{ + [TestClass] + public class ExternalReferenceInfoToPathConverterTest + { + private readonly Mock mockLogger = new Mock(); + + [TestMethod] + public async Task When_ConvertingExternalDocRefInfoToPath_WithCommonCase_ThenTestPass() + { + var externalDocRef1 = new ExternalDocumentReferenceInfo() + { + Path = @"/path1" + }; + var externalDocRef2 = new ExternalDocumentReferenceInfo() + { + Path = @"/path2" + }; + var externalDocRef3 = new ExternalDocumentReferenceInfo() + { + Path = @"/path3" + }; + var externalDocRef4 = new ExternalDocumentReferenceInfo() + { + Path = @"/path4" + }; + + var externalDocRefs = new List() + { + externalDocRef1, externalDocRef2, externalDocRef3, externalDocRef4 + }; + + var externalDocRefChannel = Channel.CreateUnbounded(); + foreach (var externalDocRef in externalDocRefs) + { + await externalDocRefChannel.Writer.WriteAsync(externalDocRef); + } + + externalDocRefChannel.Writer.Complete(); + + var converter = new ExternalReferenceInfoToPathConverter(mockLogger.Object); + var (results, errors) = converter.Convert(externalDocRefChannel); + + var paths = await results.ReadAllAsync().ToListAsync(); + + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.ErrorType}"); + } + + var count = 1; + await foreach (string path in results.ReadAllAsync()) + { + Assert.Equals($"path{count}", path); + count++; + } + + Assert.IsTrue(paths.Count == externalDocRefs.Count); + } + + [TestMethod] + public async Task When_ConvertingExternalDocRefInfoToPath_WithMissingPath_ThenTestPass() + { + var externalDocRef1 = new ExternalDocumentReferenceInfo() + { + Path = @"/path1" + }; + var externalDocRef2 = new ExternalDocumentReferenceInfo() + { + Path = @"/path2" + }; + var externalDocRef3 = new ExternalDocumentReferenceInfo() + { + Path = @"/path3" + }; + var externalDocRef4 = new ExternalDocumentReferenceInfo() { }; + + var externalDocRefs = new List() + { + externalDocRef1, externalDocRef2, externalDocRef3, externalDocRef4 + }; + + var externalDocRefChannel = Channel.CreateUnbounded(); + foreach (var externalDocRef in externalDocRefs) + { + await externalDocRefChannel.Writer.WriteAsync(externalDocRef); + } + + externalDocRefChannel.Writer.Complete(); + + var converter = new ExternalReferenceInfoToPathConverter(mockLogger.Object); + var (results, errors) = converter.Convert(externalDocRefChannel); + + var paths = await results.ReadAllAsync().ToListAsync(); + var errorList = await errors.ReadAllAsync().ToListAsync(); + + await foreach (FileValidationResult error in errors.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.ErrorType}"); + } + + Assert.IsTrue(paths.Count == 3); + Assert.IsTrue(errorList.Count == 1); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Entities/FileValidationResultTest.cs b/test/Microsoft.Sbom.Api.Tests/Entities/FileValidationResultTest.cs new file mode 100644 index 00000000..10bd3ca5 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Entities/FileValidationResultTest.cs @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Contracts.Entities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using System.Text; +using EntityErrorType = Microsoft.Sbom.Contracts.Enums.ErrorType; + +namespace Microsoft.Sbom.Api.Tests.Entities +{ + [TestClass] + public class FileValidationResultTest + { + [TestMethod] + [DataRow(ErrorType.AdditionalFile, EntityErrorType.FileError)] + [DataRow(ErrorType.FilteredRootPath, EntityErrorType.FileError)] + [DataRow(ErrorType.ManifestFolder, EntityErrorType.FileError)] + [DataRow(ErrorType.MissingFile, EntityErrorType.FileError)] + [DataRow(ErrorType.InvalidHash, EntityErrorType.HashingError)] + [DataRow(ErrorType.UnsupportedHashAlgorithm, EntityErrorType.HashingError)] + [DataRow(ErrorType.JsonSerializationError, EntityErrorType.JsonSerializationError)] + [DataRow(ErrorType.None, EntityErrorType.None)] + [DataRow(ErrorType.PackageError, EntityErrorType.PackageError)] + [DataRow(ErrorType.Other, EntityErrorType.Other)] + public void FileValidationResultErrorTypeMapping(ErrorType input, EntityErrorType output) + { + var fileValidationResult = new FileValidationResult() { ErrorType = input, Path = "random" }; + var entityError = fileValidationResult.ToEntityError(); + + Assert.AreEqual(entityError.ErrorType, output); + Assert.AreEqual(entityError.Details, null); + + if (input == ErrorType.PackageError) + { + Assert.AreEqual(((PackageEntity)entityError.Entity).Path, "random"); + Assert.AreEqual(((PackageEntity)entityError.Entity).Name, "random"); + Assert.AreEqual(entityError.Entity.GetType(), typeof(PackageEntity)); + } + else + { + Assert.AreEqual(((FileEntity)entityError.Entity).Path, "random"); + Assert.AreEqual(entityError.Entity.GetType(), typeof(FileEntity)); + } + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs b/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs new file mode 100644 index 00000000..2dc7dd73 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Entities/output/ValidationResultGeneratorTests.cs @@ -0,0 +1,299 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; + +namespace Microsoft.Sbom.Api.Entities.Output.Tests +{ + [TestClass] + public class ValidationResultGeneratorTests + { + [TestMethod] + public void ValidationResultGenerator_ShouldGenerateReportWithoutFailures() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: false); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(12) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Success, validationResultOutput.Result); + Assert.AreEqual(0, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(0, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(2, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + [TestMethod] + public void ValidationResultGenerator_ShouldGenerateReportWithoutFailuresIfIgnoreMissing() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(12) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Success, validationResultOutput.Result); + Assert.AreEqual(0, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(0, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(2, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + [TestMethod] + public void ValidationResultGenerator_IncorrectHashShouldCauseFailure() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: false); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + failures.Add(new FileValidationResult() + { + Path = "/child2/grandchild1/file9", + ErrorType = ErrorType.InvalidHash + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(11) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Failure, validationResultOutput.Result); + Assert.AreEqual(1, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(1, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(11, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(2, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + [TestMethod] + public void ValidationResultGenerator_MissingFileShouldCauseFailure() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: false); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + failures.Add(new FileValidationResult() + { + Path = "/child2/grandchild2/file10", + ErrorType = ErrorType.MissingFile + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(11) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Failure, validationResultOutput.Result); + Assert.AreEqual(1, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(1, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(11, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(2, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + [TestMethod] + public void ValidationResultGenerator_MissingFileShouldNotCauseFailureIfIgnoreMissing() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + failures.Add(new FileValidationResult() + { + Path = "/child2/grandchild2/file10", + ErrorType = ErrorType.MissingFile + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(12) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Success, validationResultOutput.Result); + Assert.AreEqual(0, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(0, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(3, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + [TestMethod] + public void ValidationResultGenerator_ShouldFailOnlyOnWrongHashIfIgnoreMissing() + { + var manifestData = GetDefaultManifestData(); + Mock configurationMock = GetDefaultConfigurationMock(ignoreMissing: true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + var failures = new List(); + + failures.Add(new FileValidationResult() + { + Path = "/_manifest/manifestjson", + ErrorType = ErrorType.ManifestFolder + }); + + failures.Add(new FileValidationResult() + { + Path = "/child5/file8", + ErrorType = ErrorType.FilteredRootPath + }); + + failures.Add(new FileValidationResult() + { + Path = "/child2/grandchild2/file10", + ErrorType = ErrorType.MissingFile + }); + + failures.Add(new FileValidationResult() + { + Path = "/child2/grandchild1/file9", + ErrorType = ErrorType.InvalidHash + }); + + var validationResultOutput = validationResultGenerator + .WithSuccessCount(11) + .WithTotalDuration(TimeSpan.FromSeconds(5)) + .WithValidationResults(failures) + .Build(); + + Assert.AreEqual(Result.Failure, validationResultOutput.Result); + Assert.AreEqual(1, validationResultOutput.ValidationErrors.Count); + Assert.AreEqual(12, validationResultOutput.Summary.ValidationTelemetery.TotalFilesInManifest); + Assert.AreEqual(1, validationResultOutput.Summary.ValidationTelemetery.FilesFailedCount); + Assert.AreEqual(11, validationResultOutput.Summary.ValidationTelemetery.FilesSuccessfulCount); + Assert.AreEqual(3, validationResultOutput.Summary.ValidationTelemetery.FilesSkippedCount); + } + + private static Mock GetDefaultConfigurationMock(bool ignoreMissing) + { + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.ValidateSignature).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = ignoreMissing }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + return configurationMock; + } + + private static ManifestData GetDefaultManifestData() + { + IDictionary hashDictionary = new Dictionary + { + ["/_manifest/manifestjson"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/_manifest/manifestjsonhash" } }, + ["/child1/file1"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child1/file1hash" } }, + ["/child1/file2"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child1/file2hash" } }, + ["/child2/file3"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file3hash" } }, + ["/child2/file4"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file4hash" } }, + ["/child2/file5"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file5hash" } }, + ["/child3/file11"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child3/file11hash" } }, + ["/child3/file12"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child3/file12hash" } }, + ["/child2/grandchild1/file6"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file6hash" } }, + ["/child5/file8"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child5/file8hash" } }, + ["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "incorrectHash" } }, + ["/child2/grandchild2/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "missingfile" } } + }; + return new ManifestData + { + HashesMap = new ConcurrentDictionary(hashDictionary, StringComparer.InvariantCultureIgnoreCase), + Count = hashDictionary.Keys.Count + }; + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs new file mode 100644 index 00000000..9b2625d2 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/ComponentToPackageInfoConverterTests.cs @@ -0,0 +1,215 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.Internal; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Api.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using ILogger = Serilog.ILogger; +using HashAlgorithmName = Microsoft.Sbom.Contracts.Enums.AlgorithmName; +using PackageInfo = Microsoft.Sbom.Contracts.SBOMPackage; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class ComponentToPackageInfoConverterTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockConfiguration = new Mock(); + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + + [TestInitialize] + public void Setup() + { + } + + public ComponentToPackageInfoConverterTests() + { + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "root" }); + + manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + manifestGeneratorProvider.Init(); + } + + [TestMethod] + public async Task ConvertTestAsync() + { + var scannedComponents = new List() + { + new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new NuGetComponent("nugetpackage", "1.0.0") + }, + new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new NuGetComponent("nugetpackage2", "1.0.0") + }, + new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new GitComponent(new Uri("http://test.uri"), "hash") + }, + new ScannedComponent + { + LocationsFoundAt = "test".Split(), + Component = new MavenComponent("groupId", "artifactId", "1.0.0") + } + }; + + var (output, errors) = await ConvertScannedComponents(scannedComponents); + + var expectedPackageNames = new List + { + "nugetpackage", "nugetpackage2", "http://test.uri/ : hash - Git", "groupId.artifactId" + }; + + CollectionAssert.AreEquivalent(expectedPackageNames, output.Select(c => c.PackageName).ToList()); + + Assert.IsFalse(errors?.Any()); + } + + [TestMethod] + public async Task ConvertNuGet_AuthorPopulated() + { + var scannedComponent = new ScannedComponent + { + Component = new NuGetComponent("nugetpackage", "1.0.0") + { + Authors = new[] { "author1", "author2" } + } + }; + + var packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.AreEqual($"Organization: {((NuGetComponent)scannedComponent.Component).Authors.First()}", packageInfo.Supplier); + } + + [TestMethod] + public async Task ConvertNuGet_AuthorNotPopulated() + { + var scannedComponent = new ScannedComponent + { + Component = new NuGetComponent("nugetpackage", "1.0.0") { Authors = null } + }; + + var packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.IsNull(packageInfo.Supplier); + } + + [TestMethod] + public async Task ConvertNpm_AuthorPopulated_Name() + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("nugetpackage", "1.0.0", author: new NpmAuthor("Suzy Author")) + }; + + PackageInfo packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.AreEqual($"Organization: {((NpmComponent)scannedComponent.Component).Author.Name}", packageInfo.Supplier); + } + + [TestMethod] + public async Task ConvertNpm_AuthorPopulated_NameAndEmail() + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("nugetpackage", "1.0.0", author: new NpmAuthor("Suzy Author", "suzya@contoso.com")) + }; + + PackageInfo packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.AreEqual($"Organization: {((NpmComponent)scannedComponent.Component).Author.Name} ({((NpmComponent)scannedComponent.Component).Author.Email})", packageInfo.Supplier); + } + + [TestMethod] + public async Task ConvertNpm_AuthorNotPopulated() + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("npmpackage", "1.0.0") { Author = null } + }; + + var packageInfo = await ConvertScannedComponent(scannedComponent); + + Assert.IsNull(packageInfo.Supplier); + } + + [TestMethod] + public async Task ConvertWorksWithBuildComponentPathNull() + { + var scannedComponents = new List() + { + new ScannedComponent + { + Component = new NuGetComponent("nugetpackage", "1.0.0") + }, + new ScannedComponent + { + Component = new NuGetComponent("nugetpackage2", "1.0.0") + }, + new ScannedComponent + { + Component = new GitComponent(new Uri("http://test.uri"), "hash") + }, + new ScannedComponent + { + Component = new MavenComponent("groupId", "artifactId", "1.0.0") + } + }; + + var (output, errors) = await ConvertScannedComponents(scannedComponents); + + var expectedPackageNames = new List + { + "nugetpackage", "nugetpackage2", "http://test.uri/ : hash - Git", "groupId.artifactId" + }; + + CollectionAssert.AreEquivalent(expectedPackageNames, output.Select(c => c.PackageName).ToList()); + + Assert.IsFalse(errors?.Any()); + } + + private async Task ConvertScannedComponent(ScannedComponent scannedComponent) + { + var componentsChannel = Channel.CreateUnbounded(); + await componentsChannel.Writer.WriteAsync(scannedComponent); + componentsChannel.Writer.Complete(); + var packageInfoConverter = new ComponentToPackageInfoConverter(mockLogger.Object); + var (output, _) = packageInfoConverter.Convert(componentsChannel); + var packageInfo = await output.ReadAsync(); + return packageInfo; + } + + private async Task<(IEnumerable, IEnumerable)> ConvertScannedComponents(IEnumerable scannedComponents) + { + var componentsChannel = Channel.CreateUnbounded(); + foreach (var scannedComponent in scannedComponents) + { + await componentsChannel.Writer.WriteAsync(scannedComponent); + } + + componentsChannel.Writer.Complete(); + var packageInfoConverter = new ComponentToPackageInfoConverter(mockLogger.Object); + var (output, errors) = packageInfoConverter.Convert(componentsChannel); + return (await output.ReadAllAsync().ToListAsync(), await errors.ReadAllAsync().ToListAsync()); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/DirectoryWalkerTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/DirectoryWalkerTests.cs new file mode 100644 index 00000000..fe5b522e --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/DirectoryWalkerTests.cs @@ -0,0 +1,111 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class DirectoryWalkerTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockConfiguration = new Mock(); + + [TestInitialize] + public void TestInitialize() + { + mockConfiguration.Setup(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Source = SettingSource.Default, Value = true }); + } + + [TestMethod] + public async Task DirectoryWalkerTests_ValidRoot_SucceedsAsync() + { + HashSet files = new HashSet + { + @"Test\Sample\NoRead.txt", + @"Test\Sample\Sample.txt", + }; + + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + mockFSUtils.SetupSequence(m => m.GetDirectories(It.IsAny(), true)) + .Returns(new List() { "Sample" }) + .Returns(new List()); + mockFSUtils.Setup(m => m.GetFilesInDirectory(It.Is(d => d == "Sample"), true)).Returns(files).Verifiable(); + mockFSUtils.Setup(m => m.GetFilesInDirectory(It.Is(d => d == "Test"), true)).Returns(new List()).Verifiable(); + + var filesChannelReader = new DirectoryWalker(mockFSUtils.Object, mockLogger.Object, mockConfiguration.Object).GetFilesRecursively(@"Test"); + + await foreach (string file in filesChannelReader.file.ReadAllAsync()) + { + Assert.IsTrue(files.Remove(file)); + } + + await foreach (string file in filesChannelReader.file.ReadAllAsync()) + { + Assert.IsTrue(files.Remove(file)); + } + + await foreach (Entities.FileValidationResult error in filesChannelReader.errors.ReadAllAsync()) + { + Assert.Fail($"Error thrown for {error.Path}: {error.ErrorType}"); + } + + Assert.IsTrue(files.Count == 0); + mockFSUtils.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(InvalidPathException))] + public void DirectoryWalkerTests_DirectoryDoesntExist_Fails() + { + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.DirectoryExists(It.IsAny())).Returns(false).Verifiable(); + new DirectoryWalker(mockFSUtils.Object, mockLogger.Object, mockConfiguration.Object).GetFilesRecursively(@"BadDir"); + mockFSUtils.VerifyAll(); + } + + [TestMethod] + public async Task DirectoryWalkerTests_UnreachableFile_FailsAsync() + { + HashSet files = new HashSet + { + @"Test\SampleBadDir\Test.txt" + }; + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + mockFSUtils.SetupSequence(m => m.GetDirectories(It.IsAny(), true)) + .Returns(new List() { "Sample" }) + .Returns(new List() { "Failed" }); + mockFSUtils.Setup(m => m.GetFilesInDirectory(It.Is(d => d == "Sample"), true)).Returns(files).Verifiable(); + mockFSUtils.Setup(m => m.GetFilesInDirectory(It.Is(d => d == "Test"), true)).Returns(new List()).Verifiable(); + mockFSUtils.Setup(m => m.GetFilesInDirectory(It.Is(d => d == "Failed"), true)).Throws(new UnauthorizedAccessException()).Verifiable(); + + var filesChannelReader = new DirectoryWalker(mockFSUtils.Object, mockLogger.Object, mockConfiguration.Object).GetFilesRecursively(@"Test"); + int errorCount = 0; + + await foreach (Entities.FileValidationResult error in filesChannelReader.errors.ReadAllAsync()) + { + errorCount++; + } + + await foreach (string file in filesChannelReader.file.ReadAllAsync()) + { + Assert.IsTrue(files.Remove(file)); + } + + Assert.IsTrue(errorCount == 1); + Assert.IsTrue(files.Count == 0); + mockFSUtils.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs b/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs new file mode 100644 index 00000000..699c0844 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/ExternalDocumentReferenceWriterTest.cs @@ -0,0 +1,100 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Recorder; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading.Channels; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Tests.Executors +{ + [TestClass] + public class ExternalDocumentReferenceWriterTest + { + private Mock mockLogger = new Mock(); + private Mock recorderMock = new Mock(); + private Mock fileSystemUtilsMock = new Mock(); + + + [TestMethod] + public async Task PassExternalDocumentReferenceInfosChannel_ReturnsJsonDocWithSerializer() + { + var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + manifestGeneratorProvider.Init(); + var metadataBuilder = new MetadataBuilder( + mockLogger.Object, + manifestGeneratorProvider, + Constants.TestManifestInfo, + recorderMock.Object); + var jsonFilePath = "/root/_manifest/manifest.json"; + var sbomConfig = new SbomConfig(fileSystemUtilsMock.Object) + { + ManifestInfo = Constants.TestManifestInfo, + ManifestJsonDirPath = "/root/_manifest", + ManifestJsonFilePath = jsonFilePath, + MetadataBuilder = metadataBuilder, + Recorder = new SbomPackageDetailsRecorder() + }; + + ExternalDocumentReferenceInfo externalDocumentReferenceInfo = new ExternalDocumentReferenceInfo(); + externalDocumentReferenceInfo.ExternalDocumentName = "name"; + externalDocumentReferenceInfo.DocumentNamespace = "namespace"; + var checksum = new Checksum(); + checksum.Algorithm = AlgorithmName.SHA1; + checksum.ChecksumValue = "abc"; + externalDocumentReferenceInfo.Checksum = new List { checksum }; + + var externalDocumentReferenceInfos = new List { externalDocumentReferenceInfo }; + var externalDocumentReferenceInfosChannel = Channel.CreateUnbounded(); + foreach (var data in externalDocumentReferenceInfos) + { + await externalDocumentReferenceInfosChannel.Writer.WriteAsync(data); + } + + externalDocumentReferenceInfosChannel.Writer.Complete(); + + var externalDocumentReferenceWriter = new ExternalDocumentReferenceWriter(manifestGeneratorProvider, mockLogger.Object); + var (results, errors) = externalDocumentReferenceWriter.Write(externalDocumentReferenceInfosChannel, new List { sbomConfig }); + + await foreach (var result in results.ReadAllAsync()) + { + JsonElement root = result.Document.RootElement; + + Assert.IsNotNull(root); + + if (root.TryGetProperty("SpdxDocument", out JsonElement documentNamespace)) + { + Assert.AreEqual("namespace", documentNamespace.GetString()); + } + else + { + Assert.Fail("SpdxDocument property not found"); + } + + if (root.TryGetProperty("ExternalDocumentId", out JsonElement externalDocumentId)) + { + Assert.AreEqual("name", externalDocumentId.GetString()); + } + else + { + Assert.Fail("ExternalDocumentId property not found"); + } + } + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs new file mode 100644 index 00000000..262246e9 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/FileHasherTests.cs @@ -0,0 +1,331 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.Collections.Generic; +using Moq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Serilog; +using System.Linq; +using System.Collections.Concurrent; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Convertors; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class FileHasherTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockConfiguration = new Mock(); + + private readonly ConcurrentDictionary hashDict = new ConcurrentDictionary(); + private HashSet fileList = new HashSet(); + + [TestInitialize] + public void TestInitialize() + { + fileList = new HashSet() + { + "test1", + "test2", + "test3" + }; + foreach (var file in fileList) + { + hashDict[file] = new Checksum[] + { + new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = $"{file}_hash" } + }; + } + + ManifestDataSingleton.ResetDictionary(); + } + + [TestMethod] + public async Task FileHasherTest_Validate_MultipleFiles_SucceedsAsync() + { + var hashCodeGeneratorMock = new Mock(); + var manifestPathConverter = new Mock(); + + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + hashCodeGeneratorMock.Setup(m => m.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns(new Checksum[] + { + new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + }); + manifestPathConverter.Setup(m => m.Convert(It.IsAny())).Returns((string r) => (r, true)); + + var files = Channel.CreateUnbounded(); + (ChannelReader file, ChannelReader error) fileHashes + = new FileHasher(hashCodeGeneratorMock.Object, + manifestPathConverter.Object, + mockLogger.Object, + mockConfiguration.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + .Run(files); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(file); + } + + files.Writer.Complete(); + + await foreach (InternalSBOMFileInfo fileHash in fileHashes.file.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(fileHash.Path)); + Assert.AreEqual("hash", fileHash.Checksum.First().ChecksumValue); + Assert.IsNull(fileHash.FileTypes); + } + + Assert.IsTrue(fileHashes.error.Count == 0); + hashCodeGeneratorMock.VerifyAll(); + manifestPathConverter.VerifyAll(); + mockConfiguration.VerifyAll(); + } + + [TestMethod] + public async Task FileHasherTest_Validate_ManifestPathConverterThrows_ReturnsValidationFailureAsync() + { + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + var hashCodeGeneratorMock = new Mock(); + var manifestPathConverter = new Mock(); + + hashCodeGeneratorMock.Setup(m => m.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns(new Checksum[] + { + new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + }); + + manifestPathConverter.Setup(m => m.Convert(It.IsAny())).Returns((string r) => (r, true)); + manifestPathConverter.Setup(m => m.Convert(It.Is(d => d == "test2"))).Throws(new InvalidPathException()); + + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + manifestPathConverter.Object, + mockLogger.Object, + mockConfiguration.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = ManifestDataSingleton.Instance + }; + + var files = Channel.CreateUnbounded(); + (ChannelReader file, ChannelReader error) fileHashes + = fileHasher.Run(files); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(file); + } + + files.Writer.Complete(); + var errorCount = 0; + var filesCount = 0; + + await foreach (var fileHash in fileHashes.file.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(fileHash.Path)); + Assert.AreEqual("hash", fileHash.Checksum.First().ChecksumValue); + Assert.IsNull(fileHash.FileTypes); + filesCount++; + } + + await foreach (FileValidationResult error in fileHashes.error.ReadAllAsync()) + { + Assert.AreEqual(Entities.ErrorType.Other, error.ErrorType); + errorCount++; + } + + Assert.AreEqual(3, ManifestDataSingleton.Instance.HashesMap.Count); + Assert.AreEqual(2, filesCount); + Assert.AreEqual(1, errorCount); + hashCodeGeneratorMock.VerifyAll(); + hashCodeGeneratorMock.Verify(h => h.GenerateHashes(It.IsAny(), It.IsAny()), Times.Exactly(2)); + manifestPathConverter.VerifyAll(); + mockConfiguration.VerifyAll(); + } + + + [TestMethod] + public async Task FileHasherTest_Validate_HashError_ReturnsValidationFailureAsync() + { + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + var hashCodeGeneratorMock = new Mock(); + var manifestPathConverter = new Mock(); + + hashCodeGeneratorMock.SetupSequence(m => m.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns(new Checksum[] + { + new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + }) + .Returns(new Checksum[] + { + new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "" } + }) + .Throws(new UnauthorizedAccessException("Can't access file")); + manifestPathConverter.Setup(m => m.Convert(It.IsAny())).Returns((string r) => (r, true)); + + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + manifestPathConverter.Object, + mockLogger.Object, + mockConfiguration.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = ManifestDataSingleton.Instance + }; + + var files = Channel.CreateUnbounded(); + (ChannelReader file, ChannelReader error) fileHashes + = fileHasher.Run(files); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(file); + } + + files.Writer.Complete(); + var errorCount = 0; + var filesCount = 0; + + await foreach (InternalSBOMFileInfo fileHash in fileHashes.file.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(fileHash.Path)); + Assert.AreEqual("hash", fileHash.Checksum.First().ChecksumValue); + Assert.IsNull(fileHash.FileTypes); + filesCount++; + } + + await foreach (FileValidationResult error in fileHashes.error.ReadAllAsync()) + { + Assert.AreEqual(Entities.ErrorType.Other, error.ErrorType); + errorCount++; + } + + Assert.AreEqual(1, ManifestDataSingleton.Instance.HashesMap.Count); + Assert.AreEqual(1, filesCount); + Assert.AreEqual(2, errorCount); + hashCodeGeneratorMock.VerifyAll(); + manifestPathConverter.VerifyAll(); + mockConfiguration.VerifyAll(); + } + + [TestMethod] + public async Task FileHasherTest_Generate_MultipleFiles_SucceedsAsync() + { + var hashCodeGeneratorMock = new Mock(); + var manifestPathConverter = new Mock(); + + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Generate); + + hashCodeGeneratorMock.Setup(m => m.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns(new Checksum[] + { + new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = "hash" } + }); + + var manifestInfoList = new List + { + ManifestInfo.Parse("test:1"), + ManifestInfo.Parse("test:2") + }; + + var generator1 = new Mock(); + var generator2 = new Mock(); + + generator1.Setup(g => g.RegisterManifest()).Returns(ManifestInfo.Parse("test:1")); + generator2.Setup(g => g.RequiredHashAlgorithms).Returns(new AlgorithmName[] { AlgorithmName.SHA256 }); + generator2.Setup(g => g.RegisterManifest()).Returns(ManifestInfo.Parse("test:2")); + + var manifestGenProvider = new ManifestGeneratorProvider(new IManifestGenerator[] + { + generator1.Object, + generator2.Object + }); + + manifestGenProvider.Init(); + + var sbomConfigs = new Mock(); + sbomConfigs.Setup(s => s.GetManifestInfos()).Returns(manifestInfoList); + + manifestPathConverter.Setup(m => m.Convert(It.IsAny())).Returns((string r) => (r, true)); + + var files = Channel.CreateUnbounded(); + (ChannelReader file, ChannelReader error) fileHashes + = new FileHasher(hashCodeGeneratorMock.Object, + manifestPathConverter.Object, + mockLogger.Object, + mockConfiguration.Object, + sbomConfigs.Object, + manifestGenProvider, + new FileTypeUtils()) + .Run(files); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(file); + } + + files.Writer.Complete(); + + await foreach (InternalSBOMFileInfo fileHash in fileHashes.file.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(fileHash.Path)); + Assert.AreEqual("hash", fileHash.Checksum.First().ChecksumValue); + Assert.IsNull(fileHash.FileTypes); + } + + Assert.IsTrue(fileHashes.error.Count == 0); + hashCodeGeneratorMock.VerifyAll(); + manifestPathConverter.VerifyAll(); + mockConfiguration.VerifyAll(); + } + + private sealed class ManifestDataSingleton + { + static readonly IDictionary hashDictionary = new Dictionary + { + ["test1"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "test1_hash" } }, + ["test2"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "test2_hash" } }, + ["test3"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "test3_hash" } }, + }; + private static readonly Lazy + lazy = + new Lazy + (() => new ManifestData { HashesMap = new ConcurrentDictionary(hashDictionary, StringComparer.InvariantCultureIgnoreCase) }); + + public static ManifestData Instance { get { return lazy.Value; } } + + private ManifestDataSingleton() { } + + public static void ResetDictionary() + { + lazy.Value.HashesMap = new ConcurrentDictionary(hashDictionary, StringComparer.InvariantCultureIgnoreCase); + } + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/FileListEnumeratorTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/FileListEnumeratorTests.cs new file mode 100644 index 00000000..8bdf2c74 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/FileListEnumeratorTests.cs @@ -0,0 +1,120 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System; +using System.Collections.Generic; +using System.Threading.Tasks; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class FileListEnumeratorTests + { + private readonly Mock mockLogger = new Mock(); + + [TestMethod] + public async Task ListWalkerTests_ValidListFile_SucceedsAsync() + { + List files = new List + { + @"d:\directorya\directoryb\file1.txt", + @"d:\directorya\directoryc\file3.txt", + }; + + string fileText = string.Join(Environment.NewLine, files); + string testFileName = "somefile"; + + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.ReadAllText(It.Is(d => d == testFileName))).Returns(fileText).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == testFileName))).Returns(true).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == files[0]))).Returns(true).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == files[1]))).Returns(true).Verifiable(); + mockFSUtils.Setup(m => m.AbsolutePath(It.Is(d => d == files[0]))).Returns(files[0]); + mockFSUtils.Setup(m => m.AbsolutePath(It.Is(d => d == files[1]))).Returns(files[1]); + + var filesChannelReader = new FileListEnumerator(mockFSUtils.Object, mockLogger.Object).GetFilesFromList(testFileName); + int errorCount = 0; + + await foreach (Entities.FileValidationResult error in filesChannelReader.errors.ReadAllAsync()) + { + Assert.AreEqual(Entities.ErrorType.MissingFile, error.ErrorType); + errorCount++; + } + + await foreach (string file in filesChannelReader.file.ReadAllAsync()) + { + Assert.IsTrue(files.Remove(file)); + } + + Assert.IsTrue(errorCount == 0); + Assert.IsTrue(files.Count == 0); + mockFSUtils.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentException))] + public void ListWalkerTests_ListFile_Null_Fails() + { + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.DirectoryExists(It.IsAny())).Returns(false).Verifiable(); + new FileListEnumerator(mockFSUtils.Object, mockLogger.Object).GetFilesFromList(null); + mockFSUtils.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(InvalidPathException))] + public void ListWalkerTests_DirectoryDoesntExist_Fails() + { + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.DirectoryExists(It.IsAny())).Returns(false).Verifiable(); + new FileListEnumerator(mockFSUtils.Object, mockLogger.Object).GetFilesFromList(@"BadDir"); + mockFSUtils.VerifyAll(); + } + + [TestMethod] + public async Task ListWalkerTests_UnreachableFile_FailsAsync() + { + List files = new List + { + @"d:\directorya\directoryb\file1.txt", + @"d:\directorya\directoryc\file3.txt", + }; + + string fileText = string.Join(Environment.NewLine, files); + string testFileName = "somefile"; + + var mockFSUtils = new Mock(); + mockFSUtils.Setup(m => m.ReadAllText(It.Is(d => d == testFileName))).Returns(fileText).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == testFileName))).Returns(true).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == files[0]))).Returns(true).Verifiable(); + mockFSUtils.Setup(m => m.FileExists(It.Is(d => d == files[1]))).Returns(false).Verifiable(); + mockFSUtils.Setup(m => m.AbsolutePath(It.Is(d => d == files[0]))).Returns(files[0]); + mockFSUtils.Setup(m => m.AbsolutePath(It.Is(d => d == files[1]))).Returns(files[1]); + + var filesChannelReader = new FileListEnumerator(mockFSUtils.Object, mockLogger.Object).GetFilesFromList(testFileName); + int errorCount = 0; + + await foreach (Entities.FileValidationResult error in filesChannelReader.errors.ReadAllAsync()) + { + Assert.AreEqual(Entities.ErrorType.MissingFile, error.ErrorType); + errorCount++; + } + + await foreach (string file in filesChannelReader.file.ReadAllAsync()) + { + Assert.IsTrue(files.Remove(file)); + } + + Assert.IsTrue(errorCount == 1); + Assert.IsTrue(files.Count == 1); + mockFSUtils.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs new file mode 100644 index 00000000..a9524bb4 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/HashValidatorTests.cs @@ -0,0 +1,156 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.Threading.Channels; +using System.Threading.Tasks; +using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class HashValidatorTests + { + [TestMethod] + public async Task HashValidatorTest_ValidHash_SucceedsAsync() + { + var fileList = new HashSet() + { + "TEST1", + "TEST2", + "TEST3" + }; + ConcurrentDictionary hashDict = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); + foreach (var file in fileList) + { + hashDict[file.ToLower()] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = $"{file}_hash" } }; + } + + var configuration = new Mock(); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + var files = Channel.CreateUnbounded(); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(new InternalSBOMFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + } + + files.Writer.Complete(); + + var validator = new HashValidator(configuration.Object, new ManifestData { HashesMap = hashDict }); + var validationResults = validator.Validate(files); + + await foreach (FileValidationResult output in validationResults.output.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(output.Path)); + } + + Assert.AreEqual(0, fileList.Count); + Assert.IsTrue(validationResults.errors.Count == 0); + } + + [TestMethod] + public async Task HashValidatorTest_InValidHash_ReturnsValidationErrorAsync() + { + var fileList = new HashSet() + { + "TEST1", + "TEST2", + "TEST3" + }; + + ConcurrentDictionary hashDict = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); + foreach (var file in fileList) + { + hashDict[file.ToLower()] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = $"{file}_hashInvalid" } }; + } + + var configuration = new Mock(); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + var files = Channel.CreateUnbounded(); + foreach (var file in fileList) + { + await files.Writer.WriteAsync(new InternalSBOMFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + } + + files.Writer.Complete(); + + var validator = new HashValidator(configuration.Object, new ManifestData { HashesMap = hashDict }); + var validationResults = validator.Validate(files); + + await foreach (FileValidationResult output in validationResults.output.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(output.Path)); + } + + await foreach (FileValidationResult error in validationResults.errors.ReadAllAsync()) + { + Assert.AreEqual(ErrorType.InvalidHash, error.ErrorType); + Assert.IsTrue(fileList.Remove(error.Path)); + } + + Assert.AreEqual(0, fileList.Count); + } + + [TestMethod] + public async Task HashValidatorTest_AdditionalFile_ReturnsAdditionalFileFailureAsync() + { + var fileList = new HashSet() + { + "TEST1", + "TEST2", + "TEST3" + }; + + ConcurrentDictionary hashDict = new ConcurrentDictionary(StringComparer.InvariantCultureIgnoreCase); + foreach (var file in fileList) + { + hashDict[file.ToLower()] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = $"{file}_hash" } }; + } + + var configuration = new Mock(); + configuration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + + var files = Channel.CreateUnbounded(); + var errors = Channel.CreateUnbounded(); + + foreach (var file in fileList) + { + await files.Writer.WriteAsync(new InternalSBOMFileInfo { Path = file.ToUpper(), Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"{file}_hash" } } }); + } + + // Additional file. + await files.Writer.WriteAsync(new InternalSBOMFileInfo { Path = "TEST4", Checksum = new Checksum[] { new Checksum { Algorithm = Constants.DefaultHashAlgorithmName, ChecksumValue = $"TEST4_hash" } } }); + + files.Writer.Complete(); + errors.Writer.Complete(); + + var validator = new HashValidator(configuration.Object, new ManifestData { HashesMap = hashDict }); + var validationResults = validator.Validate(files); + + await foreach (FileValidationResult error in validationResults.errors.ReadAllAsync()) + { + Assert.AreEqual(ErrorType.AdditionalFile, error.ErrorType); + Assert.AreEqual("TEST4", error.Path); + } + + await foreach (FileValidationResult output in validationResults.output.ReadAllAsync()) + { + Assert.IsTrue(fileList.Remove(output.Path)); + } + + Assert.AreEqual(0, fileList.Count); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs new file mode 100644 index 00000000..f2c4db13 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/PackagesWalkerTests.cs @@ -0,0 +1,234 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog.Events; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ILogger = Serilog.ILogger; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class PackagesWalkerTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockConfiguration = new Mock(); + private readonly Mock mockSbomConfigs = new Mock(); + private readonly Mock mockFileSystemUtils = new Mock(); + + public PackagesWalkerTests() + { + ISbomConfig sbomConfig = new SbomConfig(mockFileSystemUtils.Object) + { + ManifestJsonFilePath = "testpath" + }; + mockConfiguration.SetupGet(c => c.Verbosity).Returns(new ConfigurationSetting { Value = LogEventLevel.Information }); + mockSbomConfigs.Setup(s => s.TryGet(It.IsAny(), out sbomConfig)).Returns(true); + } + + [TestMethod] + public async Task ScanSuccessTestAsync() + { + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("componentName", $"{i}") + }; + + scannedComponents.Add(scannedComponent); + } + + var scannedComponentOther = new ScannedComponent + { + Component = new NpmComponent("componentName", "3") + }; + + scannedComponents.Add(scannedComponentOther); + + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + + int countDistinctComponents = 0; + + await foreach (ScannedComponent package in packagesChannelReader.output.ReadAllAsync()) + { + countDistinctComponents++; + Assert.IsTrue(scannedComponents.Remove(package)); + } + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.Message}"); + } + + Assert.IsTrue(scannedComponents.Count == 1); + Assert.IsTrue(countDistinctComponents == 3); + mockDetector.VerifyAll(); + } + + [TestMethod] + public async Task ScanCombinePackagesWithSameNameDifferentCase() + { + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("componentName", $"{i}") + }; + + scannedComponents.Add(scannedComponent); + } + + var scannedComponentOther = new ScannedComponent + { + // Component with changed case. should also match 'componentName' and + // thus only 3 components should be detected. + Component = new NpmComponent("ComponentName", "3") + }; + + scannedComponents.Add(scannedComponentOther); + + + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + + int countDistinctComponents = 0; + + await foreach (ScannedComponent package in packagesChannelReader.output.ReadAllAsync()) + { + countDistinctComponents++; + Assert.IsTrue(scannedComponents.Remove(package)); + } + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.Message}"); + } + + Assert.IsTrue(scannedComponents.Count == 1); + Assert.IsTrue(countDistinctComponents == 3); + mockDetector.VerifyAll(); + } + + [TestMethod] + public void ScanWithNullOrEmptyPathSuccessTest() + { + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + walker.GetComponents(null); + walker.GetComponents(""); + + mockDetector.Verify(mock => mock.Scan(It.IsAny()), Times.Never()); + } + + [TestMethod] + public async Task ScanFailureTestAsync() + { + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Error, + ComponentsFound = null + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + ComponentDetectorException actualError = null; + + await foreach (ScannedComponent package in packagesChannelReader.output.ReadAllAsync()) + { + Assert.Fail("Packages were still returned when the detector failed."); + } + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + actualError = error; + } + + Assert.IsNotNull(actualError); + mockDetector.VerifyAll(); + } + + [TestMethod] + public async Task ScanIgnoreSbomComponents() + { + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("componentName", $"{i}") + }; + + scannedComponents.Add(scannedComponent); + } + + var scannedComponentOther = new ScannedComponent + { + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.com"), "componentName", "123", "abcdf", "path1") + }; + + scannedComponents.Add(scannedComponentOther); + + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new PackagesWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + + var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.Message}"); + } + + Assert.IsTrue(scannedComponents.Where(c => !(c.Component is SpdxComponent)).ToList().Count == discoveredComponents.Count); + mockDetector.VerifyAll(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/RelationshipGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Executors/RelationshipGeneratorTest.cs new file mode 100644 index 00000000..a3d206d3 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/RelationshipGeneratorTest.cs @@ -0,0 +1,123 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.Collections.Generic; +using System.Text.Json; +using System.Threading; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class RelationshipGeneratorTest + { + /// + /// This repros a channel being orhpaned by not closing it in the face of exceptions + /// + [TestMethod] + public async Task RunShouldHandleExceptionWithoutOrphaningChannel() + { + Mock mock = new Mock(); + + var m = new ManifestGeneratorProvider(new IManifestGenerator[] { mock.Object }); + + var rg = new RelationshipGenerator(m); + var r = new Relationship() { RelationshipType = RelationshipType.DEPENDS_ON }; + var rs = new List { r }; + + var mi = new ManifestInfo(); + mi.Name = "Test"; + mi.Version = "1"; + mock.Setup(m => m.RegisterManifest()).Returns(mi); + m.Init(); + + mock.Setup(m => m.GenerateJsonDocument(It.IsAny())).Throws(new InvalidOperationException()); + + ChannelReader channel = rg.Run(rs.GetEnumerator(), mi); + + // This timeout will cause an OperationCanceledException to be thrown if the channel is orphaned + using var cts = new CancellationTokenSource(TimeSpan.FromSeconds(3)); + + // This will immidately return if the channel is closed. + // If the channel is orphaned this will block until the timeout is reached + // which will fail the test. + await channel.WaitToReadAsync(cts.Token); + } + + [TestMethod] + public async Task RunShouldReturnTwoResults() + { + Mock mock = new Mock(); + + var m = new ManifestGeneratorProvider(new IManifestGenerator[] { mock.Object }); + + var rg = new RelationshipGenerator(m); + var r = new Relationship() { RelationshipType = RelationshipType.DEPENDS_ON, SourceElementId = "one", TargetElementId = "two" }; + var r2 = new Relationship() { RelationshipType = RelationshipType.CONTAINS, SourceElementId = "three", TargetElementId = "four" }; + var rs = new List { r, r2 }; + + var mi = new ManifestInfo(); + mi.Name = "Test"; + mi.Version = "1"; + mock.Setup(m => m.RegisterManifest()).Returns(mi); + m.Init(); + + var j1 = JsonDocument.Parse(JsonSerializer.Serialize(r)); + var j2 = JsonDocument.Parse(JsonSerializer.Serialize(r2)); + + var g1 = new GenerationResult { Document = j1 }; + var g2 = new GenerationResult { Document = j2 }; + + mock.Setup(m => m.GenerateJsonDocument(It.Is(r => r.RelationshipType == RelationshipType.DEPENDS_ON))).Returns(g1); + mock.Setup(m => m.GenerateJsonDocument(It.Is(r => r.RelationshipType == RelationshipType.CONTAINS))).Returns(g2); + + ChannelReader channel = rg.Run(rs.GetEnumerator(), mi); + + var docs = new List(); + await foreach (JsonDocument jsonDoc in channel.ReadAllAsync()) + { + docs.Add(jsonDoc); + } + + Assert.IsTrue(docs.Contains(j1)); + Assert.IsTrue(docs.Contains(j2)); + Assert.IsTrue(docs.Count == 2); + } + + + [TestMethod] + public async Task RunShouldNotFailWithNull() + { + Mock mock = new Mock(); + + var m = new ManifestGeneratorProvider(new IManifestGenerator[] { mock.Object }); + + var rg = new RelationshipGenerator(m); + var rs = new List(); + + var mi = new ManifestInfo(); + mi.Name = "Test"; + mi.Version = "1"; + mock.Setup(m => m.RegisterManifest()).Returns(mi); + m.Init(); + + ChannelReader channel = rg.Run(rs.GetEnumerator(), mi); + + var docs = new List(); + await foreach (JsonDocument jsonDoc in channel.ReadAllAsync()) + { + docs.Add(jsonDoc); + } + + Assert.IsTrue(docs.Count == 0); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs new file mode 100644 index 00000000..b995bb1d --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/SBOMComponentsWalkerTests.cs @@ -0,0 +1,126 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog.Events; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using ILogger = Serilog.ILogger; + +namespace Microsoft.Sbom.Api.Executors.Tests +{ + [TestClass] + public class SBOMComponentsWalkerTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock mockConfiguration = new Mock(); + private readonly Mock mockSbomConfigs = new Mock(); + private readonly Mock mockFileSystem = new Mock(); + + public SBOMComponentsWalkerTests() + { + ISbomConfig sbomConfig = new SbomConfig(mockFileSystem.Object) + { + ManifestJsonFilePath = "testpath" + }; + mockConfiguration.SetupGet(c => c.Verbosity).Returns(new ConfigurationSetting { Value = LogEventLevel.Information }); + mockSbomConfigs.Setup(s => s.TryGet(It.IsAny(), out sbomConfig)).Returns(true); + } + + [TestMethod] + public async Task GetComponents() + { + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "componentName", $"123{i}", "abcdef", $"path{i}"), + DetectorId = "SPDX22SBOM" + }; + + scannedComponents.Add(scannedComponent); + } + + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + + var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.Message}"); + } + + Assert.IsTrue(scannedComponents.Count == discoveredComponents.Count); + mockDetector.VerifyAll(); + } + + [TestMethod] + public async Task GetComponentsWithFiltering() + { + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new SpdxComponent("SPDX-2.2", new Uri("http://test.uri"), "componentName", $"123{i}", "abcdef", $"path{i}"), + DetectorId = "SPDX22SBOM" + }; + + scannedComponents.Add(scannedComponent); + } + + var nonSbomComponent = new ScannedComponent + { + Component = new NpmComponent("componentName", "123"), + DetectorId = "notSPDX22SBOM" + }; + scannedComponents.Add(nonSbomComponent); + + var mockDetector = new Mock(new Mock().Object, new Mock().Object); + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + var walker = new SBOMComponentsWalker(mockLogger.Object, mockDetector.Object, mockConfiguration.Object, mockSbomConfigs.Object); + var packagesChannelReader = walker.GetComponents("root"); + + var discoveredComponents = await packagesChannelReader.output.ReadAllAsync().ToListAsync(); + + await foreach (ComponentDetectorException error in packagesChannelReader.error.ReadAllAsync()) + { + Assert.Fail($"Caught exception: {error.Message}"); + } + + Assert.IsTrue(scannedComponents.Where(c => c.Component is SpdxComponent).ToList().Count == discoveredComponents.Count); + mockDetector.VerifyAll(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs b/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs new file mode 100644 index 00000000..62b65bdb --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Executors/SPDXSBOMReaderForExternalDocumentReferenceTests.cs @@ -0,0 +1,195 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Tests.Executors +{ + [TestClass] + public class SPDXSBOMReaderForExternalDocumentReferenceTests + { + private readonly Mock mockHashGenerator = new Mock(); + private readonly Mock mockLogger = new Mock(); + private readonly ISbomConfigProvider sbomConfigs; + private readonly Mock mockConfiguration = new Mock(); + private readonly ManifestGeneratorProvider manifestGeneratorProvider; + private readonly Mock fileSystemMock = new Mock(); + + private const string jsonMissingName = "{\"documentNamespace\": \"namespace\", \"spdxVersion\": \"SPDX-2.2\", \"documentDescribes\":[\"SPDXRef - RootPackage\"]}"; + private const string jsonMissingNamespace = "{\"name\": \"docname\",\"spdxVersion\": \"SPDX-2.2\", \"documentDescribes\":[\"SPDXRef - RootPackage\"]}"; + private const string jsonMissingVersion = "{\"name\": \"docname\",\"documentNamespace\": \"namespace\",\"documentDescribes\":[\"SPDXRef - RootPackage\"]}"; + private const string jsonInvalidVersion = "{\"name\": \"docname\",\"documentNamespace\": \"namespace\", \"spdxVersion\": \"SPDX-2.1\", \"documentDescribes\":[\"SPDXRef - RootPackage\"]}"; + private const string jsonMissingDocumentDescribe = "{\"name\": \"docname\",\"documentNamespace\": \"namespace\", \"spdxVersion\": \"SPDX-2.2\"}"; + + public SPDXSBOMReaderForExternalDocumentReferenceTests() + { + mockConfiguration.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + mockConfiguration.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + mockConfiguration.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "root" }); + + manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + manifestGeneratorProvider.Init(); + + var sbomConfigsMock = new Mock(); + sbomConfigsMock.Setup(c => c.GetManifestInfos()).Returns(new[] { new ManifestInfo { Name = "TestManifest", Version = "1.0.0" } }); + sbomConfigs = sbomConfigsMock.Object; + } + + [TestMethod] + public async Task When_ParseSBOMFile_WithValidSPDXJson_ThenTestPass() + { + mockHashGenerator.Setup(h => h.GenerateHashes(It.IsAny(), It.IsAny())) + .Returns((string fileName, AlgorithmName[] algos) => + algos.Select(a => + new Checksum + { + ChecksumValue = "hash", + Algorithm = a + }) + .ToArray()); + + string json = "{\"name\": \"docname\",\"documentNamespace\": \"namespace\", \"spdxVersion\": \"SPDX-2.2\", \"documentDescribes\":[\"SPDXRef - RootPackage\"]}"; + fileSystemMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(json)); + + List sbomLocations = new List + { + @"d:\directorya\directoryb\file1.spdx.json" + }; + + var sbomLocationChannel = Channel.CreateUnbounded(); + foreach (var sbomLocation in sbomLocations) + { + await sbomLocationChannel.Writer.WriteAsync(sbomLocation); + } + + sbomLocationChannel.Writer.Complete(); + + var spdxSBOMReaderForExternalDocumentReference = new SPDXSBOMReaderForExternalDocumentReference(mockHashGenerator.Object, mockLogger.Object, mockConfiguration.Object, sbomConfigs, manifestGeneratorProvider, fileSystemMock.Object); + var (output, errors) = spdxSBOMReaderForExternalDocumentReference.ParseSBOMFile(sbomLocationChannel); + await foreach (ExternalDocumentReferenceInfo externalDocumentReferenceInfo in output.ReadAllAsync()) + { + Assert.AreEqual("namespace", externalDocumentReferenceInfo.DocumentNamespace); + } + + Assert.IsFalse(await errors.ReadAllAsync().AnyAsync()); + } + + [TestMethod] + public async Task When_ParseSBOMFile_WithIllFormatedJson_ThenReadAsyncFail() + { + mockHashGenerator.Setup(h => h.GenerateHashes(It.IsAny(), It.IsAny())) + .Returns((string fileName, AlgorithmName[] algos) => + algos.Select(a => + new Checksum + { + ChecksumValue = "hash", + Algorithm = a + }) + .ToArray()); + string json = "{\"name\": ,\"documentNamespace\": \"namespace\"}"; + fileSystemMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(json)); + + List sbomLocations = new List + { + @"d:\directorya\directoryb\file1.spdx.json" + }; + + var sbomLocationChannel = Channel.CreateUnbounded(); + foreach (var sbomLocation in sbomLocations) + { + await sbomLocationChannel.Writer.WriteAsync(sbomLocation); + } + + sbomLocationChannel.Writer.Complete(); + + var spdxSBOMReaderForExternalDocumentReference = new SPDXSBOMReaderForExternalDocumentReference(mockHashGenerator.Object, mockLogger.Object, mockConfiguration.Object, sbomConfigs, manifestGeneratorProvider, fileSystemMock.Object); + var (output, errors) = spdxSBOMReaderForExternalDocumentReference.ParseSBOMFile(sbomLocationChannel); + + Assert.IsTrue(await errors.ReadAllAsync().AnyAsync()); + Assert.IsFalse(await output.ReadAllAsync().AnyAsync()); + } + + [TestMethod] + public async Task When_ParseSBOMFile_WithNonSPDXFile_ThenDoNotReadFiles() + { + List nonSpdxSbomLocations = new List + { + @"d:\directorya\directoryb\file1.json" + }; + + var sbomLocationChannel = Channel.CreateUnbounded(); + foreach (var sbomLocation in nonSpdxSbomLocations) + { + await sbomLocationChannel.Writer.WriteAsync(sbomLocation); + } + + sbomLocationChannel.Writer.Complete(); + + var spdxSBOMReaderForExternalDocumentReference = new SPDXSBOMReaderForExternalDocumentReference(mockHashGenerator.Object, mockLogger.Object, mockConfiguration.Object, sbomConfigs, manifestGeneratorProvider, fileSystemMock.Object); + var (output, errors) = spdxSBOMReaderForExternalDocumentReference.ParseSBOMFile(sbomLocationChannel); + + mockHashGenerator.VerifyNoOtherCalls(); + fileSystemMock.VerifyNoOtherCalls(); + + Assert.IsFalse(await errors.ReadAllAsync().AnyAsync()); + Assert.IsFalse(await output.ReadAllAsync().AnyAsync()); + } + + [TestMethod] + [DataRow(jsonMissingName)] + [DataRow(jsonMissingNamespace)] + [DataRow(jsonMissingVersion)] + [DataRow(jsonInvalidVersion)] + [DataRow(jsonMissingDocumentDescribe)] + public async Task When_ParseSBOMFile_WithSPDXDocumentIssues_ThenThrowException(string inputJson) + { + mockHashGenerator.Setup(h => h.GenerateHashes(It.IsAny(), It.IsAny())) + .Returns((string fileName, AlgorithmName[] algos) => + algos.Select(a => + new Checksum + { + ChecksumValue = "hash", + Algorithm = a + }) + .ToArray()); + + fileSystemMock.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString(inputJson)); + + List sbomLocations = new List + { + @"d:\directorya\directoryb\file1.spdx.json" + }; + + var sbomLocationChannel = Channel.CreateUnbounded(); + foreach (var sbomLocation in sbomLocations) + { + await sbomLocationChannel.Writer.WriteAsync(sbomLocation); + } + + sbomLocationChannel.Writer.Complete(); + + var spdxSBOMReaderForExternalDocumentReference = new SPDXSBOMReaderForExternalDocumentReference(mockHashGenerator.Object, mockLogger.Object, mockConfiguration.Object, sbomConfigs, manifestGeneratorProvider, fileSystemMock.Object); + + var (output, errors) = spdxSBOMReaderForExternalDocumentReference.ParseSBOMFile(sbomLocationChannel); + + Assert.IsTrue(await errors.ReadAllAsync().AnyAsync()); + Assert.IsFalse(await output.ReadAllAsync().AnyAsync()); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Filters/DownloadedRootPathFilterTests.cs b/test/Microsoft.Sbom.Api.Tests/Filters/DownloadedRootPathFilterTests.cs new file mode 100644 index 00000000..1ca7edff --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Filters/DownloadedRootPathFilterTests.cs @@ -0,0 +1,64 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Microsoft.Sbom.Api.Filters; +using System; +using System.Collections.Generic; +using System.Text; +using Moq; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Utils; +using Serilog; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Filters.Tests +{ + [TestClass] + public class DownloadedRootPathFilterTests + { + private readonly Mock logger = new Mock(); + + [TestMethod] + public void DownloadedRootPathFilterTest_NoFilterPath_Succeeds() + { + var fileSystemMock = new Mock(); + + var configMock = new Mock(); + configMock.SetupGet(c => c.RootPathFilter).Returns((ConfigurationSetting)null); + + var filter = new DownloadedRootPathFilter(configMock.Object, fileSystemMock.Object, logger.Object); + filter.Init(); + + Assert.IsTrue(filter.IsValid("hello")); + Assert.IsTrue(filter.IsValid(null)); + Assert.IsTrue(filter.IsValid("c:/test")); + fileSystemMock.VerifyAll(); + configMock.VerifyAll(); + } + + [TestMethod] + public void DownloadedRootPathFilterTest_FilterPath_Succeeds() + { + var fileSystemMock = new Mock(); + fileSystemMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string r, string p) => $"{r}/{p}"); + + var configMock = new Mock(); + configMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "C:/test" }); + configMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "validPath" }); + + var filter = new DownloadedRootPathFilter(configMock.Object, fileSystemMock.Object, logger.Object); + filter.Init(); + + Assert.IsTrue(filter.IsValid("c:/test/validPath/test")); + Assert.IsTrue(filter.IsValid("c:/test/validPath")); + Assert.IsTrue(filter.IsValid("c:/test/validPath/test/me")); + Assert.IsFalse(filter.IsValid(null)); + Assert.IsFalse(filter.IsValid("c:/test/InvalidPath")); + Assert.IsFalse(filter.IsValid("c:/test/InvalidPath/f")); + + fileSystemMock.VerifyAll(); + configMock.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Filters/ManifestFolderFilterTests.cs b/test/Microsoft.Sbom.Api.Tests/Filters/ManifestFolderFilterTests.cs new file mode 100644 index 00000000..34a56be9 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Filters/ManifestFolderFilterTests.cs @@ -0,0 +1,43 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Filters.Tests +{ + [TestClass] + public class ManifestFolderFilterTests + { + [TestMethod] + public void ManifestFolderFilterTest_CheckAllManifestFolder_Succeeds() + { + var fileSystemMock = new Mock(); + var mockOSUtils = new Mock(); + mockOSUtils.Setup(o => o.GetFileSystemStringComparisonType()).Returns(StringComparison.CurrentCultureIgnoreCase); + + var configMock = new Mock(); + configMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = "C:/test/_manifest" }); + + var filter = new ManifestFolderFilter(configMock.Object, fileSystemMock.Object, mockOSUtils.Object); + filter.Init(); + + Assert.IsTrue(filter.IsValid("c:/test")); + Assert.IsFalse(filter.IsValid(null)); + Assert.IsTrue(filter.IsValid("c:/test/me")); + Assert.IsTrue(filter.IsValid("me")); + Assert.IsTrue(filter.IsValid("d:/me")); + Assert.IsTrue(filter.IsValid("c:/test\\me")); + Assert.IsTrue(filter.IsValid("c:\\test/me")); + Assert.IsFalse(filter.IsValid("c:/test/_manifest")); + Assert.IsFalse(filter.IsValid("c:/test/_manifest/manifest.json")); + Assert.IsFalse(filter.IsValid("c:\\test\\_manifest")); + Assert.IsFalse(filter.IsValid("c:/test/_manifest\\manifest.json")); + fileSystemMock.VerifyAll(); + configMock.VerifyAll(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Hashing/HashCodeGeneratorTests.cs b/test/Microsoft.Sbom.Api.Tests/Hashing/HashCodeGeneratorTests.cs new file mode 100644 index 00000000..36abe8f1 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Hashing/HashCodeGeneratorTests.cs @@ -0,0 +1,65 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using System; +using System.IO; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Api.Tests; + +namespace Microsoft.Sbom.Api.Hashing.Tests +{ + [TestClass] + public class HashCodeGeneratorTests + { + [TestMethod] + public void GenerateHashTest_Returs2Hashes_Succeeds() + { + var hashAlgorithmNames = new + AlgorithmName[] { AlgorithmName.SHA256, AlgorithmName.SHA512 }; + Checksum[] expectedHashes = new Checksum[] + { + new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "185F8DB32271FE25F561A6FC938B2E264306EC304EDA518007D1764826381969" }, + new Checksum { Algorithm = AlgorithmName.SHA512, ChecksumValue = "3615F80C9D293ED7402687F94B22D58E529B8CC7916F8FAC7FDDF7FBD5AF4CF777D3D795A7A00A16BF7E7F3FB9561EE9BAAE480DA9FE7A18769E71886B03F315" } + }; + + var mockFileSystemUtils = new Mock(); + mockFileSystemUtils.Setup(f => f.OpenRead(It.IsAny())).Returns(TestUtils.GenerateStreamFromString("Hello")); + + var hashCodeGenerator = new HashCodeGenerator(mockFileSystemUtils.Object); + Checksum[] fileHashes = hashCodeGenerator.GenerateHashes("/tmp/file", hashAlgorithmNames); + + Assert.AreEqual(2, fileHashes.Length); + CollectionAssert.AreEqual(expectedHashes, fileHashes); + mockFileSystemUtils.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(IOException))] + public void GenerateHashTest_FileReadFails_Throws() + { + var hashAlgorithmNames = new AlgorithmName[] { AlgorithmName.SHA256, AlgorithmName.SHA512 }; + Checksum[] expectedHashes = new Checksum[] + { + new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = string.Empty }, + new Checksum { Algorithm = AlgorithmName.SHA512, ChecksumValue = string.Empty } + }; + + var mockFileSystemUtils = new Mock(); + mockFileSystemUtils.Setup(f => f.OpenRead(It.IsAny())).Throws(new IOException()); + + var hashCodeGenerator = new HashCodeGenerator(mockFileSystemUtils.Object); + hashCodeGenerator.GenerateHashes("/tmp/file", hashAlgorithmNames); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public void GenerateHashTest_NullFileSystemUtils_Throws() + { + _ = new HashCodeGenerator(null); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Manifest/ManifestConfigProviderTests.cs b/test/Microsoft.Sbom.Api.Tests/Manifest/ManifestConfigProviderTests.cs new file mode 100644 index 00000000..cd150251 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Manifest/ManifestConfigProviderTests.cs @@ -0,0 +1,151 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Manifest.ManifestConfigHandlers; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Ninject.Activation; +using PowerArgs; +using System.Collections.Generic; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Manifest.Tests +{ + [TestClass] + public class ManifestConfigProviderTests + { + private readonly IMetadataBuilderFactory mockMetadataBuilderFactory; + public ManifestConfigProviderTests() + { + var mockBuilderFactory = new Mock(); + mockMetadataBuilderFactory = mockBuilderFactory.Object; + } + + private Mock mockFileSystemUtils; + + [TestInitialize] + public void Setup() + { + mockFileSystemUtils = new Mock(); + + mockFileSystemUtils.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + mockFileSystemUtils.Setup(m => m.JoinPaths(It.IsAny(), It.IsAny())) + .Returns((string p1, string p2) => PathUtils.Join(p1, p2)); + + mockFileSystemUtils.Setup(m => m.JoinPaths(It.IsAny(), It.IsAny(), It.IsAny())) + .Returns((string p1, string p2, string p3) => PathUtils.Join(p1, p2, p3)); + } + + [TestMethod] + public void ManifestConfigProviderTest_Generate_SPDX22_Succeeds() + { + var mockConfiguration = new Mock(); + var mockContext = new Mock(); + + mockConfiguration.Setup(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + mockConfiguration.Setup(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + mockConfiguration.Setup(c => c.ManifestToolAction).Returns(ManifestToolActions.Generate); + mockConfiguration + .Setup(c => c.ManifestInfo) + .Returns( + new ConfigurationSetting> + { + Value = new List { Constants.SPDX22ManifestInfo } + }); + + var configHandlerArray = new IManifestConfigHandler[] + { + new SPDX22ManifestConfigHandler(mockConfiguration.Object, mockFileSystemUtils.Object, mockMetadataBuilderFactory), + }; + + var configProvider = new ManifestConfigProvider(configHandlerArray); + + var config = configProvider.Create(mockContext.Object) as SbomConfig; + + Assert.IsNotNull(config); + Assert.IsTrue(config.ManifestInfo == Constants.SPDX22ManifestInfo); + + string sbomDirPath = PathUtils.Join("/root", + Constants.ManifestFolder, + $"{Constants.SPDX22ManifestInfo.Name.ToLower()}_{Constants.SPDX22ManifestInfo.Version.ToLower()}"); + + // sbom file path is manifest.spdx.json in the sbom directory. + string sbomFilePath = PathUtils.Join(sbomDirPath, $"manifest.{Constants.SPDX22ManifestInfo.Name.ToLower()}.json"); + Assert.AreEqual(config.ManifestJsonDirPath, sbomDirPath); + Assert.AreEqual(config.ManifestJsonFilePath, sbomFilePath); + Mock.VerifyAll(); + } + + // This test should change as the values are not non deterministic. + [ExpectedException(typeof(ValidationArgException))] + public void ManifestConfigProviderTest_Generate_SPDX22_Fails() + { + var mockConfiguration = new Mock(); + var mockContext = new Mock(); + + mockConfiguration.Setup(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + mockConfiguration.Setup(c => c.ManifestToolAction).Returns(ManifestToolActions.Generate); + mockConfiguration + .Setup(c => c.ManifestInfo) + .Returns( + new ConfigurationSetting> + { + Value = new List { Constants.SPDX22ManifestInfo } + }); + var configHandlerArray = new IManifestConfigHandler[] + { + }; + + var configProvider = new ManifestConfigProvider(configHandlerArray); + + var config = configProvider.Create(mockContext.Object) as SbomConfig; + + Assert.IsNotNull(config); + Assert.IsTrue(config.ManifestInfo == Constants.SPDX22ManifestInfo); + + var sbomDirPath = PathUtils.Join("/root", + Constants.ManifestFolder, + $"{Constants.SPDX22ManifestInfo.Name.ToLower()}_{Constants.SPDX22ManifestInfo.Version.ToLower()}"); + + // sbom file path is manifest.spdx.json in the sbom directory. + var sbomFilePath = PathUtils.Join(sbomDirPath, $"manifest.{Constants.SPDX22ManifestInfo.Name.ToLower()}.json"); + Assert.IsTrue(config.ManifestJsonDirPath == sbomDirPath); + Assert.IsTrue(config.ManifestJsonFilePath == sbomFilePath); + Mock.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(ValidationArgException))] + public void ManifestConfigProviderTest_Validate_SPDX_Fails() + { + var mockConfiguration = new Mock(); + var mockContext = new Mock(); + + mockFileSystemUtils + .Setup(f => f.FileExists( + It.Is(d => d.Replace("\\", "/") == "/root/_manifest/spdx_2.2/manifest.spdx.json"))) + .Returns(true); + + mockConfiguration.Setup(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + mockConfiguration.Setup(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + mockConfiguration.Setup(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + + var configHandlerArray = new IManifestConfigHandler[] + { + new SPDX22ManifestConfigHandler(mockConfiguration.Object, mockFileSystemUtils.Object, mockMetadataBuilderFactory) + }; + + var configProvider = new ManifestConfigProvider(configHandlerArray); + + configProvider.Create(mockContext.Object); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj new file mode 100644 index 00000000..667992d9 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Microsoft.Sbom.Api.Tests.csproj @@ -0,0 +1,30 @@ + + + + netcoreapp3.1 + false + + + + TRACE + + + + + all + runtime; build; native; contentfiles; analyzers; buildtransitive + + + + + + + + + + + + + + + diff --git a/test/Microsoft.Sbom.Api.Tests/Output/ManifestToolJsonSerializerTests.cs b/test/Microsoft.Sbom.Api.Tests/Output/ManifestToolJsonSerializerTests.cs new file mode 100644 index 00000000..f59354b5 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Output/ManifestToolJsonSerializerTests.cs @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; +using System.IO; +using System.Text; +using System.Text.Json; + +namespace Microsoft.Sbom.Api.Output.Tests +{ + [TestClass] + public class ManifestToolJsonSerializerTests + { + private readonly string metadataString = "{\"header\":\"value\"}"; + + [TestMethod] + public void ManifestToolJsonSerializerTest_HappyPath_Succeeds() + { + var jsonDoc = JsonDocument.Parse("{\"hello\":\"world\"}"); + + string result = null; + + using (var stream = new MemoryStream()) + { + using (var serializer = new ManifestToolJsonSerializer(stream)) + { + serializer.StartJsonObject(); + serializer.StartJsonArray("Outputs"); + serializer.Write(jsonDoc); + serializer.EndJsonArray(); + serializer.WriteJsonString(metadataString); + serializer.FinalizeJsonObject(); + } + + result = Encoding.UTF8.GetString(stream.ToArray()); + } + + var expected = JsonSerializer.Serialize(JsonDocument.Parse("{\"Outputs\":[{\"hello\":\"world\"}],\"header\":\"value\"}"), new JsonSerializerOptions { WriteIndented = true }); + Assert.AreEqual(expected, result); + + using var stream2 = new MemoryStream(); + using var utfJsonWriter = new Utf8JsonWriter(stream2); + try + { + jsonDoc.WriteTo(utfJsonWriter); + Assert.Fail("Json document was not disposed by the serializer"); + } + catch (Exception e) + { + Assert.AreEqual(typeof(ObjectDisposedException), e.GetType()); + } + } + + [TestMethod] + public void ManifestToolJsonSerializerTest_HeaderWithoutArrayStart_Succeeds() + { + var jsonDoc = JsonDocument.Parse("{\"hello\":\"world\"}"); + + string result = null; + + using (var stream = new MemoryStream()) + { + using (var serializer = new ManifestToolJsonSerializer(stream)) + { + serializer.StartJsonObject(); + serializer.StartJsonArray("Outputs"); + serializer.Write(jsonDoc); + serializer.EndJsonArray(); + serializer.WriteJsonString(metadataString); + serializer.FinalizeJsonObject(); + } + + result = Encoding.UTF8.GetString(stream.ToArray()); + } + + var expected = JsonSerializer.Serialize(JsonDocument.Parse("{\"Outputs\":[{\"hello\":\"world\"}],\"header\":\"value\"}"), new JsonSerializerOptions { WriteIndented = true }); + Assert.AreEqual(expected, result); + + using var stream2 = new MemoryStream(); + using var utfJsonWriter = new Utf8JsonWriter(stream2); + try + { + jsonDoc.WriteTo(utfJsonWriter); + Assert.Fail("Json document was not disposed by the serializer"); + } + catch (Exception e) + { + Assert.AreEqual(typeof(ObjectDisposedException), e.GetType()); + } + } + + [TestMethod] + [ExpectedException(typeof(ObjectDisposedException))] + public void ManifestToolJsonSerializerTest_WriteDisposedJsonDocument_Fails() + { + var jsonDoc = JsonDocument.Parse("{\"hello\":\"world\"}"); + + using var stream = new MemoryStream(); + using var serializer = new ManifestToolJsonSerializer(stream); + + jsonDoc.Dispose(); + + serializer.StartJsonObject(); + serializer.WriteJsonString(metadataString); + serializer.Write(jsonDoc); + serializer.FinalizeJsonObject(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/PathUtils.cs b/test/Microsoft.Sbom.Api.Tests/PathUtils.cs new file mode 100644 index 00000000..f85e5a2a --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/PathUtils.cs @@ -0,0 +1,47 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.IO; + +namespace Microsoft.Sbom.Api.Tests +{ + /// + /// This class is responsible for providing Path functions that are provided + /// in either .net framework or .net core. + /// + /// + /// Comments are from API. + /// + internal class PathUtils + { + /// + /// Create a relative path from one path to another. Paths will be resolved before calculating the difference. + /// Default path comparison for the active platform will be used (OrdinalIgnoreCase for Windows or Mac, Ordinal for Unix). + /// + /// The source path the output should be relative to. This path is always considered to be a directory. + /// The destination path. + /// The relative path or if the paths don't share the same root. + public static string GetRelativePath(string relativeTo, string path) + { + return Path.GetRelativePath(relativeTo, path); + } + + /// + /// Unlike Combine(), Join() methods do not consider rooting. They simply combine paths, ensuring that there + /// is a directory separator between them. + /// + public static string Join(string path1, string path2) + { + return Path.Join(path1, path2); + } + + /// + /// Unlike Combine(), Join() methods do not consider rooting. They simply combine paths, ensuring that there + /// is a directory separator between them. + /// + public static string Join(string path1, string path2, string path3) + { + return Path.Join(path1, path2, path3); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs new file mode 100644 index 00000000..4129dd58 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/SBOMGeneratorTest.cs @@ -0,0 +1,89 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Workflows; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Entities; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Ninject; +using System.Collections.Generic; +using System.IO; +using System.Threading.Tasks; +using EntityErrorType = Microsoft.Sbom.Contracts.Enums.ErrorType; + +namespace Microsoft.Sbom.Api.Tests +{ + [TestClass] + public class SBOMGeneratorTest + { + private Mock fileSystemMock = new Mock(); + private SBOMGenerator generator; + private StandardKernel kernel; + private Mock mockWorkflow; + private Mock mockRecorder; + private RuntimeConfiguration runtimeConfiguration; + + [TestInitialize] + public void Setup() + { + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true).Verifiable(); + fileSystemMock.Setup(f => f.DirectoryHasReadPermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemMock.Setup(f => f.DirectoryHasWritePermissions(It.IsAny())).Returns(true).Verifiable(); + fileSystemMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string p1, string p2) => Path.Join(p1, p2)); + + kernel = new StandardKernel(new Bindings()); + kernel.Unbind(); + kernel.Unbind(); + kernel.Unbind(); + + kernel.Bind().ToConstant(fileSystemMock.Object); + generator = new SBOMGenerator(kernel, fileSystemMock.Object); + mockWorkflow = new Mock(); + mockRecorder = new Mock(); + + runtimeConfiguration = new RuntimeConfiguration + { + NamespaceUriBase = "https://base.uri" + }; + } + + [TestMethod] + public async Task When_GenerateSbomAsync_WithRecordedErrors_Then_PopulateEntityErrors() + { + var fileValidationResults = new List(); + fileValidationResults.Add(new FileValidationResult() { Path = "random", ErrorType = ErrorType.Other }); + + kernel.Bind().ToMethod(x => mockWorkflow.Object).Named(nameof(SBOMGenerationWorkflow)); + kernel.Bind().ToMethod(x => mockRecorder.Object).InSingletonScope(); + mockRecorder.Setup(c => c.Errors).Returns(fileValidationResults).Verifiable(); + mockWorkflow.Setup(c => c.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); + + var result = await generator.GenerateSBOMAsync("rootPath", "compPath", new SBOMMetadata(), configuration: runtimeConfiguration); + + Assert.AreEqual(1, result.Errors.Count); + Assert.AreEqual(EntityErrorType.Other, result.Errors[0].ErrorType); + Assert.AreEqual("random", ((FileEntity)result.Errors[0].Entity).Path); + mockRecorder.Verify(); + mockWorkflow.Verify(); + } + + [TestMethod] + public async Task When_GenerateSbomAsync_WithNoRecordedErrors_Then_EmptyEntityErrors() + { + kernel.Bind().ToMethod(x => mockWorkflow.Object).Named(nameof(SBOMGenerationWorkflow)); + kernel.Bind().ToMethod(x => mockRecorder.Object).InSingletonScope(); + mockRecorder.Setup(c => c.Errors).Returns(new List()).Verifiable(); + mockWorkflow.Setup(c => c.RunAsync()).Returns(Task.FromResult(true)).Verifiable(); + + var result = await generator.GenerateSBOMAsync("rootPath", "compPath", new SBOMMetadata(), configuration: runtimeConfiguration); + + Assert.AreEqual(0, result.Errors.Count); + mockRecorder.Verify(); + mockWorkflow.Verify(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/SignValidator/SignValidationProviderTests.cs b/test/Microsoft.Sbom.Api.Tests/SignValidator/SignValidationProviderTests.cs new file mode 100644 index 00000000..a481e98c --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/SignValidator/SignValidationProviderTests.cs @@ -0,0 +1,70 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System; +using System.Runtime.InteropServices; +using Microsoft.Sbom.Api.Exceptions; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Extensions; + +namespace Microsoft.Sbom.Api.SignValidator.Tests +{ + [TestClass] + public class SignValidationProviderTests + { + IMock mockLogger; + + [TestInitialize] + public void Setup() + { + mockLogger = new Mock(); + } + + [TestMethod] + public void SignValidationProvider_AddsValidator_Succeeds() + { + var mockSignValidator = new Mock(); + mockSignValidator.SetupGet(s => s.SupportedPlatform).Returns(OSPlatform.Windows); + + var mockOSUtils = new Mock(); + mockOSUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Windows); + + var signValidator = new SignValidationProvider(new ISignValidator[] { mockSignValidator.Object }, mockLogger.Object, mockOSUtils.Object); + signValidator.Init(); + Assert.IsTrue(signValidator.Get().Equals(mockSignValidator.Object)); + + mockOSUtils.VerifyAll(); + mockSignValidator.VerifyAll(); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public void SignValidationProvider_NullValidators_Throws() + { + var mockOSUtils = new Mock(); + mockOSUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Windows); + + var signValidator = new SignValidationProvider(null, mockLogger.Object, mockOSUtils.Object); + signValidator.Init(); + signValidator.Get(); + } + + [TestMethod] + [ExpectedException(typeof(SignValidatorNotFoundException))] + public void SignValidationProvider_NotFoundValidators_Throws() + { + var mockSignValidator = new Mock(); + mockSignValidator.SetupGet(s => s.SupportedPlatform).Returns(OSPlatform.Windows); + + var mockOSUtils = new Mock(); + mockOSUtils.Setup(o => o.GetCurrentOSPlatform()).Returns(OSPlatform.Linux); + + var signValidator = new SignValidationProvider(new ISignValidator[] { mockSignValidator.Object }, mockLogger.Object, mockOSUtils.Object); + signValidator.Init(); + signValidator.Get(); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs b/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs new file mode 100644 index 00000000..a511b933 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/TestManifestGenerator.cs @@ -0,0 +1,150 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text.Json; + +namespace Microsoft.Sbom.Api.Tests +{ + class TestManifestGenerator : IManifestGenerator + { + public AlgorithmName[] RequiredHashAlgorithms => new[] { + AlgorithmName.SHA256 + }; + + public string Version { get; set; } = "1.0.0"; + + public IList HeaderKeys => throw new NotImplementedException(); + + public string FilesArrayHeaderName => "Outputs"; + + public string PackagesArrayHeaderName => "Packages"; + + public string RelationshipsArrayHeaderName => "Relationships"; + + public string ExternalDocumentRefArrayHeaderName => "externalDocumentRefs"; + + public GenerationResult GenerateJsonDocument(InternalSBOMFileInfo fileInfo) + { + if (fileInfo is null) + { + throw new ArgumentNullException(nameof(fileInfo)); + } + + if (fileInfo.Checksum == null || fileInfo.Checksum.Count() == 0) + { + throw new ArgumentException(nameof(fileInfo.Checksum)); + } + + if (string.IsNullOrWhiteSpace(fileInfo.Path)) + { + throw new ArgumentException(nameof(fileInfo.Path)); + } + + var jsonString = $@" +{{ + ""Source"":""{fileInfo.Path}"", + ""Sha256Hash"":""{fileInfo.Checksum.Where(h => h.Algorithm == AlgorithmName.SHA256).Select(h => h.ChecksumValue).FirstOrDefault()}"" +}} +"; + + return new GenerationResult + { + Document = JsonDocument.Parse(jsonString), + ResultMetadata = new ResultMetadata + { + EntityId = $"{fileInfo.Path}_{Guid.NewGuid()}" + } + }; + } + + public GenerationResult GenerateJsonDocument(SBOMPackage packageInfo) + { + var jsonString = $@" +{{ + ""Name"": ""{packageInfo.PackageName}"" +}} +"; + + return new GenerationResult + { + Document = JsonDocument.Parse(jsonString), + ResultMetadata = new ResultMetadata + { + EntityId = $"{packageInfo.PackageName}_{Guid.NewGuid()}" + } + }; + } + + public GenerationResult GenerateJsonDocument(Relationship relationship) + { + return new GenerationResult + { + Document = JsonDocument.Parse(JsonSerializer.Serialize(relationship)) + }; + } + + public GenerationResult GenerateJsonDocument(ExternalDocumentReferenceInfo externalDocumentReferenceInfo) + { + var jsonString = $@" + {{ + ""ExternalDocumentId"":""{externalDocumentReferenceInfo.ExternalDocumentName}"", + ""SpdxDocument"":""{externalDocumentReferenceInfo.DocumentNamespace}"" + }} + "; + + return new GenerationResult + { + Document = JsonDocument.Parse(jsonString), + ResultMetadata = new ResultMetadata + { + EntityId = $"{externalDocumentReferenceInfo.ExternalDocumentName}_{Guid.NewGuid()}" + } + }; + } + + public GenerationResult GenerateRootPackage(IInternalMetadataProvider _) + { + var jsonString = $@" +{{ + ""Name"": ""rootPackage"" +}} +"; + + return new GenerationResult + { + Document = JsonDocument.Parse(jsonString), + ResultMetadata = new ResultMetadata + { + DocumentId = "doc-rootPackage-Id", + EntityId = "rootPackage-Id" + } + }; + } + + public IDictionary GetMetadataDictionary(IInternalMetadataProvider internalMetadataProvider) + { + return new Dictionary + { + { "Version", "1.0.0" }, + { "Build", internalMetadataProvider.GetMetadata(MetadataKey.Build_BuildId) }, + { "Definition", internalMetadataProvider.GetMetadata(MetadataKey.Build_DefinitionName) }, + }; + } + + public ManifestInfo RegisterManifest() + { + return new ManifestInfo + { + Name = "TestManifest", + Version = "1.0.0" + }; + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/TestUtils.cs b/test/Microsoft.Sbom.Api.Tests/TestUtils.cs new file mode 100644 index 00000000..d0e4a6d2 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/TestUtils.cs @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System; +using System.Collections.Generic; +using System.IO; +using System.Text; + +namespace Microsoft.Sbom.Api.Tests +{ + internal class TestUtils + { + public static Stream GenerateStreamFromString(string s) + { + MemoryStream stream = new MemoryStream(); + StreamWriter writer = new StreamWriter(stream); + writer.Write(s); + writer.Flush(); + stream.Position = 0; + return stream; + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectionCliArgumentBuilderTests.cs b/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectionCliArgumentBuilderTests.cs new file mode 100644 index 00000000..792b538f --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectionCliArgumentBuilderTests.cs @@ -0,0 +1,158 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class ComponentDetectionCliArgumentBuilderTests + { + [TestMethod] + public void Build_Simple() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/", string.Join(" ", result)); + } + + [TestMethod] + public void Build_Verbosity() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .Verbosity(ComponentDetection.Common.VerbosityMode.Verbose) + .SourceDirectory("X:/hello/world"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Verbose --SourceDirectory X:/hello/world", string.Join(" ", result)); + } + + [TestMethod] + public void Build_WithDetectorArgs() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddDetectorArg("Hello", "World") + .AddDetectorArg("world", "hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --DetectorArgs Hello=World,world=hello", string.Join(" ", result)); + } + + [TestMethod] + public void Build_WithArgs() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("ManifestFile", "Hello") + .AddArg("--DirectoryExclusionList", "X:/hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --ManifestFile Hello --DirectoryExclusionList X:/hello", string.Join(" ", result)); + } + + [TestMethod] + public void Build_WithArgsDuplicate() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("ManifestFile", "Hello") + .AddArg("--DirectoryExclusionList", "X:/hello") + .AddArg("ManifestFile", "Hello") + .AddArg("--DirectoryExclusionList", "X:/hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --ManifestFile Hello --DirectoryExclusionList X:/hello", string.Join(" ", result)); + } + + [TestMethod] + public void Build_ParseAndAddArgs() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .ParseAndAddArgs("--ManifestFile Hello --DirectoryExclusionList X:/hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --ManifestFile Hello --DirectoryExclusionList X:/hello", string.Join(" ", result)); + } + + [TestMethod] + public void Build_ParseAndAddArgsDuplicate() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .ParseAndAddArgs("--ManifestFile Hello --DirectoryExclusionList X:/hello") + .ParseAndAddArgs("--ManifestFile Hello --DirectoryExclusionList X:/hello") + .AddArg("ManifestFile", "Hello") + .AddArg("--DirectoryExclusionList", "X:/hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --ManifestFile Hello --DirectoryExclusionList X:/hello", string.Join(" ", result)); + } + + [TestMethod] + public void Build_AddNoValueArgs() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .Verbosity(ComponentDetection.Common.VerbosityMode.Normal) + .SourceDirectory("X:/") + .ParseAndAddArgs("--ManifestFile Hello --DirectoryExclusionList X:/hello") + .AddArg("Help"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Normal --SourceDirectory X:/ --ManifestFile Hello --DirectoryExclusionList X:/hello --Help", string.Join(" ", result)); + } + + [TestMethod] + public void Build_AddDetectorArgsWeirdWay() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("DetectorArgs", "SPDX=hello") + .AddDetectorArg("Hello", "World") + .AddDetectorArg("world", "hello"); + + var result = builder.Build(); + Assert.AreEqual("scan --Verbosity Quiet --SourceDirectory X:/ --DetectorArgs SPDX=hello,Hello=World,world=hello", string.Join(" ", result)); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public void Build_WithNullValue() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("ManifestFile", null) + .AddArg("--DirectoryExclusionList", "X:/hello"); + + builder.Build(); + } + + [TestMethod] + [ExpectedException(typeof(ArgumentNullException))] + public void Build_WithInvalidArg() + { + var builder = new ComponentDetectionCliArgumentBuilder() + .Scan() + .SourceDirectory("X:/") + .AddArg("ManifestFile", "value") + .AddArg("--", "X:/hello"); + + builder.Build(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectorCachedExecutorTest.cs b/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectorCachedExecutorTest.cs new file mode 100644 index 00000000..e0c9ffa2 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/ComponentDetectorCachedExecutorTest.cs @@ -0,0 +1,52 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.Sbom.Api.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class ComponentDetectorCachedExecutorTest + { + private readonly Mock logger = new Mock(); + private readonly Mock detector = new Mock(); + + [TestInitialize] + public void TestInitialize() + { + logger.Reset(); + detector.Reset(); + } + + [TestMethod] + public void Scan() + { + var executor = new ComponentDetectorCachedExecutor(logger.Object, detector.Object); + var arguments = new string[] { "a", "b", "c" }; + var expectedResult = new ScanResult(); + + detector.Setup(x => x.Scan(arguments)).Returns(expectedResult); + var result = executor.Scan(arguments); + Assert.AreEqual(result, expectedResult); + Assert.IsTrue(detector.Invocations.Count == 1); + } + + [TestMethod] + public void ScanWithCache() + { + var executor = new ComponentDetectorCachedExecutor(logger.Object, detector.Object); + var arguments = new string[] { "a", "b", "c" }; + var expectedResult = new ScanResult(); + + detector.Setup(x => x.Scan(arguments)).Returns(expectedResult); + executor.Scan(arguments); + var result = executor.Scan(arguments); + Assert.AreEqual(result, expectedResult); + Assert.IsTrue(detector.Invocations.Count == 1); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/ExternalReferenceDeduplicatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Utils/ExternalReferenceDeduplicatorTests.cs new file mode 100644 index 00000000..a69fa881 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/ExternalReferenceDeduplicatorTests.cs @@ -0,0 +1,141 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class ExternalReferenceDeduplicatorTests + { + private readonly ChannelUtils channelUtils = new ChannelUtils(); + + [TestMethod] + public async Task When_DeduplicatingExternalDocRefInfo_WithSingleChannel_ThenTestPass() + { + var references = new List() + { + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/1" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/2" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/2" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/3" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/4" + }, + }; + + var inputChannel = Channel.CreateUnbounded(); + + foreach (var reference in references) + { + await inputChannel.Writer.WriteAsync(reference); + } + + inputChannel.Writer.Complete(); + + var deduplicator = new ExternalReferenceDeduplicator(); + var output = deduplicator.Deduplicate(inputChannel); + + var results = await output.ReadAllAsync().ToListAsync(); + + Assert.AreEqual(results.Count, references.Count - 1); + } + + [TestMethod] + public async Task When_DeduplicatingExternalDocRefInfo_WithConcurrentChannel_ThenTestPass() + { + var references = new List() + { + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/1" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/2" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/2" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/3" + }, + new ExternalDocumentReferenceInfo() + { + DocumentNamespace = "http://sbom.test/4" + }, + }; + + var deduplicator = new ExternalReferenceDeduplicator(); + + var task1 = Task.Run(async () => + { + var inputChannel = Channel.CreateUnbounded(); + + foreach (var reference in references) + { + await inputChannel.Writer.WriteAsync(reference); + } + + inputChannel.Writer.Complete(); + + var output = deduplicator.Deduplicate(inputChannel); + + return output; + }); + + var task2 = Task.Run(async () => + { + var inputChannel = Channel.CreateUnbounded(); + + foreach (var reference in references) + { + await inputChannel.Writer.WriteAsync(reference); + } + + inputChannel.Writer.Complete(); + + var output = deduplicator.Deduplicate(inputChannel); + + return output; + }); + + await Task.WhenAll(task1, task2); + var result = channelUtils.Merge(new ChannelReader[] { task1.Result, task2.Result }); + var resultList = await result.ReadAllAsync().ToListAsync(); + + Assert.AreEqual(resultList.Count, references.Count - 1); + } + + [TestMethod] + public void When_GetKeyForExternalDocRef_ThenTestPass() + { + var deduplicator = new ExternalReferenceDeduplicator(); + + Assert.AreEqual("http://sbom.test/1", deduplicator.GetKey(new ExternalDocumentReferenceInfo() { DocumentNamespace = "http://sbom.test/1" })); + Assert.AreEqual(null, deduplicator.GetKey(null)); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/FileSystemUtilsExtensionTest.cs b/test/Microsoft.Sbom.Api.Tests/Utils/FileSystemUtilsExtensionTest.cs new file mode 100644 index 00000000..82980751 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/FileSystemUtilsExtensionTest.cs @@ -0,0 +1,50 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Extensions; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class FileSystemUtilsExtensionTest + { + private readonly Mock fileSystemUtilMock = new Mock(); + private readonly Mock osUtilMock = new Mock(); + private FileSystemUtilsExtension fileSystemUtilsExtension; + + private const string sourcePath = "/source/path"; + + [TestInitialize] + public void Setup() + { + fileSystemUtilsExtension = new FileSystemUtilsExtension() + { + FileSystemUtils = fileSystemUtilMock.Object, + OsUtils = osUtilMock.Object, + }; + osUtilMock.Setup(o => o.GetFileSystemStringComparisonType()).Returns(System.StringComparison.InvariantCultureIgnoreCase); + fileSystemUtilMock.Setup(f => f.AbsolutePath(sourcePath)).Returns($"C:{sourcePath}"); + } + + [TestMethod] + public void When_TargetPathIsOutsideOfSourcePath_Return_False() + { + var targetPath = "/source/outsidePath"; + fileSystemUtilMock.Setup(f => f.AbsolutePath(targetPath)).Returns($"C:{targetPath}"); + + Assert.IsFalse(fileSystemUtilsExtension.IsTargetPathInSource(targetPath, sourcePath)); + } + + [TestMethod] + public void When_TargetPathIsInsideOfSourcePath_Return_True() + { + var targetPath = "/source/path/insidePath"; + fileSystemUtilMock.Setup(f => f.AbsolutePath(targetPath)).Returns($"C:{targetPath}"); + + Assert.IsTrue(fileSystemUtilsExtension.IsTargetPathInSource(targetPath, sourcePath)); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/FileTypeUtilsTest.cs b/test/Microsoft.Sbom.Api.Tests/Utils/FileTypeUtilsTest.cs new file mode 100644 index 00000000..d4a3970a --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/FileTypeUtilsTest.cs @@ -0,0 +1,37 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class FileTypeUtilsTest + { + private readonly FileTypeUtils fileTypeUtils = new FileTypeUtils(); + + [TestMethod] + public void When_GetFileTypeBy_WithSpdxFile_ThenReturnSPDXType() + { + var types = fileTypeUtils.GetFileTypesBy("random.spdx.json"); + Assert.AreEqual(1, types.Count); + Assert.AreEqual(FileType.SPDX, types[0]); + } + + [TestMethod] + public void When_GetFileTypeBy_WithNonNullFile_ThenReturnNull() + { + var types = fileTypeUtils.GetFileTypesBy("random"); + Assert.IsNull(types); + } + + [TestMethod] + public void When_GetFileTypeBy_WithNullFile_ThenReturnNull() + { + var types = fileTypeUtils.GetFileTypesBy(null); + Assert.IsNull(types); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/IdentifierUtilsTests.cs b/test/Microsoft.Sbom.Api.Tests/Utils/IdentifierUtilsTests.cs new file mode 100644 index 00000000..7b73a1e8 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/IdentifierUtilsTests.cs @@ -0,0 +1,36 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Common.Utils; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System; + +namespace DropValidator.Api.Utils.Tests +{ + [TestClass] + public class IdentifierUtilsTests + { + [TestMethod] + public void TryGetGuidFromShortGuidTest_Succeeds() + { + var shortGuid = IdentifierUtils.GetShortGuid(Guid.NewGuid()); + Assert.IsNotNull(shortGuid); + + Assert.IsTrue(IdentifierUtils.TryGetGuidFromShortGuid(shortGuid, out Guid guid)); + Assert.IsFalse(guid.Equals(Guid.Empty)); + } + + [TestMethod] + public void TryGetGuidFromShortGuidTest_BadString_Fails_DoesntThrow() + { + Assert.IsFalse(IdentifierUtils.TryGetGuidFromShortGuid(string.Empty, out Guid guid1)); + Assert.IsTrue(guid1.Equals(Guid.Empty)); + + Assert.IsFalse(IdentifierUtils.TryGetGuidFromShortGuid(null, out Guid guid2)); + Assert.IsTrue(guid2.Equals(Guid.Empty)); + + Assert.IsFalse(IdentifierUtils.TryGetGuidFromShortGuid("asdf", out Guid guid3)); + Assert.IsTrue(guid3.Equals(Guid.Empty)); + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/OSUtilsTest.cs b/test/Microsoft.Sbom.Api.Tests/Utils/OSUtilsTest.cs new file mode 100644 index 00000000..4ea9517a --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/OSUtilsTest.cs @@ -0,0 +1,96 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using System.Collections.Generic; +using Moq; +using Serilog; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections; +using Microsoft.Sbom.Common; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class OSUtilsTest + { + private readonly Mock logger = new Mock(); + + private readonly Mock environment = new Mock(); + + private OSUtils osUtils; + + const string variable = "Packaging.Variable"; + + [TestInitialize] + public void TestInitialize() + { + logger.Reset(); + environment.Reset(); + } + + [TestMethod] + public void GetEnvironmentVariable_SingleEnvVar() + { + IDictionary d = new Dictionary() + { + { "Agent", "a" }, + { variable, "true" }, + }; + + environment.Setup(o => o.GetEnvironmentVariables()).Returns(d); + osUtils = new OSUtils(logger.Object, environment.Object); + + Assert.AreEqual("true", osUtils.GetEnvironmentVariable(variable)); + environment.VerifyAll(); + logger.VerifyAll(); + } + + [TestMethod] + public void GetEnvironmentVariable_DuplicateEnvVar() + { + IDictionary d = new Dictionary() + { + { "Agent", "a" }, + { variable, "true" }, + { variable.ToLower(), "trueLower" }, + { variable.ToUpper(), "trueUpper" }, + }; + + environment.Setup(o => o.GetEnvironmentVariables()).Returns(d); + osUtils = new OSUtils(logger.Object, environment.Object); + + Assert.AreEqual("true", osUtils.GetEnvironmentVariable(variable)); + environment.VerifyAll(); + logger.Verify(o => o.Warning($"There are duplicate environment variables in different case for {variable}, the value used is true"), Times.Once()); + } + + [TestMethod] + public void GetEnvironmentVariable_Null() + { + IDictionary d = new Dictionary() + { + { "Agent", "a" }, + }; + + environment.Setup(o => o.GetEnvironmentVariables()).Returns(d); + osUtils = new OSUtils(logger.Object, environment.Object); + + Assert.AreEqual(null, osUtils.GetEnvironmentVariable(variable)); + environment.VerifyAll(); + logger.VerifyAll(); + } + + [TestMethod] + public void GetEnvironmentVariable_NullFromEmptyEnvVar() + { + IDictionary d = new Dictionary() { }; + + environment.Setup(o => o.GetEnvironmentVariables()).Returns(d); + osUtils = new OSUtils(logger.Object, environment.Object); + + Assert.AreEqual(null, osUtils.GetEnvironmentVariable(variable)); + environment.VerifyAll(); + logger.VerifyAll(); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Utils/SBOMFileDedeplicatorTests.cs b/test/Microsoft.Sbom.Api.Tests/Utils/SBOMFileDedeplicatorTests.cs new file mode 100644 index 00000000..666b5a17 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Utils/SBOMFileDedeplicatorTests.cs @@ -0,0 +1,142 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Contracts; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Channels; +using System.Threading.Tasks; + +namespace Microsoft.Sbom.Api.Tests.Utils +{ + [TestClass] + public class SBOMFileDedeplicatorTests + { + private ChannelUtils channelUtils = new ChannelUtils(); + + [TestMethod] + public async Task When_DeduplicatingSBOMFile_WithSingleChannel_ThenTestPass() + { + var sbomFiles = new List() + { + new InternalSBOMFileInfo() + { + Path = "./file1.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file2.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file2.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file3.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file4.txt" + } + }; + + var inputChannel = Channel.CreateUnbounded(); + + foreach (var sbomFile in sbomFiles) + { + await inputChannel.Writer.WriteAsync(sbomFile); + } + + inputChannel.Writer.Complete(); + + var deduplicator = new InternalSBOMFileInfoDeduplicator(); + var output = deduplicator.Deduplicate(inputChannel); + + var results = await output.ReadAllAsync().ToListAsync(); + + Assert.AreEqual(results.Count, sbomFiles.Count - 1); + } + + [TestMethod] + public async Task When_DeduplicatingSBOMFile_WithConcurrentChannel_ThenTestPass() + { + var sbomFiles = new List() + { + new InternalSBOMFileInfo() + { + Path = "./file1.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file2.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file2.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file3.txt" + }, + new InternalSBOMFileInfo() + { + Path = "./file4.txt" + } + }; + + var deduplicator = new InternalSBOMFileInfoDeduplicator(); + + var task1 = Task.Run(async () => + { + var inputChannel = Channel.CreateUnbounded(); + + foreach (var fileInfo in sbomFiles) + { + await inputChannel.Writer.WriteAsync(fileInfo); + } + + inputChannel.Writer.Complete(); + + var output = deduplicator.Deduplicate(inputChannel); + + return output; + }); + + var task2 = Task.Run(async () => + { + var inputChannel = Channel.CreateUnbounded(); + + foreach (var fileInfo in sbomFiles) + { + await inputChannel.Writer.WriteAsync(fileInfo); + } + + inputChannel.Writer.Complete(); + + var output = deduplicator.Deduplicate(inputChannel); + + return output; + }); + + await Task.WhenAll(task1, task2); + var result = channelUtils.Merge(new ChannelReader[] { task1.Result, task2.Result }); + var resultList = await result.ReadAllAsync().ToListAsync(); + + Assert.AreEqual(resultList.Count, sbomFiles.Count - 1); + } + + [TestMethod] + public void When_GetKeyForSBOMFile_ThenTestPass() + { + var deduplicator = new InternalSBOMFileInfoDeduplicator(); + + Assert.AreEqual("./file1.txt", deduplicator.GetKey(new InternalSBOMFileInfo() { Path = "./file1.txt" })); + Assert.AreEqual(null, deduplicator.GetKey(null)); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/DropHashValidationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/DropHashValidationWorkflowTests.cs new file mode 100644 index 00000000..c0502c2f --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/DropHashValidationWorkflowTests.cs @@ -0,0 +1,644 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Convertors; +using Microsoft.Sbom.Api.Entities.Output; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Filters; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Common; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Serilog; +using System; +using System.Collections.Concurrent; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; +using ErrorType = Microsoft.Sbom.Api.Entities.ErrorType; + +namespace Microsoft.Sbom.Api.Workflows.Tests +{ + [TestClass] + public class DropHashValidationWorkflowTests + { + private readonly Mock mockLogger = new Mock(); + private readonly Mock recorder = new Mock(); + private readonly Mock mockOSUtils = new Mock(); + private readonly Mock fileSystemExtension = new Mock(); + + [TestInitialize] + public void TestInitialize() + { + fileSystemExtension.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(true); + } + + [TestMethod] + public async Task DropHashValidationWorkflowTestAsync_ReturnsSuccessAndValidationFailures_Succeeds() + { + Mock fileSystemMock = GetDefaultFileSystemMock(); + var manifestData = GetDefaultManifestData(); + + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var hashCodeGeneratorMock = new Mock(); + hashCodeGeneratorMock.Setup(h => h.GenerateHashes( + It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + hashCodeGeneratorMock.Setup(h => h.GenerateHashes( + It.Is(a => a == "/root/child2/grandchild1/file10"), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Throws(new FileNotFoundException()); + + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = false }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + + var outputWriterMock = new Mock(); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileHasher = new FileHasher( + hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemExtension.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = manifestData + }; + + var recorderMock = new Mock().Object; + + var workflow = new DropValidatorWorkflow( + configurationMock.Object, + new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + new ManifestFolderFilterer(manifestFilterMock, + mockLogger.Object), + new ChannelUtils(), + fileHasher, + new HashValidator(configurationMock.Object, manifestData), + manifestData, + validationResultGenerator, + outputWriterMock.Object, + mockLogger.Object, + signValidatorMock.Object, + new ManifestFileFilterer( + manifestData, + rootFileFilterMock, + configurationMock.Object, + mockLogger.Object, + fileSystemMock.Object), + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsFalse(result); + + var nodeValidationResults = validationResultGenerator.NodeValidationResults; + + var additionalFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.AdditionalFile).ToList(); + Assert.AreEqual(1, additionalFileErrors.Count); + Assert.AreEqual("/child2/grandchild1/file7", additionalFileErrors.First().Path); + + var missingFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.MissingFile).ToList(); + Assert.AreEqual(1, missingFileErrors.Count); + Assert.AreEqual("/child2/grandchild2/file10", missingFileErrors.First().Path); + + var invalidHashErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.InvalidHash).ToList(); + Assert.AreEqual(1, invalidHashErrors.Count); + Assert.AreEqual("/child2/grandchild1/file9", invalidHashErrors.First().Path); + + var otherErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.Other).ToList(); + Assert.AreEqual(1, otherErrors.Count); + Assert.AreEqual("/child2/grandchild1/file10", otherErrors.First().Path); + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + } + + [TestMethod] + public async Task DropHashValidationWorkflowTestAsync_NoErrors() + { + Mock fileSystemMock = GetDefaultFileSystemMock(); + + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var recorderMock = new Mock().Object; + var manifestData = GetDefaultManifestData(); + + manifestData.HashesMap["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file9hash" } }; + manifestData.HashesMap["/child2/grandchild1/file7"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file7hash" } }; + manifestData.HashesMap["/child2/grandchild1/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file10hash" } }; + + manifestData.HashesMap.Remove("/child2/grandchild2/file10"); + + var hashCodeGeneratorMock = new Mock(); + hashCodeGeneratorMock.Setup(h => h.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = false }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + + var outputWriterMock = new Mock(); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemExtension.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = manifestData + }; + + var workflow = new DropValidatorWorkflow( + configurationMock.Object, + new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + new ManifestFolderFilterer(manifestFilterMock, + mockLogger.Object), + new ChannelUtils(), + fileHasher, + new HashValidator(configurationMock.Object, manifestData), + manifestData, + validationResultGenerator, + outputWriterMock.Object, + mockLogger.Object, + signValidatorMock.Object, + new ManifestFileFilterer( + manifestData, + rootFileFilterMock, + configurationMock.Object, + mockLogger.Object, + fileSystemMock.Object), + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsTrue(result); + + var nodeValidationResults = validationResultGenerator.NodeValidationResults; + + var additionalFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.AdditionalFile).ToList(); + Assert.AreEqual(0, additionalFileErrors.Count); + + var missingFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.MissingFile).ToList(); + Assert.AreEqual(0, missingFileErrors.Count); + + var invalidHashErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.InvalidHash).ToList(); + Assert.AreEqual(0, invalidHashErrors.Count); + + var otherErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.Other).ToList(); + Assert.AreEqual(0, otherErrors.Count); + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + } + + [TestMethod] + public async Task DropHashValidationWorkflowTestAsync_IgnoreMissingTrue() + { + Mock fileSystemMock = GetDefaultFileSystemMock(); + + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var recorderMock = new Mock().Object; + var manifestData = GetDefaultManifestData(); + + manifestData.HashesMap["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file9hash" } }; + manifestData.HashesMap["/child2/grandchild1/file7"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file7hash" } }; + manifestData.HashesMap["/child2/grandchild1/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file10hash" } }; + + var hashCodeGeneratorMock = new Mock(); + hashCodeGeneratorMock.Setup(h => h.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + + var outputWriterMock = new Mock(); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemExtension.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = manifestData + }; + + var workflow = new DropValidatorWorkflow( + configurationMock.Object, + new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + new ManifestFolderFilterer(manifestFilterMock, + mockLogger.Object), + new ChannelUtils(), + fileHasher, + new HashValidator(configurationMock.Object, manifestData), + manifestData, + validationResultGenerator, + outputWriterMock.Object, + mockLogger.Object, + signValidatorMock.Object, + new ManifestFileFilterer( + manifestData, + rootFileFilterMock, + configurationMock.Object, + mockLogger.Object, + fileSystemMock.Object), + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsTrue(result); + + var nodeValidationResults = validationResultGenerator.NodeValidationResults; + + var additionalFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.AdditionalFile).ToList(); + Assert.AreEqual(0, additionalFileErrors.Count); + + var missingFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.MissingFile).ToList(); + Assert.AreEqual(1, missingFileErrors.Count); + + var invalidHashErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.InvalidHash).ToList(); + Assert.AreEqual(0, invalidHashErrors.Count); + + var otherErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.Other).ToList(); + Assert.AreEqual(0, otherErrors.Count); + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + } + + [TestMethod] + public async Task DropHashValidationWorkflowTestAsync_IgnoreMissingTrueAndInvalidHashShouldFail() + { + Mock fileSystemMock = GetDefaultFileSystemMock(); + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var recorderMock = new Mock().Object; + var manifestData = GetDefaultManifestData(); + + manifestData.HashesMap["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file9hash" } }; + manifestData.HashesMap["/child2/grandchild1/file7"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file7hash" } }; + manifestData.HashesMap["/child2/grandchild1/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file10hash" } }; + + var hashCodeGeneratorMock = new Mock(); + hashCodeGeneratorMock.Setup(h => h.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = true }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + + var outputWriterMock = new Mock(); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemExtension.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = manifestData + }; + + var workflow = new DropValidatorWorkflow( + configurationMock.Object, + new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + new ManifestFolderFilterer(manifestFilterMock, + mockLogger.Object), + new ChannelUtils(), + fileHasher, + new HashValidator(configurationMock.Object, manifestData), + manifestData, + validationResultGenerator, + outputWriterMock.Object, + mockLogger.Object, + signValidatorMock.Object, + new ManifestFileFilterer( + manifestData, + rootFileFilterMock, + configurationMock.Object, + mockLogger.Object, + fileSystemMock.Object), + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsTrue(result); + + var nodeValidationResults = validationResultGenerator.NodeValidationResults; + + var additionalFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.AdditionalFile).ToList(); + Assert.AreEqual(0, additionalFileErrors.Count); + + var missingFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.MissingFile).ToList(); + Assert.AreEqual(1, missingFileErrors.Count); + + var invalidHashErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.InvalidHash).ToList(); + Assert.AreEqual(0, invalidHashErrors.Count); + + var otherErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.Other).ToList(); + Assert.AreEqual(0, otherErrors.Count); + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + } + + [TestMethod] + public async Task DropHashValidationWorkflowTestAsync_IgnoreMissingFalse() + { + Mock fileSystemMock = GetDefaultFileSystemMock(); + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + var recorderMock = new Mock().Object; + var manifestData = GetDefaultManifestData(); + + manifestData.HashesMap["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file9hash" } }; + manifestData.HashesMap["/child2/grandchild1/file7"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file7hash" } }; + manifestData.HashesMap["/child2/grandchild1/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file10hash" } }; + + var hashCodeGeneratorMock = new Mock(); + hashCodeGeneratorMock.Setup(h => h.GenerateHashes(It.IsAny(), + new AlgorithmName[] { Constants.DefaultHashAlgorithmName })) + .Returns((string fileName, AlgorithmName[] algos) => + new Checksum[] + { + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = Constants.DefaultHashAlgorithmName + } + }); + + var configurationMock = new Mock(); + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.HashAlgorithm).Returns(new ConfigurationSetting { Value = Constants.DefaultHashAlgorithmName }); + configurationMock.SetupGet(c => c.RootPathFilter).Returns(new ConfigurationSetting { Value = "child1;child2;child3" }); + configurationMock.SetupGet(c => c.IgnoreMissing).Returns(new ConfigurationSetting { Value = false }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Validate); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(true); + + var validationResultGenerator = new ValidationResultGenerator(configurationMock.Object, manifestData); + + var outputWriterMock = new Mock(); + + var rootFileFilterMock = new DownloadedRootPathFilter(configurationMock.Object, fileSystemMock.Object, mockLogger.Object); + rootFileFilterMock.Init(); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + var fileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemExtension.Object), + mockLogger.Object, + configurationMock.Object, + new Mock().Object, + new ManifestGeneratorProvider(null), + new FileTypeUtils()) + { + ManifestData = manifestData + }; + + var workflow = new DropValidatorWorkflow( + configurationMock.Object, + new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + new ManifestFolderFilterer(manifestFilterMock, + mockLogger.Object), + new ChannelUtils(), + fileHasher, + new HashValidator(configurationMock.Object, manifestData), + manifestData, + validationResultGenerator, + outputWriterMock.Object, + mockLogger.Object, + signValidatorMock.Object, + new ManifestFileFilterer( + manifestData, + rootFileFilterMock, + configurationMock.Object, + mockLogger.Object, + fileSystemMock.Object), + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsFalse(result); + + var nodeValidationResults = validationResultGenerator.NodeValidationResults; + + var additionalFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.AdditionalFile).ToList(); + Assert.AreEqual(0, additionalFileErrors.Count); + + var missingFileErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.MissingFile).ToList(); + Assert.AreEqual(1, missingFileErrors.Count); + + var invalidHashErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.InvalidHash).ToList(); + Assert.AreEqual(0, invalidHashErrors.Count); + + var otherErrors = nodeValidationResults.Where(a => a.ErrorType == ErrorType.Other).ToList(); + Assert.AreEqual(0, otherErrors.Count); + + configurationMock.VerifyAll(); + signValidatorMock.VerifyAll(); + fileSystemMock.VerifyAll(); + } + + [TestMethod] + public async Task SignValidationFailsDoesntRunWorkflow_Fails() + { + var signValidatorMock = new Mock(); + signValidatorMock.Setup(s => s.Validate()).Returns(false); + var recorderMock = new Mock().Object; + + var workflow = new DropValidatorWorkflow( + null, + null, + null, + null, + null, + null, + GetDefaultManifestData(), + null, + null, + mockLogger.Object, + signValidatorMock.Object, + null, + recorderMock); + + var result = await workflow.RunAsync(); + Assert.IsFalse(result); + signValidatorMock.VerifyAll(); + } + + private static Mock GetDefaultFileSystemMock() + { + var fileSystemMock = new Mock(); + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "/root"), true)).Returns(new string[] { "child1", "child2", "child3", "_manifest" }); + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "child1"), true)).Returns(new string[] { }); + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "child2"), true)).Returns(new string[] { "grandchild1", "grandchild2" }); + + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child1"), true)).Returns(new string[] { "/root/child1/file1", "/root/child1/file2" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child2"), true)).Returns(new string[] { "/root/child2/file3", "/root/child2/file4", "/root/child2/file5" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child3"), true)).Returns(new string[] { "/root/child3/file11", "/root/child3/file12" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "_manifest"), true)).Returns(new string[] { "/root/_manifest/manifest.json", "/root/_manifest/manifest.cat" }); + + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "grandchild1"), true)).Returns(new string[] { "/root/child2/grandchild1/file6", "/root/child2/grandchild1/file10" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "grandchild2"), true)).Returns(new string[] { "/root/child2/grandchild1/file7", "/root/child2/grandchild1/file9" }); + + fileSystemMock.Setup(f => f.JoinPaths(It.IsAny(), It.IsAny())).Returns((string r, string p) => $"{r}/{p}"); + return fileSystemMock; + } + + private ManifestData GetDefaultManifestData() + { + IDictionary hashDictionary = new Dictionary + { + ["/child1/file1"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child1/file1hash" } }, + ["/child1/file2"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child1/file2hash" } }, + ["/child2/file3"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file3hash" } }, + ["/child2/file4"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file4hash" } }, + ["/child2/file5"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/file5hash" } }, + ["/child3/file11"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child3/file11hash" } }, + ["/child3/file12"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child3/file12hash" } }, + ["/child2/grandchild1/file6"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child2/grandchild1/file6hash" } }, + ["/child5/file8"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "/root/child5/file8hash" } }, + ["/child2/grandchild1/file9"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "incorrectHash" } }, + ["/child2/grandchild2/file10"] = new Checksum[] { new Checksum { Algorithm = AlgorithmName.SHA256, ChecksumValue = "missingfile" } } + }; + return new ManifestData + { + HashesMap = new ConcurrentDictionary(hashDictionary, StringComparer.InvariantCultureIgnoreCase) + }; + } + } +} \ No newline at end of file diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs new file mode 100644 index 00000000..9845b7cf --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/Helpers/RelationshipsArrayGeneratorTest.cs @@ -0,0 +1,185 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Recorder; +using Microsoft.Sbom.Api.Workflows.Helpers; +using Microsoft.Sbom.Common; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using MoreLinq; +using Serilog; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Threading.Tasks; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Tests.Workflows.Helpers +{ + [TestClass] + public class RelationshipsArrayGeneratorTest + { + private RelationshipsArrayGenerator relationshipsArrayGenerator; + + private readonly Mock recorderMock = new Mock(); + private readonly Mock sbomConfigsMock = new Mock(); + private readonly Mock relationshipGeneratorMock = new Mock(new ManifestGeneratorProvider(null)); + private readonly Mock loggerMock = new Mock(); + private readonly Mock mockLogger = new Mock(); + private readonly Mock fileSystemUtilsMock = new Mock(); + + ManifestGeneratorProvider manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + ISbomPackageDetailsRecorder recorder; + IMetadataBuilder metadataBuilder; + ISbomConfig sbomConfig; + ManifestInfo manifestInfo = new ManifestInfo(); + + private const string documentId = "documentId"; + private const string rootPackageId = "rootPackageId"; + private const string fileId1 = "fileId1"; + private const string fileId2 = "fileId2"; + private InternalSBOMFileInfo file1 = new InternalSBOMFileInfo() { Path = fileId1 }; + private const string packageId1 = "packageId1"; + private const string externalDocRefId1 = "externalDocRefId1"; + private const string manifestJsonDirPath = "/root/_manifest"; + private const string jsonFilePath = "/root/_manifest/manifest.json"; + + List relationships; + + [TestInitialize] + public void Setup() + { + recorder = new SbomPackageDetailsRecorder(); + relationships = new List(); + relationshipGeneratorMock.Setup(r => r.Run(It.IsAny>(), It.IsAny())) + .Callback, ManifestInfo>((relationship, manifestInfo) => + { + while (relationship.MoveNext()) + { + relationships.Add(relationship.Current); + } + }); + relationshipGeneratorMock.CallBase = true; + relationshipsArrayGenerator = new RelationshipsArrayGenerator() + { + ChannelUtils = new ChannelUtils(), + Recorder = recorderMock.Object, + Generator = relationshipGeneratorMock.Object, + Log = loggerMock.Object, + SbomConfigs = sbomConfigsMock.Object, + }; + manifestGeneratorProvider.Init(); + metadataBuilder = new MetadataBuilder( + mockLogger.Object, + manifestGeneratorProvider, + Constants.TestManifestInfo, + recorderMock.Object); + sbomConfig = new SbomConfig(fileSystemUtilsMock.Object) + { + ManifestInfo = Constants.TestManifestInfo, + ManifestJsonDirPath = manifestJsonDirPath, + ManifestJsonFilePath = jsonFilePath, + MetadataBuilder = metadataBuilder, + Recorder = recorder, + }; + fileSystemUtilsMock.Setup(f => f.CreateDirectory(manifestJsonDirPath)); + fileSystemUtilsMock.Setup(f => f.OpenWrite(jsonFilePath)).Returns(new MemoryStream()); + + sbomConfig.StartJsonSerialization(); + sbomConfig.JsonSerializer.StartJsonObject(); + + sbomConfigsMock.Setup(s => s.GetManifestInfos()).Returns(new List { manifestInfo }); + sbomConfigsMock.Setup(s => s.Get(manifestInfo)).Returns(sbomConfig); + } + + [TestMethod] + public async Task When_BaseGenerationDataExist_DescribesRelationshipsAreGenerated() + { + recorder.RecordDocumentId(documentId); + recorder.RecordRootPackageId(rootPackageId); + var results = await relationshipsArrayGenerator.GenerateAsync(); + + Assert.AreEqual(0, results.Count); + Assert.AreEqual(1, relationships.Count); + + var describesRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DESCRIBES); + Assert.AreEqual(1, describesRelationships.Count()); + var describesRelationship = describesRelationships.First(); + Assert.AreEqual(rootPackageId, describesRelationship.TargetElementId); + Assert.AreEqual(documentId, describesRelationship.SourceElementId); + } + + [TestMethod] + public async Task When_SPDXFileGenerationDataExist_DescribedByRelationshipsAreGenerated() + { + recorder.RecordDocumentId(documentId); + recorder.RecordRootPackageId(rootPackageId); + recorder.RecordFileId(fileId1); + recorder.RecordFileId(fileId2); + recorder.RecordSPDXFileId(fileId1); + var results = await relationshipsArrayGenerator.GenerateAsync(); + + Assert.AreEqual(0, results.Count); + Assert.AreEqual(2, relationships.Count); + + var describedByRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DESCRIBED_BY); + Assert.AreEqual(1, describedByRelationships.Count()); + var describedByRelationship = describedByRelationships.First(); + Assert.AreEqual(documentId, describedByRelationship.TargetElementId); + Assert.AreEqual(fileId1, describedByRelationship.SourceElementId); + } + + [TestMethod] + public async Task When_ExternalDocRefGenerationDataExist_PreReqRelationshipsAreGenerated() + { + recorder.RecordDocumentId(documentId); + recorder.RecordRootPackageId(rootPackageId); + recorder.RecordExternalDocumentReferenceIdAndRootElement(externalDocRefId1, rootPackageId); + var results = await relationshipsArrayGenerator.GenerateAsync(); + + Assert.AreEqual(0, results.Count); + Assert.AreEqual(2, relationships.Count); + + var preReqForRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.PREREQUISITE_FOR); + Assert.AreEqual(1, preReqForRelationships.Count()); + var preReqForRelationship = preReqForRelationships.First(); + Assert.AreEqual(rootPackageId, preReqForRelationship.TargetElementId); + Assert.AreEqual(rootPackageId, preReqForRelationship.SourceElementId); + Assert.AreEqual(externalDocRefId1, preReqForRelationship.TargetElementExternalReferenceId); + } + + [TestMethod] + public async Task When_PackageGenerationDataExist_DependOnRelationshipsAreGenerated() + { + recorder.RecordDocumentId(documentId); + recorder.RecordRootPackageId(rootPackageId); + recorder.RecordPackageId(packageId1); + var results = await relationshipsArrayGenerator.GenerateAsync(); + + Assert.AreEqual(0, results.Count); + Assert.AreEqual(2, relationships.Count); + + var dependsOnRelationships = relationships.Where(r => r.RelationshipType == RelationshipType.DEPENDS_ON); + Assert.AreEqual(1, dependsOnRelationships.Count()); + var dependsOnRelationship = dependsOnRelationships.First(); + Assert.AreEqual(packageId1, dependsOnRelationship.TargetElementId); + Assert.AreEqual(rootPackageId, dependsOnRelationship.SourceElementId); + } + + [TestMethod] + public async Task When_NoGenerationDataExist_NoRelationshipsAreGenerated() + { + var results = await relationshipsArrayGenerator.GenerateAsync(); + + Assert.AreEqual(0, results.Count); + Assert.AreEqual(0, relationships.Count); + } + } +} diff --git a/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs new file mode 100644 index 00000000..3e48a4a5 --- /dev/null +++ b/test/Microsoft.Sbom.Api.Tests/Workflows/ManifestGenerationWorkflowTests.cs @@ -0,0 +1,446 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Api.Tests; +using Microsoft.Sbom.Extensions; +using Microsoft.Sbom.Extensions.Entities; +using Microsoft.ComponentDetection.Contracts.TypedComponent; +using Microsoft.ComponentDetection.Contracts.BcdeModels; +using Microsoft.VisualStudio.TestTools.UnitTesting; +using Moq; +using Newtonsoft.Json.Linq; +using Serilog.Events; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Text; +using System.Threading.Channels; +using System.Threading.Tasks; +using Microsoft.Sbom.Common.Config; +using Microsoft.Sbom.Api.Convertors; +using Microsoft.Sbom.Api.Entities; +using Microsoft.Sbom.Api.Executors; +using Microsoft.Sbom.Api.Filters; +using Microsoft.Sbom.Api.Hashing; +using Microsoft.Sbom.Api.Manifest; +using Microsoft.Sbom.Api.Manifest.Configuration; +using Microsoft.Sbom.Api.Output; +using Microsoft.Sbom.Api.Output.Telemetry; +using Microsoft.Sbom.Api.Providers; +using Microsoft.Sbom.Api.Providers.FilesProviders; +using Microsoft.Sbom.Api.Providers.PackagesProviders; +using Microsoft.Sbom.Api.Recorder; +using Microsoft.Sbom.Api.Utils; +using Microsoft.Sbom.Api.Workflows.Helpers; +using ILogger = Serilog.ILogger; +using Microsoft.ComponentDetection.Contracts; +using Microsoft.Sbom.Api.Providers.ExternalDocumentReferenceProviders; +using Microsoft.Sbom.Contracts; +using Microsoft.Sbom.Contracts.Enums; +using Checksum = Microsoft.Sbom.Contracts.Checksum; +using Microsoft.Sbom.Common; +using Constants = Microsoft.Sbom.Api.Utils.Constants; + +namespace Microsoft.Sbom.Api.Workflows.Tests +{ + [TestClass] + public class ManifestGenerationWorkflowTests + { + private readonly Mock recorderMock = new Mock(); + + private readonly Mock fileSystemMock = new Mock(); + private readonly Mock configurationMock = new Mock(); + private readonly Mock mockLogger = new Mock(); + private readonly Mock hashCodeGeneratorMock = new Mock(); + private readonly Mock mockOSUtils = new Mock(); + private readonly Mock mockConfigHandler = new Mock(); + private readonly Mock mockMetadataProvider = new Mock(); + private readonly Mock mockDetector = new Mock(new Mock().Object, new Mock().Object); + private readonly Mock relationshipArrayGenerator = new Mock(); + private readonly Mock packageInfoConverterMock = new Mock(); + private readonly Mock sBOMReaderForExternalDocumentReferenceMock = new Mock(); + private readonly Mock fileSystemUtilsExtensionMock = new Mock(); + + [TestInitialize] + public void Setup() + { + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + fileSystemUtilsExtensionMock.Setup(f => f.IsTargetPathInSource(It.IsAny(), It.IsAny())).Returns(true); + } + + [TestMethod] + [DataRow(true, true)] + [DataRow(false, false)] + [DataRow(true, false)] + [DataRow(false, true)] + public async Task ManifestGenerationWorkflowTests_Succeeds(bool deleteExistingManifestDir, bool isDefaultSourceManifestDirPath) + { + var manifestGeneratorProvider = new ManifestGeneratorProvider(new IManifestGenerator[] { new TestManifestGenerator() }); + manifestGeneratorProvider.Init(); + + var metadataBuilder = new MetadataBuilder( + mockLogger.Object, + manifestGeneratorProvider, + Constants.TestManifestInfo, + recorderMock.Object); + var jsonFilePath = "/root/_manifest/manifest.json"; + + ISbomConfig sbomConfig = new SbomConfig(fileSystemMock.Object) + { + ManifestInfo = Constants.TestManifestInfo, + ManifestJsonDirPath = "/root/_manifest", + ManifestJsonFilePath = jsonFilePath, + MetadataBuilder = metadataBuilder, + Recorder = new SbomPackageDetailsRecorder() + }; + + mockConfigHandler.Setup(c => c.TryGetManifestConfig(out sbomConfig)).Returns(true); + mockMetadataProvider.SetupGet(m => m.MetadataDictionary).Returns(new Dictionary + { + { MetadataKey.Build_BuildId, 12 }, + { MetadataKey.Build_DefinitionName, "test" }, + }); + + var sbomConfigs = new SbomConfigProvider(new IManifestConfigHandler[] { mockConfigHandler.Object }, + new IMetadataProvider[] { mockMetadataProvider.Object }, + mockLogger.Object, + recorderMock.Object); + + using var manifestStream = new MemoryStream(); + using var manifestWriter = new StreamWriter(manifestStream); + using var sha256Stream = new MemoryStream(); + using var sha256Writer = new StreamWriter(sha256Stream); + + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + if (isDefaultSourceManifestDirPath) + { + configurationMock.SetupGet(c => c.ManifestDirPath) + .Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest") }); + fileSystemMock.Setup(f => f.DirectoryExists(It.Is(d => d == PathUtils.Join("/root", "_manifest")))) + .Returns(deleteExistingManifestDir); + + if (deleteExistingManifestDir) + { + mockOSUtils.Setup(o => o.GetEnvironmentVariable(It.IsAny())).Returns("true"); + fileSystemMock.Setup(f => f.DeleteDir(It.IsAny(), true)).Verifiable(); + } + } + else + { + configurationMock.SetupGet(c => c.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest"), Source = SettingSource.CommandLine }); + } + + configurationMock.SetupGet(c => c.BuildDropPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.Parallelism).Returns(new ConfigurationSetting { Value = 3 }); + configurationMock.SetupGet(c => c.ManifestToolAction).Returns(ManifestToolActions.Generate); + configurationMock.SetupGet(c => c.Verbosity).Returns(new ConfigurationSetting { Value = LogEventLevel.Information }); + configurationMock.SetupGet(c => c.BuildComponentPath).Returns(new ConfigurationSetting { Value = "/root" }); + configurationMock.SetupGet(c => c.FollowSymlinks).Returns(new ConfigurationSetting { Value = true }); + + fileSystemMock + .Setup(f => f.CreateDirectory( + It.Is(d => d == "/root/_manifest"))) + .Returns(new DirectoryInfo("/")); + fileSystemMock + .Setup(f => f.OpenWrite( + It.Is(d => d == "/root/_manifest/manifest.json"))) + .Returns(manifestWriter.BaseStream); + + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "/root"), true)).Returns(new string[] { "child1", "child2", "child3", "_manifest" }); + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "child1"), true)).Returns(new string[] { }); + fileSystemMock.Setup(f => f.GetDirectories(It.Is(c => c == "child2"), true)).Returns(new string[] { "grandchild1", "grandchild2" }); + + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child1"), true)).Returns(new string[] { "/root/child1/file1", "/root/child1/file2" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child2"), true)).Returns(new string[] { "/root/child2/file3", "/root/child2/file4", "/root/child2/file5" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "child3"), true)).Returns(new string[] { "/root/child3/file11", "/root/child3/file12" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "_manifest"), true)).Returns(new string[] { "/root/_manifest/manifest.json", "/root/_manifest/manifest.cat" }); + + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "grandchild1"), true)).Returns(new string[] { "/root/child2/grandchild1/file6", "/root/child2/grandchild1/file10" }); + fileSystemMock.Setup(f => f.GetFilesInDirectory(It.Is(c => c == "grandchild2"), true)).Returns(new string[] { "/root/child2/grandchild1/file7", "/root/child2/grandchild1/file9" }); + + fileSystemMock.Setup(f => f.GetRelativePath(It.IsAny(), It.IsAny())) + .Returns((string r, string p) => PathUtils.GetRelativePath(r, p)); + + fileSystemMock.Setup(f => f.FileExists(It.Is(c => c == jsonFilePath))).Returns(true); + fileSystemMock.Setup(f => f.OpenRead(It.Is(c => c == jsonFilePath))).Returns(TestUtils.GenerateStreamFromString("randomContent")); + fileSystemMock.Setup(f => f.OpenWrite(It.Is(c => c == "/root/_manifest/manifest.json.sha256"))).Returns(sha256Writer.BaseStream); + + hashCodeGeneratorMock.Setup(h => h.GenerateHashes(It.IsAny(), It.IsAny())) + .Returns((string fileName, AlgorithmName[] algos) => + algos.Select(a => + new Checksum + { + ChecksumValue = $"{fileName}hash", + Algorithm = a + }) + .ToArray()); + + var manifestFilterMock = new ManifestFolderFilter(configurationMock.Object, fileSystemMock.Object, mockOSUtils.Object); + manifestFilterMock.Init(); + + var scannedComponents = new List(); + for (int i = 1; i < 4; i++) + { + var scannedComponent = new ScannedComponent + { + Component = new NpmComponent("componentName", $"{i}") + }; + + scannedComponents.Add(scannedComponent); + } + + var scanResult = new ScanResult + { + ResultCode = ProcessingResultCode.Success, + ComponentsFound = scannedComponents + }; + + mockDetector.Setup(o => o.Scan(It.IsAny())).Returns(scanResult); + + var packagesChannel = Channel.CreateUnbounded(); + var errorsChannel = Channel.CreateUnbounded(); + foreach (var component in scannedComponents) + { + await packagesChannel.Writer.WriteAsync(new SBOMPackage { PackageName = component.GetHashCode().ToString() }); + } + + packagesChannel.Writer.Complete(); + errorsChannel.Writer.Complete(); + packageInfoConverterMock + .Setup(p => p.Convert(It.IsAny>())) + .Returns((packagesChannel, errorsChannel)); + + var externalDocumentReferenceChannel = Channel.CreateUnbounded(); + var externalDocumentReferenceErrorsChannel = Channel.CreateUnbounded(); + await externalDocumentReferenceChannel.Writer.WriteAsync(new ExternalDocumentReferenceInfo + { + DocumentNamespace = "namespace", + ExternalDocumentName = "name", + Checksum = new List { new Checksum { Algorithm = AlgorithmName.SHA1, + ChecksumValue = "abc" + } } + }); + externalDocumentReferenceChannel.Writer.Complete(); + externalDocumentReferenceErrorsChannel.Writer.Complete(); + sBOMReaderForExternalDocumentReferenceMock + .Setup(p => p.ParseSBOMFile(It.IsAny>())) + .Returns((externalDocumentReferenceChannel, externalDocumentReferenceErrorsChannel)); + + var directoryTraversingProvider = new DirectoryTraversingFileToJsonProvider + { + ChannelUtils = new ChannelUtils(), + DirectoryWalker = new DirectoryWalker(fileSystemMock.Object, mockLogger.Object, configurationMock.Object), + FileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemUtilsExtensionMock.Object), + mockLogger.Object, + configurationMock.Object, + sbomConfigs, + manifestGeneratorProvider, + new FileTypeUtils()), + FileFilterer = new ManifestFolderFilterer(manifestFilterMock, mockLogger.Object), + FileHashWriter = new FileInfoWriter(manifestGeneratorProvider, + mockLogger.Object, + fileSystemUtilsExtensionMock.Object, + configurationMock.Object), + Log = mockLogger.Object, + Configuration = configurationMock.Object, + InternalSBOMFileInfoDeduplicator = new InternalSBOMFileInfoDeduplicator() + }; + + var fileListBasedProvider = new FileListBasedFileToJsonProvider + { + ChannelUtils = new ChannelUtils(), + FileHasher = new FileHasher(hashCodeGeneratorMock.Object, + new DropValidatorManifestPathConverter(configurationMock.Object, mockOSUtils.Object, fileSystemMock.Object, fileSystemUtilsExtensionMock.Object), + mockLogger.Object, + configurationMock.Object, + sbomConfigs, + manifestGeneratorProvider, + new FileTypeUtils()), + FileFilterer = new ManifestFolderFilterer(manifestFilterMock, mockLogger.Object), + FileHashWriter = new FileInfoWriter(manifestGeneratorProvider, + mockLogger.Object, + fileSystemUtilsExtensionMock.Object, + configurationMock.Object), + + Log = mockLogger.Object, + Configuration = configurationMock.Object, + ListWalker = new FileListEnumerator(fileSystemMock.Object, mockLogger.Object) + }; + + var cgPackagesProvider = new CGScannedPackagesProvider + { + ChannelUtils = new ChannelUtils(), + Log = mockLogger.Object, + Configuration = configurationMock.Object, + PackageInfoConverter = packageInfoConverterMock.Object, + PackageInfoJsonWriter = new PackageInfoJsonWriter(manifestGeneratorProvider, + mockLogger.Object), + PackagesWalker = new PackagesWalker(mockLogger.Object, mockDetector.Object, configurationMock.Object, sbomConfigs), + SBOMConfigs = sbomConfigs + }; + + var externalDocumentReferenceProvider = new ExternalDocumentReferenceProvider + { + ChannelUtils = new ChannelUtils(), + ExternalDocumentReferenceWriter = new ExternalDocumentReferenceWriter(manifestGeneratorProvider, + mockLogger.Object), + SPDXSBOMReaderForExternalDocumentReference = sBOMReaderForExternalDocumentReferenceMock.Object, + Log = mockLogger.Object, + Configuration = configurationMock.Object, + ListWalker = new FileListEnumerator(fileSystemMock.Object, mockLogger.Object) + }; + + var sourcesProvider = new List + { + { fileListBasedProvider }, + { directoryTraversingProvider }, + { cgPackagesProvider }, + {externalDocumentReferenceProvider } + }; + + var fileArrayGenerator = new FileArrayGenerator + { + SourcesProviders = sourcesProvider, + Log = mockLogger.Object, + Configuration = configurationMock.Object, + SBOMConfigs = sbomConfigs, + Recorder = recorderMock.Object + }; + + var packageArrayGenerator = new PackageArrayGenerator + { + ChannelUtils = new ChannelUtils(), + Configuration = configurationMock.Object, + Log = mockLogger.Object, + SBOMConfigs = sbomConfigs, + SourcesProviders = sourcesProvider, + Recorder = recorderMock.Object + }; + + var externalDocumentReferenceGenerator = new ExternalDocumentReferenceGenerator + { + SourcesProviders = sourcesProvider, + Log = mockLogger.Object, + Configuration = configurationMock.Object, + SBOMConfigs = sbomConfigs, + Recorder = recorderMock.Object + }; + + relationshipArrayGenerator + .Setup(r => r.GenerateAsync()) + .ReturnsAsync(await Task.FromResult(new List())); + + var workflow = new SBOMGenerationWorkflow + { + Configuration = configurationMock.Object, + FileSystemUtils = fileSystemMock.Object, + Log = mockLogger.Object, + FileArrayGenerator = fileArrayGenerator, + PackageArrayGenerator = packageArrayGenerator, + RelationshipsArrayGenerator = relationshipArrayGenerator.Object, + ExternalDocumentReferenceGenerator = externalDocumentReferenceGenerator, + SBOMConfigs = sbomConfigs, + OSUtils = mockOSUtils.Object, + Recorder = recorderMock.Object, + }; + + Assert.IsTrue(await workflow.RunAsync()); + + var result = Encoding.UTF8.GetString(manifestStream.ToArray()); + var resultJson = JObject.Parse(result); + + Assert.AreEqual(resultJson["Version"], "1.0.0"); + Assert.AreEqual(resultJson["Build"], 12); + Assert.AreEqual(resultJson["Definition"], "test"); + + var outputs = resultJson["Outputs"]; + JArray sortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); + var expectedJson = JObject.Parse(goodJson); + JArray expectedSortedOutputs = new JArray(outputs.OrderBy(obj => (string)obj["Source"])); + + var packages = resultJson["Packages"]; + Assert.IsTrue(packages.Count() == 4); + + Assert.IsTrue(JToken.DeepEquals(sortedOutputs, expectedSortedOutputs)); + + configurationMock.VerifyAll(); + fileSystemMock.VerifyAll(); + hashCodeGeneratorMock.VerifyAll(); + mockLogger.VerifyAll(); + fileSystemMock.Verify(x => x.FileExists(jsonFilePath), Times.Once); + fileSystemMock.Verify(x => x.OpenWrite($"{jsonFilePath}.sha256"), Times.Once); + } + + [TestMethod] + public async Task ManifestGenerationWorkflowTests_SBOMDirExists_Throws() + { + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + var sbomConfig = new SbomConfig(fileSystemMock.Object) + { + ManifestInfo = Constants.TestManifestInfo, + ManifestJsonDirPath = "/root/_manifest", + ManifestJsonFilePath = "/root/_manifest/manifest.json" + }; + var workflow = new SBOMGenerationWorkflow + { + Configuration = configurationMock.Object, + FileSystemUtils = fileSystemMock.Object, + Log = mockLogger.Object, + SBOMConfigs = new Mock().Object, + Recorder = recorderMock.Object + }; + + Assert.IsFalse(await workflow.RunAsync()); + } + + [TestMethod] + public async Task ManifestGenerationWorkflowTests_SBOMDir_NotDefault_NotDeleted() + { + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + var sbomConfig = new SbomConfig(fileSystemMock.Object) + { + ManifestInfo = Constants.TestManifestInfo, + ManifestJsonDirPath = "/root/_manifest", + ManifestJsonFilePath = "/root/_manifest/manifest.json" + }; + configurationMock.SetupGet(x => x.ManifestDirPath).Returns(new ConfigurationSetting { Value = PathUtils.Join("/root", "_manifest"), Source = SettingSource.CommandLine }); + fileSystemMock.Setup(f => f.DirectoryExists(It.IsAny())).Returns(true); + fileSystemMock.Setup(f => f.DeleteDir(It.IsAny(), true)).Verifiable(); + var fileArrayGeneratorMock = new Mock(); + fileArrayGeneratorMock.Setup(f => f.GenerateAsync()).ReturnsAsync(new List { new FileValidationResult() }); + + var workflow = new SBOMGenerationWorkflow + { + Configuration = configurationMock.Object, + FileSystemUtils = fileSystemMock.Object, + Log = mockLogger.Object, + SBOMConfigs = new Mock().Object, + Recorder = recorderMock.Object, + FileArrayGenerator = fileArrayGeneratorMock.Object + }; + + var result = await workflow.RunAsync(); + + fileSystemMock.Verify(f => f.DeleteDir(It.IsAny(), true), Times.Never); + Assert.IsFalse(result); + } + + private const string goodJson = "{\"Outputs\":[{\"Source\":\"/child1/file2\",\"AzureArtifactsHash\":" + + "\"/root/child1/file2hash\",\"Sha256Hash\":\"/root/child1/file2hash\"},{\"Source\":\"/child1/file1\"," + + "\"AzureArtifactsHash\":\"/root/child1/file1hash\",\"Sha256Hash\":\"/root/child1/file1hash\"},{\"Source\":" + + "\"/child2/file3\",\"AzureArtifactsHash\":\"/root/child2/file3hash\",\"Sha256Hash\":\"/root/child2/file3hash\"}" + + ",{\"Source\":\"/child2/file4\",\"AzureArtifactsHash\":\"/root/child2/file4hash\",\"Sha256Hash\":\"/root/child2" + + "/file4hash\"},{\"Source\":\"/child2/grandchild1/file6\",\"AzureArtifactsHash\":\"/root/child2/grandchild1/" + + "file6hash\",\"Sha256Hash\":\"/root/child2/grandchild1/file6hash\"},{\"Source\":\"/child2/grandchild1/file10" + + "\",\"AzureArtifactsHash\":\"/root/child2/grandchild1/file10hash\",\"Sha256Hash\":\"/root/child2/grandchild1" + + "/file10hash\"},{\"Source\":\"/child2/grandchild1/file9\",\"AzureArtifactsHash\":\"/root/child2/grandchild1" + + "/file9hash\",\"Sha256Hash\":\"/root/child2/grandchild1/file9hash\"},{\"Source\":\"/child3/file11\",\"Azure" + + "ArtifactsHash\":\"/root/child3/file11hash\",\"Sha256Hash\":\"/root/child3/file11hash\"},{\"Source\":\"/chi" + + "ld2/file5\",\"AzureArtifactsHash\":\"/root/child2/file5hash\",\"Sha256Hash\":\"/root/child2/file5hash\"},{" + + "\"Source\":\"/child2/grandchild1/file7\",\"AzureArtifactsHash\":\"/root/child2/grandchild1/file7hash\",\"S" + + "ha256Hash\":\"/root/child2/grandchild1/file7hash\"},{\"Source\":\"/child3/file12\",\"AzureArtifactsHash\":" + + "\"/root/child3/file12hash\",\"Sha256Hash\":\"/root/child3/file12hash\"}]}"; + } +} diff --git a/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/IdentityUtilsTests.cs b/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/IdentityUtilsTests.cs index d00140c6..16fec072 100644 --- a/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/IdentityUtilsTests.cs +++ b/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/IdentityUtilsTests.cs @@ -1,4 +1,7 @@ -using Microsoft.Sbom.Extensions; +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + +using Microsoft.Sbom.Extensions; using Microsoft.Sbom.Extensions.Entities; using Microsoft.VisualStudio.TestTools.UnitTesting; using Moq; diff --git a/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/SPDXExtensionsTest.cs b/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/SPDXExtensionsTest.cs index 002bca5e..eaa01fa9 100644 --- a/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/SPDXExtensionsTest.cs +++ b/test/Microsoft.Sbom.SPDX22SBOMParser.Tests/Utils/SPDXExtensionsTest.cs @@ -1,3 +1,6 @@ +// Copyright (c) Microsoft. All rights reserved. +// Licensed under the MIT license. See LICENSE file in the project root for full license information. + using Microsoft.Sbom.Contracts; using Microsoft.Sbom.Contracts.Enums; using Microsoft.SPDX22SBOMParser.Entities; @@ -41,7 +44,7 @@ public void AddPackageUrlsTest_Success() spdxPackage.AddPackageUrls(packageInfo); var externalRef = spdxPackage.ExternalReferences.First(); Assert.AreEqual(ReferenceCategory.PACKAGE_MANAGER, externalRef.ReferenceCategory); - Assert.AreEqual(ExternalRepositoryType.purl, externalRef.Type); + Assert.AreEqual(ExternalRepositoryType.Purl, externalRef.Type); Assert.AreEqual(PackageUrl, externalRef.Locator); } @@ -81,7 +84,7 @@ public void AddPackageUrlsTest_WithEncoding_Success(string inputUrl, string expe var externalRef = spdxPackage.ExternalReferences.First(); Assert.AreEqual(ReferenceCategory.PACKAGE_MANAGER, externalRef.ReferenceCategory); - Assert.AreEqual(ExternalRepositoryType.purl, externalRef.Type); + Assert.AreEqual(ExternalRepositoryType.Purl, externalRef.Type); Assert.AreEqual(expectedUrl, externalRef.Locator); }