diff --git a/src/Sarif.Driver/DriverExtensionMethods.cs b/src/Sarif.Driver/DriverExtensionMethods.cs index 342ba0deb..3fc2cf4d9 100644 --- a/src/Sarif.Driver/DriverExtensionMethods.cs +++ b/src/Sarif.Driver/DriverExtensionMethods.cs @@ -160,12 +160,20 @@ public static bool ValidateOutputOptions(this MultipleFilesOptionsBase options) return valid; } - public static bool ValidateOutputOptions(this AnalyzeOptionsBase options) + public static bool ValidateOutputOptions(this AnalyzeOptionsBase options, IAnalysisContext context) { bool valid = true; valid &= !(options.Quiet && string.IsNullOrWhiteSpace(options.OutputFilePath)); + // baseline process now depends on output file + valid &= !(string.IsNullOrWhiteSpace(options.OutputFilePath) && !string.IsNullOrWhiteSpace(options.BaselineSarifFile)); + + if (!valid) + { + context.RuntimeErrors |= RuntimeConditions.InvalidCommandLineOption; + } + return valid; } } diff --git a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs index e319116fd..a15767542 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeCommandBase.cs @@ -108,6 +108,20 @@ public override int Run(TOptions options) bool succeeded = (RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None; + if (succeeded) + { + try + { + ProcessBaseline(_rootContext, options, FileSystem); + } + catch (Exception ex) + { + RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline; + ExecutionException = ex; + return FAILURE; + } + } + if (options.RichReturnCode) { return (int)RuntimeErrors; @@ -172,9 +186,10 @@ protected virtual void ValidateOptions(TContext context, TOptions analyzeOptions succeeded &= ValidateFile(context, analyzeOptions.OutputFilePath, shouldExist: null); succeeded &= ValidateFile(context, analyzeOptions.ConfigurationFilePath, shouldExist: true); succeeded &= ValidateFiles(context, analyzeOptions.PluginFilePaths, shouldExist: true); + succeeded &= ValidateFile(context, analyzeOptions.BaselineSarifFile, shouldExist: true); succeeded &= ValidateInvocationPropertiesToLog(context, analyzeOptions.InvocationPropertiesToLog); succeeded &= ValidateOutputFileCanBeCreated(context, analyzeOptions.OutputFilePath, analyzeOptions.Force); - succeeded &= analyzeOptions.ValidateOutputOptions(); + succeeded &= analyzeOptions.ValidateOutputOptions(context); if (!succeeded) { diff --git a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs index 8f285b50e..9567ae6d2 100644 --- a/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs +++ b/src/Sarif.Driver/Sdk/AnalyzeOptionsBase.cs @@ -104,5 +104,10 @@ public abstract class AnalyzeOptionsBase : CommonOptionsBase Default = new ResultKind[] { ResultKind.Fail }, HelpText = "A semicolon delimited list to filter output to one or more result kinds. Valid values: Fail (for literal scan results), Pass, Review, Open, NotApplicable and Informational.")] public IEnumerable Kind { get; set; } + + [Option( + "baseline", + HelpText = "A Sarif file to be used as baseline")] + public string BaselineSarifFile { get; set; } } } diff --git a/src/Sarif.Driver/Sdk/ExitReason.cs b/src/Sarif.Driver/Sdk/ExitReason.cs index 3cfd9fd5e..ec11b377f 100644 --- a/src/Sarif.Driver/Sdk/ExitReason.cs +++ b/src/Sarif.Driver/Sdk/ExitReason.cs @@ -12,6 +12,7 @@ public enum ExitReason UnhandledExceptionInEngine, NoRulesLoaded, NoValidAnalysisTargets, - InvalidCommandLineOption + InvalidCommandLineOption, + ExceptionProcessingBaseline } } diff --git a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs index af1ac9f6d..aa7ac7c70 100644 --- a/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs +++ b/src/Sarif.Driver/Sdk/MultithreadedAnalyzeCommandBase.cs @@ -101,6 +101,20 @@ public override int Run(TOptions options) bool succeeded = (RuntimeErrors & ~RuntimeConditions.Nonfatal) == RuntimeConditions.None; + if (succeeded) + { + try + { + ProcessBaseline(_rootContext, options, FileSystem); + } + catch (Exception ex) + { + RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline; + ExecutionException = ex; + return FAILURE; + } + } + if (options.RichReturnCode) { return (int)RuntimeErrors; @@ -451,8 +465,10 @@ protected virtual void ValidateOptions(TOptions options, TContext context) succeeded &= ValidateFile(context, options.OutputFilePath, shouldExist: null); succeeded &= ValidateFile(context, options.ConfigurationFilePath, shouldExist: true); succeeded &= ValidateFiles(context, options.PluginFilePaths, shouldExist: true); + succeeded &= ValidateFile(context, options.BaselineSarifFile, shouldExist: true); succeeded &= ValidateInvocationPropertiesToLog(context, options.InvocationPropertiesToLog); succeeded &= ValidateOutputFileCanBeCreated(context, options.OutputFilePath, options.Force); + succeeded &= options.ValidateOutputOptions(context); if (!succeeded) { diff --git a/src/Sarif.Driver/Sdk/PluginDriverCommand.cs b/src/Sarif.Driver/Sdk/PluginDriverCommand.cs index b83e10a64..2e6236ed7 100644 --- a/src/Sarif.Driver/Sdk/PluginDriverCommand.cs +++ b/src/Sarif.Driver/Sdk/PluginDriverCommand.cs @@ -3,8 +3,14 @@ using System; using System.Collections.Generic; +using System.IO; +using System.Linq; using System.Reflection; +using Microsoft.CodeAnalysis.Sarif.Baseline.ResultMatching; + +using Newtonsoft.Json; + namespace Microsoft.CodeAnalysis.Sarif.Driver { public abstract class PluginDriverCommand : DriverCommand @@ -30,5 +36,57 @@ public IEnumerable RetrievePluginAssemblies(IEnumerable defa return assemblies; } + + protected virtual void ProcessBaseline(IAnalysisContext context, T driverOptions, IFileSystem fileSystem) + { + if (!(driverOptions is AnalyzeOptionsBase options)) + { + return; + } + + if (string.IsNullOrEmpty(options.BaselineSarifFile) || string.IsNullOrEmpty(options.OutputFilePath)) + { + return; + } + + var serializer = new JsonSerializer + { + Formatting = options.PrettyPrint || (!options.PrettyPrint && !options.Minify) ? + Formatting.Indented : + Formatting.None + }; + + var baselineFile = SarifLog.Load(options.BaselineSarifFile); + var currentSarifLog = SarifLog.Load(options.OutputFilePath); + SarifLog baseline; + try + { + ISarifLogMatcher matcher = ResultMatchingBaselinerFactory.GetDefaultResultMatchingBaseliner(); + baseline = matcher.Match(new SarifLog[] { baselineFile }, new SarifLog[] { currentSarifLog }).First(); + } + catch (Exception ex) + { + throw new ExitApplicationException(DriverResources.MSG_UnexpectedApplicationExit, ex) + { + ExitReason = ExitReason.ExceptionProcessingBaseline + }; + } + + try + { + string targetFile = options.Inline ? options.BaselineSarifFile : options.OutputFilePath; + using (var writer = new JsonTextWriter(new StreamWriter(fileSystem.FileCreate(targetFile)))) + { + serializer.Serialize(writer, baseline); + } + } + catch (Exception ex) + { + throw new ExitApplicationException(DriverResources.MSG_UnexpectedApplicationExit, ex) + { + ExitReason = ExitReason.ExceptionWritingToLogFile + }; + } + } } } diff --git a/src/Sarif/RuntimeConditions.cs b/src/Sarif/RuntimeConditions.cs index b705f03df..5b44dd402 100644 --- a/src/Sarif/RuntimeConditions.cs +++ b/src/Sarif/RuntimeConditions.cs @@ -48,6 +48,7 @@ public enum RuntimeConditions : uint ExceptionAccessingFile = 0x4000, ExceptionInstantiatingSkimmers = 0x8000, OutputFileAlreadyExists = 0x10000, + ExceptionProcessingBaseline = 0x20000, // Non-fatal conditions UnassignedNonfatal = 0x01F00000, diff --git a/src/Test.UnitTests.Sarif.Driver/DriverExtensionMethodsTests.cs b/src/Test.UnitTests.Sarif.Driver/DriverExtensionMethodsTests.cs index 88cbd1eb0..88e9e5022 100644 --- a/src/Test.UnitTests.Sarif.Driver/DriverExtensionMethodsTests.cs +++ b/src/Test.UnitTests.Sarif.Driver/DriverExtensionMethodsTests.cs @@ -243,26 +243,27 @@ public void ValidatingMultipleFilesOutputOptions_ProducesExpectedResults() public void ValidateAnalyzeOutputOptions_ProducesExpectedResults() { AnalyzeOptionsBase analyzeOptionsBase = new TestAnalyzeOptions(); + TestAnalysisContext context = new TestAnalysisContext(); // quiet false, output empty analyzeOptionsBase.Quiet = false; analyzeOptionsBase.OutputFilePath = null; - Assert.True(analyzeOptionsBase.ValidateOutputOptions()); + Assert.True(analyzeOptionsBase.ValidateOutputOptions(context)); // quiet false, output non-empty analyzeOptionsBase.Quiet = false; analyzeOptionsBase.OutputFilePath = "doodle"; - Assert.True(analyzeOptionsBase.ValidateOutputOptions()); + Assert.True(analyzeOptionsBase.ValidateOutputOptions(context)); // quiet true, output empty analyzeOptionsBase.Quiet = true; analyzeOptionsBase.OutputFilePath = null; - Assert.False(analyzeOptionsBase.ValidateOutputOptions()); + Assert.False(analyzeOptionsBase.ValidateOutputOptions(context)); // quiet true, output non-empty analyzeOptionsBase.Quiet = true; analyzeOptionsBase.OutputFilePath = "doodle"; - Assert.True(analyzeOptionsBase.ValidateOutputOptions()); + Assert.True(analyzeOptionsBase.ValidateOutputOptions(context)); } private class ValidateOutputFormatOptionsTestCase diff --git a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs index 6d0689ac7..c40033120 100644 --- a/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs +++ b/src/Test.UnitTests.Sarif.Driver/Sdk/AnalyzeCommandBaseTests.cs @@ -290,6 +290,22 @@ public void ExceptionRaisedInvokingAnalyze() ); } + [Fact] + public void ExceptionRaisedProcessingBaseline() + { + var options = new TestAnalyzeOptions() + { + TestRuleBehaviors = TestRuleBehaviors.RaiseExceptionProcessingBaseline, + TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() }, + }; + + ExceptionTestHelper( + RuntimeConditions.ExceptionProcessingBaseline, + expectedExitReason: ExitReason.ExceptionProcessingBaseline, + analyzeOptions: options + ); + } + [Fact] public void ExceptionRaisedInvokingAnalyze_PersistInnerException() @@ -452,6 +468,47 @@ public void MissingOutputFile() } } + [Fact] + public void MissingBaselineFile() + { + string outputFilePath = Path.GetTempFileName() + ".sarif"; + string baselineFilePath = Path.GetTempFileName() + ".sarif"; + + var options = new TestAnalyzeOptions() + { + TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() }, + OutputFilePath = outputFilePath, + BaselineSarifFile = baselineFilePath + }; + + ExceptionTestHelper( + RuntimeConditions.MissingFile, + expectedExitReason: ExitReason.InvalidCommandLineOption, + analyzeOptions: options); + } + + [Fact] + public void BaselineWithoutOutputFile() + { + string path = Path.GetTempFileName() + ".sarif"; + + using (FileStream stream = File.Create(path, 1, FileOptions.DeleteOnClose)) + { + var options = new TestAnalyzeOptions() + { + TargetFileSpecifiers = new string[] { GetThisTestAssemblyFilePath() }, + Quiet = true, + OutputFilePath = null, + BaselineSarifFile = path + }; + + ExceptionTestHelper( + RuntimeConditions.InvalidCommandLineOption, + expectedExitReason: ExitReason.InvalidCommandLineOption, + analyzeOptions: options); + } + } + [Fact] public void AnalyzeCommandBase_ReportsErrorOnInvalidInvocationPropertyName() { diff --git a/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs b/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs index 1804b7e6c..f500a8078 100644 --- a/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs +++ b/src/Test.UnitTests.Sarif.Driver/TestAnalyzeCommand.cs @@ -59,5 +59,16 @@ protected override void ValidateOptions(TestAnalysisContext context, TestAnalyze base.ValidateOptions(context, options); } + + protected override void ProcessBaseline(IAnalysisContext context, TestAnalyzeOptions options, IFileSystem fileSystem) + { + if (context.Policy.GetProperty(TestRule.Behaviors).HasFlag(TestRuleBehaviors.RaiseExceptionProcessingBaseline)) + { + context.RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline; + ThrowExitApplicationException((TestAnalysisContext)context, ExitReason.ExceptionProcessingBaseline); + } + + base.ProcessBaseline(context, options, fileSystem); + } } } diff --git a/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs b/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs index 2bb22081d..2ea8bd7f4 100644 --- a/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs +++ b/src/Test.UnitTests.Sarif.Driver/TestMultithreadedAnalyzeCommand.cs @@ -70,5 +70,16 @@ protected override TestAnalysisContext DetermineApplicabilityAndAnalyze(TestAnal return base.DetermineApplicabilityAndAnalyze(context, skimmers, disabledSkimmers); } + + protected override void ProcessBaseline(IAnalysisContext context, TestAnalyzeOptions options, IFileSystem fileSystem) + { + if (context.Policy.GetProperty(TestRule.Behaviors).HasFlag(TestRuleBehaviors.RaiseExceptionProcessingBaseline)) + { + context.RuntimeErrors |= RuntimeConditions.ExceptionProcessingBaseline; + ThrowExitApplicationException((TestAnalysisContext)context, ExitReason.ExceptionProcessingBaseline); + } + + base.ProcessBaseline(context, options, fileSystem); + } } } diff --git a/src/Test.Utilities.Sarif/TestRuleBehaviors.cs b/src/Test.Utilities.Sarif/TestRuleBehaviors.cs index 49f1a10c4..dd2e50d6d 100644 --- a/src/Test.Utilities.Sarif/TestRuleBehaviors.cs +++ b/src/Test.Utilities.Sarif/TestRuleBehaviors.cs @@ -38,6 +38,8 @@ public enum TestRuleBehaviors RegardAnalysisTargetAsInvalid = 0x2000, // Assume one or more options are invalid - RegardOptionsAsInvalid = 0x4000 + RegardOptionsAsInvalid = 0x4000, + + RaiseExceptionProcessingBaseline = 0x8000 } }