FEATURE: Add --normalize-for-ghas argument to the rewrite command

src/ReleaseHistory.md

@@ -12,6 +12,7 @@
 * BUGFIX: Eliminate `IOException` and `DirectoryNotFoundException` exceptions thrown by `merge` command when splitting by rule (due to invalid file characters in rule ids). [#2513](https://github.com/microsoft/sarif-sdk/pull/2513)
 * BUGFIX: Fix classes inside NotYetAutoGenerated folder missing `virtual` keyword for public methods and properties, by regenerate and manually sync the changes. [#2537](https://github.com/microsoft/sarif-sdk/pull/2537)
 * FEATURE: Allow external set of `MaxFileSizeInKilobytes`, which will allow SDK users to change the value. (Default value is 1024) [#2578](https://github.com/microsoft/sarif-sdk/pull/2578)
+* FEATURE: Add `--normalize-for-github` argument to the `rewrite` command to get GitHub supported SARIF format. [#2581](https://github.com/microsoft/sarif-sdk/pull/2581)
 
 ## **v3.1.0** [Sdk](https://www.nuget.org/packages/Sarif.Sdk/3.1.0) | [Driver](https://www.nuget.org/packages/Sarif.Driver/3.1.0) | [Converters](https://www.nuget.org/packages/Sarif.Converters/3.1.0) | [Multitool](https://www.nuget.org/packages/Sarif.Multitool/3.1.0) | [Multitool Library](https://www.nuget.org/packages/Sarif.Multitool.Library/3.1.0)
 

src/Sarif.Multitool.Library/RewriteCommand.cs

@@ -5,6 +5,7 @@
 using System.Collections.Generic;
 using System.Diagnostics;
 using System.IO;
+using System.Linq;
 
 using Microsoft.CodeAnalysis.Sarif.Driver;
 using Microsoft.CodeAnalysis.Sarif.Readers;
@@ -64,6 +65,25 @@ public int Run(RewriteOptions options)
                     reformattedLog = new SortingVisitor().VisitSarifLog(reformattedLog);
                 }
 
+                if (options.NormalizeForGitHub)
+                {
+                    if ((reformattedLog.Runs?.Any(r => r.Artifacts?.Any(a => a.Location?.Uri?.IsAbsoluteUri == true) == true) == true) ||
+                        (reformattedLog.Runs?.Any(r => r.Results?.Any(s => s.Locations?.Any(l => l.PhysicalLocation?.ArtifactLocation?.Uri?.IsAbsoluteUri == true) == true) == true) == true))
+                    {
+                        if (options.BasePath == null || options.BasePathToken == null)
+                        {
+                            throw new ArgumentException("The input SARIF file contains absolute Uri which will not work with GitHub. Please also provide `--base-path-value` and `--base-path-token` to convert them to relative Uri.");
+                        }
+                        else
+                        {
+                            var visitor = new RebaseUriVisitor(options.BasePathToken, new Uri(options.BasePath), options.RebaseRelativeUris);
+                            reformattedLog = visitor.VisitSarifLog(reformattedLog);
+                        }
+                    }
+
+                    reformattedLog = new GitHubIngestionVisitor().VisitSarifLog(reformattedLog);
+                }
+
                 if (options.SarifOutputVersion == SarifVersion.OneZeroZero)
                 {
                     var visitor = new SarifCurrentToVersionOneVisitor();

src/Sarif.Multitool.Library/RewriteOptions.cs

@@ -16,5 +16,33 @@ public class RewriteOptions : SingleFileOptionsBase
             Default = false,
             HelpText = "Sort results in the final output file.")]
         public bool SortResults { get; set; }
+
+        [Option(
+            "normalize-for-github",
+            HelpText = "Normalize SARIF to conform to GitHub Advanced Security code scanning ingestion requirements.")]
+        public bool NormalizeForGitHub { get; set; }
+
+        [Option(
+            'b',
+            "base-path-value",
+            Required = false,
+            HelpText = "Base path value to use while rebasing all paths.  E.x. 'C:\\bld\\1234\\bin\\'"
+            )]
+        public string BasePath { get; set; }
+
+        [Option(
+            't',
+            "base-path-token",
+            Required = false,
+            HelpText = "Variable to use for the base path token (e.x. 'SRCROOT')"
+            )]
+        public string BasePathToken { get; set; }
+
+        [Option(
+            "rebase-relative-uris",
+            Default = false,
+            HelpText = "All relative uris will be rebased to the base-path-value if true. Default is false."
+            )]
+        public bool RebaseRelativeUris { get; set; }
     }
 }

src/Sarif/Visitors/GitHubIngestionVisitor.cs

@@ -2,6 +2,7 @@
 // Licensed under the MIT license. See LICENSE file in the project root for full license information.
 
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 
 namespace Microsoft.CodeAnalysis.Sarif.Visitors
@@ -12,42 +13,19 @@ public class GitHubIngestionVisitor : SarifRewritingVisitor
         // Internal static rather than private const to allow a unit test with a practical limit.
         internal static int s_MaxResults = 5000;
 
+        private Run run;
+        private int ruleIndex = -1;
         private IList<Artifact> artifacts;
         private IList<ThreadFlowLocation> threadFlowLocations;
 
         public override Run VisitRun(Run node)
         {
+            this.run = node;
             this.artifacts = node.Artifacts;
             this.threadFlowLocations = node.ThreadFlowLocations;
 
             if (node.Results != null)
             {
-                int errorsCount = 0;
-                foreach (Result result in node.Results)
-                {
-                    if (result.Level == FailureLevel.Error)
-                    {
-                        errorsCount++;
-                    }
-                }
-
-                if (errorsCount != node.Results.Count)
-                {
-                    var errors = new List<Result>();
-
-                    foreach (Result result in node.Results)
-                    {
-                        if (result.Level == FailureLevel.Error)
-                        {
-                            errors.Add(result);
-                        }
-
-                        if (errors.Count == s_MaxResults) { break; }
-                    }
-
-                    node.Results = errors;
-                }
-
                 if (node.Results.Count > s_MaxResults)
                 {
                     node.Results = node.Results.Take(s_MaxResults).ToList();
@@ -89,10 +67,7 @@ public override ThreadFlowLocation VisitThreadFlowLocation(ThreadFlowLocation no
                 {
                     // Make sure there's a place to put the message. threadFlowLocation.Location
                     // is not required, so the shared object might not have had one.
-                    if (node.Location == null)
-                    {
-                        node.Location = new Location();
-                    }
+                    node.Location ??= new Location();
 
                     node.Location.Message = message;
                 }
@@ -144,6 +119,8 @@ public override ArtifactLocation VisitArtifactLocation(ArtifactLocation node)
 
         public override Result VisitResult(Result node)
         {
+            this.ruleIndex = node.RuleIndex;
+
             if (node.Fingerprints != null)
             {
                 // GitHub appears to require that fingerprints be emitted to the
@@ -161,6 +138,32 @@ public override Result VisitResult(Result node)
             return base.VisitResult(node);
         }
 
+        public override Message VisitMessage(Message node)
+        {
+            if (node.Text == null)
+            {
+                ReportingDescriptor rule = this.ruleIndex != -1 ? this.run.Tool.Driver.Rules[this.ruleIndex] : null;
+
+                if (rule != null &&
+                    rule.MessageStrings != null &&
+                    rule.MessageStrings.TryGetValue(node.Id, out MultiformatMessageString formatString))
+                {
+                    node.Text = node.Arguments?.Count > 0
+                        ? rule.Format(node.Id, node.Arguments)
+                        : formatString?.Text;
+                }
+
+                if (node.Text == null &&
+                    this.run.Tool.Driver.GlobalMessageStrings?.TryGetValue(node.Id, out formatString) == true)
+                {
+                    node.Text = node.Arguments?.Count > 0
+                        ? string.Format(CultureInfo.CurrentCulture, formatString.Text, node.Arguments.ToArray())
+                        : formatString?.Text;
+                }
+            }
+            return base.VisitMessage(node);
+        }
+
         // Merge properties from a source to a target, preferring the existing properties
         // on the target if there are any duplicates.
         private static void MergeProperties(PropertyBagHolder target, PropertyBagHolder source)
@@ -169,10 +172,7 @@ private static void MergeProperties(PropertyBagHolder target, PropertyBagHolder
             if (source.Properties != null)
             {
                 // If so, make sure there's someplace to put them.
-                if (target.Properties == null)
-                {
-                    target.Properties = new Dictionary<string, SerializedPropertyInfo>();
-                }
+                target.Properties ??= new Dictionary<string, SerializedPropertyInfo>();
 
                 target.Properties = target.Properties.MergePreferFirst(source.Properties);
             }

src/Test.UnitTests.Sarif.Multitool.Library/RewriteCommandTests.cs

@@ -17,11 +17,78 @@
 using Newtonsoft.Json.Linq;
 
 using Xunit;
+using Xunit.Abstractions;
 
 namespace Microsoft.CodeAnalysis.Sarif.Multitool
 {
-    public class RewriteCommandTests
+    public class RewriteCommandTests : FileDiffingUnitTests
     {
+        private RewriteOptions options;
+
+        public RewriteCommandTests(ITestOutputHelper outputHelper) : base(outputHelper) { }
+
+        private static RewriteOptions CreateDefaultOptions()
+        {
+            return new RewriteOptions
+            {
+                BasePath = @"C:\vs\src\2\s\",
+                BasePathToken = "SRCROOT",
+                Inline = true,
+                SarifOutputVersion = SarifVersion.Current,
+                PrettyPrint = true,
+                NormalizeForGitHub = true,
+            };
+        }
+
+        protected override string ConstructTestOutputFromInputResource(string testFilePath, object parameter)
+        {
+            return RunRewriteCommand(testFilePath, this.options);
+        }
+
+        [Fact]
+        public void RebaseUriCommand_RebaseRunWithArtifacts()
+        {
+            string testFilePath = "RunWithArtifacts.sarif";
+
+            this.options = CreateDefaultOptions();
+
+            RunTest(testFilePath);
+        }
+
+        private string RunRewriteCommand(string testFilePath, RewriteOptions options)
+        {
+            string inputSarifLog = GetInputSarifTextFromResource(testFilePath);
+
+            string logFilePath = Path.Combine(Directory.GetCurrentDirectory(), "mylog.sarif");
+            StringBuilder transformedContents = new StringBuilder();
+
+            options.InputFilePath = logFilePath;
+            options.OutputFilePath = null;
+
+            Mock<IFileSystem> mockFileSystem = ArrangeMockFileSystem(inputSarifLog, logFilePath, transformedContents);
+
+            var RewriteCommand = new RewriteCommand(mockFileSystem.Object);
+
+            int returnCode = RewriteCommand.Run(options);
+            string actualOutput = transformedContents.ToString();
+
+            returnCode.Should().Be(0);
+
+            return actualOutput;
+        }
+
+        private static Mock<IFileSystem> ArrangeMockFileSystem(string sarifLog, string logFilePath, StringBuilder transformedContents)
+        {
+            var mockFileSystem = new Mock<IFileSystem>();
+            mockFileSystem.Setup(x => x.FileReadAllText(logFilePath)).Returns(sarifLog);
+            mockFileSystem.Setup(x => x.FileOpenRead(logFilePath)).Returns(() => new MemoryStream(Encoding.UTF8.GetBytes(sarifLog)));
+            mockFileSystem.Setup(x => x.FileCreate(logFilePath)).Returns(() => new MemoryStreamToStringBuilder(transformedContents));
+            mockFileSystem.Setup(x => x.FileWriteAllText(logFilePath, It.IsAny<string>())).Callback<string, string>((path, contents) => { transformedContents.Append(contents); });
+            mockFileSystem.Setup(x => x.DirectoryExists(It.IsAny<string>())).Returns(true);
+            mockFileSystem.Setup(x => x.DirectoryGetFiles(It.IsAny<string>(), It.IsAny<string>())).Returns(new string[] { logFilePath });
+            return mockFileSystem;
+        }
+
         public const string MinimalPrereleaseV2Text =
     @"{
   ""$schema"": ""http://json.schemastore.org/sarif-2.0.0"",

src/Test.UnitTests.Sarif.Multitool.Library/Test.UnitTests.Sarif.Multitool.Library.csproj

@@ -20,16 +20,6 @@
     <None Remove="TestData\ExportRuleDocumentationCommand\**" />
   </ItemGroup>
 
-  <ItemGroup>
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\MarkdownFullDescription.md" />
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\MarkdownShortDescription.md" />
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\NoDescription.md" />
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\NonStandardMessageStringKey.md" />
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\StandardMessageStringKey.md" />
-    <None Remove="TestData\ExportValidationRulesMetadataCommand\ExpectedOutputs\TextShortDescription.md" />
-    <None Remove="TestData\MergeCommand\ExpectedOutputs\DuplicatedResults.sarif" />
-  </ItemGroup>
-
   <ItemGroup>
     <Content Include="xunit.runner.json">
       <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
@@ -55,6 +45,8 @@
     <EmbeddedResource Include="TestData\PageCommand\elfie-arriba.sarif" />
     <EmbeddedResource Include="TestData\QueryCommand\elfie-arriba.sarif" />
     <EmbeddedResource Include="TestData\QueryCommand\WithProperties.sarif" />
+    <EmbeddedResource Include="TestData\RewriteCommand\ExpectedOutputs\RunWithArtifacts.sarif" />
+    <EmbeddedResource Include="TestData\RewriteCommand\Inputs\RunWithArtifacts.sarif" />
     <EmbeddedResource Include="TestData\RebaseUriCommand\ExpectedOutputs\RunWithArtifacts.sarif" />
     <EmbeddedResource Include="TestData\RebaseUriCommand\Inputs\RunWithArtifacts.sarif" />
     <EmbeddedResource Include="TestData\ValidateCommand\Configuration.json" />