[StaticAnalysis] CI ExampleAnalysis

.azure-pipelines/util/analyze-steps.yml

@@ -32,6 +32,9 @@ steps:
 - pwsh: 'Install-Module platyPS -Force -Confirm:$false -Scope CurrentUser'
   displayName: 'Install platyPS'
 
+- pwsh: 'Install-Module PSScriptAnalyzer'
+  displayName: 'Install PSScriptAnalyzer'
+
 - task: DotNetCoreCLI@2
   displayName: 'Generate Help'
   inputs:

.ci-config.json

@@ -87,7 +87,7 @@
         "src/{ModuleName}/**/*.md$"
       ],
       "phases": [
-        "build:module",
+        "build:dependent-module",
         "help:module"
       ]
     },

build.proj

@@ -118,7 +118,7 @@
     <Exec Command="dotnet publish $(RepoTools)BuildPackagesTask/Microsoft.Azure.Build.Tasks/Microsoft.Azure.Build.Tasks.csproj -c $(Configuration)" />
 
     <!-- Get all of the files changed in the given pull request -->
-    <FilesChangedTask RepositoryOwner="Azure" RepositoryName="azure-powershell" PullRequestNumber="$(PullRequestNumber)" TargetModule="$(TargetModule)">
+    <FilesChangedTask RepositoryOwner="Azure" RepositoryName="azure-powershell" PullRequestNumber="$(PullRequestNumber)" TargetModule="$(TargetModule)" OutputFile="$(RepoArtifacts)/FilesChanged.txt">
       <Output TaskParameter="FilesChanged" ItemName="FilesChanged" />
     </FilesChangedTask>
 
@@ -263,7 +263,12 @@
     <Exec Command="dotnet $(RepoArtifacts)StaticAnalysis/StaticAnalysis.Netcore.dll -p $(RepoArtifacts)$(Configuration) -r $(StaticAnalysisOutputDirectory) --analyzers help -u -m '%(FilterTaskResult.help)'" />
   </Target>
 
-  <Target Name="StaticAnalysis" DependsOnTargets="StaticAnalysisBreakingChange;StaticAnalysisDependency;StaticAnalysisSignature;StaticAnalysisHelp">
+  <Target Name="StaticAnalysisExample" Condition="'$(RunStaticAnalysis)' == 'true'" DependsOnTargets="Build" AfterTargets="StaticAnalysisHelp">
+    <Message Importance="high" Text="Running static analysis for example..." />
+    <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/StaticAnalysis/ExampleAnalyzer/Measure-MarkdownOrScript.ps1 -MarkdownPaths $(RepoArtifacts)/FilesChanged.txt -RulePaths $(RepoTools)/StaticAnalysis/ExampleAnalyzer/AnalyzeRules//*.psm1 -Recurse -AnalyzeScriptsInFile -OutputScriptsInFile -OutputResultsByModule &quot;"/>
+  </Target>
+
+  <Target Name="StaticAnalysis" DependsOnTargets="StaticAnalysisBreakingChange;StaticAnalysisDependency;StaticAnalysisSignature;StaticAnalysisHelp;StaticAnalysisExample">
     <Message Importance="high" Text="Running static analysis..." />
 
     <Exec Command="$(PowerShellCoreCommandPrefix) &quot;. $(RepoTools)/CheckAssemblies.ps1 -BuildConfig $(Configuration) &quot;" />

documentation/Debugging-StaticAnalysis-Errors.md

@@ -8,6 +8,7 @@ Our StaticAnalysis tools help us ensure our modules follow PowerShell guidelines
     - [Breaking Changes](#breaking-changes)
     - [Signature Issues](#signature-issues)
     - [Help Issues](#help-issues)
+    - [Example Issues](#example-issues)
 
 ## How to know if you have a StaticAnalysis Error
 If your build is failing, click on the Jenkins job inside the PR (marked as "Default" within checks).  Then check the Console Output within the Jenkins job.  If you have this error, then you have failed StaticAnalysis:
@@ -52,3 +53,14 @@ Signature issues occur when your cmdlets do not follow PowerShell standards.  Pl
 
 ### Help Issues
 Most help issues that cause StaticAnalysis to fail occur when help has not been added for a particular cmdlet.  If you have not generated help for your new cmdlets, please follow the instructions [here](https://github.com/Azure/azure-powershell/blob/main/documentation/development-docs/help-generation.md). If this is not the issue, follow the steps listed under "Remediation" for each violation listed in HelpIssues.csv.
+
+### Example Issues
+Example Issues occur when your changed markdown files a `help` folder (_e.g.,_ `src/Accounts/Accounts/help`) do not follow the script rules. Please follow the suggestion displayed in "Remediation" entry for each violation listed in `ExampleIssues.csv`. If you have an issue with severity 0 or 1 that has been approved by the Azure PowerShell team, you can suppress them following these steps:
+
+- Download the `ExampleIssues.csv` file from the Jenkins build
+- Open the file using a text editor (such as VS Code) and copy each of the errors you'd like to suppress
+- Paste each of these errors into the `ExampleIssues.csv` file found in their respective [module folder](../tools/StaticAnalysis/Exceptions) (_e.g.,_ if an example issue is being suppressed for Accounts, then you would paste the corresponding line(s) in the `tools/StaticAnalysis/Exceptions/Az.Accounts/ExampleIssue.csv` file) using the same text editor
+- Copy each of the errors you would like to suppress directly from the ExampleIssues.csv file output in the Jenkins build
+- Push the changes to the .csv file and ensure the errors no longer show up in the `ExampleIssues.csv` file output from the Jenkins build
+
+To better standardize the writing of documents, please also check the warning issues with severity 2 by downloading the `ExampleIssues.csv` file.

documentation/tooling/static-analysis.md

@@ -85,6 +85,8 @@ The dependency analyzer can be found in the [`DependencyAnalyzer`](https://githu
     - The implementation of the `IReportRecord` interface; defines what a missing assembly exception looks like when it's reported in the `MissingAssembly.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `MissingAssembly.csv` file used for exception suppressions
 - `SharedAssemblyConflict`
     - The implementation of the `IReportRecord` interface; defines what a shared conflict exception looks like when it's reported in the `SharedAssemblyConflict.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `SharedAssemblyConflict.csv` file used for exception suppressions
+- `ExampleIssue`
+    - The implementation of the `IReportRecord` interface; defines what a shared conflict exception looks like when it's reported in the `ExampleIssues.csv` file that is found in the build artifacts of a CI run, as well as how to compare a new record to a record found in the existing `ExampleIssues.csv` file used for exception suppressions    
 
 #### Help Analyzer
 

tools/BuildPackagesTask/Microsoft.Azure.Build.Tasks/FilesChangedTask.cs

@@ -49,6 +49,11 @@ public class FilesChangedTask : Task
         /// </summary>
         public string TargetModule { get; set; }
 
+        /// <summary>
+        /// Gets or set the OutputFile, store FilesChanged.txt in 'artifacts' folder
+        /// </summary>
+        public string OutputFile { get; set; }
+
         /// <summary>
         /// Gets or sets the files changed produced by the task.
         /// </summary>
@@ -148,10 +153,10 @@ public override bool Execute()
             return true;
         }
 
-        // This method will record the changed files into FilesChanged.txt under root folder for other task to consum.
+        // This method will record the changed files into FilesChanged.txt under '.\artifacts' folder for other task to consum.
         private void SerializeChangedFilesToFile(string[] FilesChanged)
         {
-            File.WriteAllLines("FilesChanged.txt", FilesChanged);
+            File.WriteAllLines(OutputFile, FilesChanged);
         }
     }
 }

tools/PrepareAutorestModule.ps1

@@ -17,7 +17,7 @@
 
 param(
 )
-$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
+$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\artifacts\FilesChanged.txt"
 
 $ALL_MODULE = "ALL_MODULE"
 

tools/PrepareForSecurityCheck.ps1

@@ -17,7 +17,7 @@
 
 param(
 )
-$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\FilesChanged.txt"
+$ChangedFiles = Get-Content -Path "$PSScriptRoot\..\artifacts\FilesChanged.txt"
 
 $SecurityTmpFolder = "$PSScriptRoot\..\SecurityTmp"
 New-Item -ItemType Directory -Force -Path $SecurityTmpFolder

tools/StaticAnalysis/ExampleAnalyzer/AnalyzeRules/CommandName.psm1

@@ -0,0 +1,139 @@
+<#
+    .SYNOPSIS
+    Custom rule for command name.
+    .NOTES
+    File: CommandName.psm1
+#>
+
+enum RuleNames {
+    Invalid_Cmdlet
+    Is_Alias
+    Capitalization_Conventions_Violated
+}
+
+<#
+    .SYNOPSIS
+    Returns invaild, alias or unrecognized cmdlets.
+#>
+function Measure-CommandName {
+    [CmdletBinding()]
+    [OutputType([Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord[]])]
+    param(
+        [Parameter(Mandatory)]
+        [ValidateNotNullOrEmpty()]
+        [System.Management.Automation.Language.ScriptBlockAst]
+        $ScriptBlockAst
+    )
+    begin{
+        Get-Item ./artifacts/Debug/Az.*/Az.*.psd1 | Import-Module -Global
+    }
+    process {
+        $Results = @()
+        $global:CommandParameterPair = @()
+        $global:Ast = $null
+
+        try {
+            [ScriptBlock]$Predicate = {
+                param([System.Management.Automation.Language.Ast]$Ast)
+                $global:Ast = $Ast
+
+                #find all command in .ps1
+                if ($Ast -is [System.Management.Automation.Language.CommandAst]) {
+                    [System.Management.Automation.Language.CommandAst]$CommandAst = $Ast
+                    if ($CommandAst.Parent.Parent -is [System.Management.Automation.Language.AssignmentStatementAst]){
+                        $ModuleCmdletExNum = $CommandAst.Parent.Parent.Parent.Parent.Parent.Name
+                    }
+                    else{
+                        $ModuleCmdletExNum = $CommandAst.Parent.Parent.Parent.Parent.Name
+                    }
+                    if ($CommandAst.InvocationOperator -eq "Unknown") {
+                        # $CommandName = $CommandAst.GetCommandName()
+                        $CommandName = $CommandAst.CommandElements[0].Extent.Text
+                        $GetCommand = Get-Command $CommandName -ErrorAction SilentlyContinue
+                        if ($GetCommand -eq $null) {
+                            # CommandName is not valid.
+                            $global:CommandParameterPair += @{
+                                CommandName = $CommandName
+                                ParameterName = "<is not valid>"
+                                ModuleCmdletExNum = $ModuleCmdletExNum
+                            }
+                            return $true
+                        }
+                        else {
+                            if ($GetCommand.CommandType -eq "Alias") {
+                                # CommandName is an alias.
+                                $global:CommandParameterPair += @{
+                                    CommandName = $CommandName
+                                    ParameterName = "<is an alias>"
+                                    ModuleCmdletExNum = $ModuleCmdletExNum
+                                }
+                                return $true
+                            }
+                            if ($CommandName -cnotmatch "^([A-Z][a-z]+)+-([A-Z][a-z0-9]*)+$") {
+                                # CommandName doesn't follow the Capitalization Conventions.
+                                $global:CommandParameterPair += @{
+                                    CommandName = $CommandName
+                                    ParameterName = "<doesn't follow the Capitalization Conventions>"
+                                    ModuleCmdletExNum = $ModuleCmdletExNum
+                                }
+                                return $true
+                            }
+                        }
+                    }
+                }
+
+                return $false
+            }
+
+            # find all false scriptblock
+            [System.Management.Automation.Language.Ast[]]$Asts = $ScriptBlockAst.FindAll($Predicate, $false)
+            for ($i = 0; $i -lt $Asts.Count; $i++) {
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<is not valid>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) is not a valid command name."
+                    $RuleName = [RuleNames]::Invalid_Cmdlet
+                    $RuleSuppressionID = "5000"
+                    $Remediation = "Check the spell of $($CommandParameterPair[$i].CommandName)."
+                }
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<is an alias>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) is an alias of `"$((Get-Alias $CommandParameterPair[$i].CommandName)[0].ResolvedCommandName)`"."
+                    $RuleName = [RuleNames]::Is_Alias
+                    $RuleSuppressionID = "5100"
+                    $Remediation = "Use formal name `"$((Get-Alias $CommandParameterPair[$i].CommandName)[0].ResolvedCommandName)`" of the alias `"$($CommandParameterPair[$i].CommandName)`"."
+                }
+                if ($global:CommandParameterPair[$i].ParameterName -eq "<doesn't follow the Capitalization Conventions>") {
+                    $Message = "$($CommandParameterPair[$i].CommandName) doesn't follow the Capitalization Conventions."
+                    $RuleName = [RuleNames]::Capitalization_Conventions_Violated
+                    $RuleSuppressionID = "5101"
+                    $name = $($CommandParameterPair[$i].CommandName)
+                    $textInfo = (Get-Culture).TextInfo
+                    $CorrectName = $textInfo.ToTitleCase(($name -split "-")[0])
+                    $CorrectName += "-Az"
+                    $CorrectName += $textInfo.ToTitleCase(($name -split "Az")[1])
+                    $Remediation = "Check the Capitalization Conventions. Suggest format: $CorrectName"
+                }
+                $ModuleCmdletExNum = $($CommandParameterPair[$i].ModuleCmdletExNum)
+                $Result = [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord]@{
+                    Message = "$ModuleCmdletExNum-@$Message@$Remediation";
+                    Extent = $Asts[$i].Extent;
+                    RuleName = $RuleName;
+                    Severity = "Error"
+                    RuleSuppressionID = $RuleSuppressionID
+                }
+                $Results += $Result
+            }
+            return $Results
+        }
+        catch {
+            $Result = [Microsoft.Windows.PowerShell.ScriptAnalyzer.Generic.DiagnosticRecord]@{
+                Message = $_.Exception.Message;
+                Extent = $global:Ast.Extent;
+                RuleName = $PSCmdlet.MyInvocation.InvocationName;
+                Severity = "Error"
+            }
+            $Results += $Result
+            return $Results
+        }
+    }
+}
+
+Export-ModuleMember -Function Measure-*