From 35dce4a5acdbb612a39d7ee5f42e08132550d05f Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Tue, 10 Aug 2021 17:01:56 +0800 Subject: [PATCH 1/7] add lgtm and sdl --- azure-pipelines.lgtm.yml | 57 +++++++ azure-pipelines.yml | 353 +++++++++++++++++++++++---------------- 2 files changed, 270 insertions(+), 140 deletions(-) create mode 100644 azure-pipelines.lgtm.yml diff --git a/azure-pipelines.lgtm.yml b/azure-pipelines.lgtm.yml new file mode 100644 index 000000000..663e8604a --- /dev/null +++ b/azure-pipelines.lgtm.yml @@ -0,0 +1,57 @@ +# ASP.NET Core (.NET Framework) +# Build and test ASP.NET Core projects targeting the full .NET Framework. +# Add steps that publish symbols, save build artifacts, and more: +# https://docs.microsoft.com/azure/devops/pipelines/languages/dotnet-core + +pr: none +trigger: none +schedules: +- cron: "0 0 * * *" + displayName: Daily midnight build + branches: + include: + - main + +pool: + vmImage: 'windows-latest' + +variables: +- name: LGTM.SnapshotIdentifiers + value: Build=Full +- name: LGTM.SnapshotMetadata + value: Owner=MDoc +- name: LGTM.UploadSnapshot + value: true +- name: Semmle.SkipAnalysis + value: true +- name: buildConfiguration + value: Release + +steps: +- task: NuGetToolInstaller@1 + displayName: Install NuGet Tool + +- task: NuGetCommand@2 + inputs: + command: 'restore' + restoreSolution: '**/*.sln' + feedsToUse: 'config' + +- task: Semmle@0 + displayName: Security - LGTM + env: + SYSTEM_ACCESSTOKEN: $(System.AccessToken) + inputs: + toolVersion: 'LatestPreRelease' + sourceCodeDirectory: '$(Build.SourcesDirectory)' + language: 'csharp' + cleanupBuildCommands: | + dotnet clean mdoc/mdoc.csproj --configuration $(buildConfiguration) + dotnet clean monodoc/monodoc.csproj --configuration $(buildConfiguration) + buildCommands: | + dotnet build mdoc/mdoc.csproj --configuration $(buildConfiguration) + dotnet build monodoc/monodoc.csproj --configuration $(buildConfiguration) + querySuite: 'Required' + timeout: '7200' + ram: '16384' + addProjectDirToScanningExclusionList: true diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 7b5213562..a35bc69aa 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -8,149 +8,222 @@ trigger: name: $(SourceBranchName)-$(Date:yyyyMMdd)$(Rev:.r) -pool: - vmImage: 'macos-latest' - variables: + solution: 'apidoctools.sln' + buildConfiguration: 'Release' mdocPath: 'bin/$(buildConfiguration)' - -steps: -- task: NuGetToolInstaller@1 - displayName: Install NuGet Tool - -- task: Bash@3 - displayName: Run Unit and Integration Tests - inputs: - targetType: 'inline' - script: 'make prepare all check CONFIGURATION=$(buildConfiguration)' - -- task: EsrpCodeSigning@1 - displayName: Sign executable and dll files - condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) - inputs: - ConnectedServiceName: 'CodeSigning-APEX' - FolderPath: '$(mdocPath)' - UseMinimatch: true - signConfigType: inlineSignParams - SessionTimeout: '60' - MaxConcurrency: '100' - MaxRetryAttempts: '5' - Pattern: | - *.dll - *.exe - inlineOperation: | - [ - { - "KeyCode": "CP-230012", - "OperationSetCode": "SigntoolSign", - "parameters": [ - { - "parameterName": "OpusName", - "parameterValue": "Microsoft" - }, - { - "parameterName": "OpusInfo", - "parameterValue": "http://www.microsoft.com" - }, - { - "parameterName": "PageHash", - "parameterValue": "/NPH" - }, + +stages: +- stage: SDLCheck + jobs: + - job: SDLCheck + pool: + vmImage: 'windows-latest' + + steps: + - task: CredScan@2 + displayName: Security - CredScan + inputs: + toolMajorVersion: 'V2' + + - task: CmdLine@2 + displayName: make prepare + inputs: + script: 'make prepare' + + - task: MSBuild@1 + displayName: MSBuild + inputs: + solution: '$(solution)' + configuration: '$(buildConfiguration)' + + - task: BinSkim@4 + displayName: Security - BinSkim + inputs: + InputType: 'Basic' + Function: 'analyze' + TargetPattern: 'guardianGlob' + AnalyzeTargetGlob: '$(mdocPath)\**.dll;$(mdocPath)\**.exe;' + + - task: SdtReport@1 + displayName: Security - SdtReport + inputs: + AllTools: false + CredScan: true + BinSkim: true + ToolLogsNotFoundAction: 'Standard' + + - task: PublishSecurityAnalysisLogs@2 + displayName: Security - Publish Scan Results + inputs: + ArtifactName: 'CodeAnalysisLogs' + ArtifactType: 'Container' + AllTools: true + ToolLogsNotFoundAction: 'Standard' + + - task: TSAUpload@1 + inputs: + tsaVersion: 'TsaV2' + codebase: 'Existing' + tsaEnvironment: 'PROD' + codeBaseName: 'Docs_default' + uploadBinSkim: true + uploadCredScan: true + uploadAsync: true + + - task: PostAnalysis@1 + displayName: Security - PostAnalysis + inputs: + AllTools: false + CredScan: true + BinSkim: true + ToolLogsNotFoundAction: 'Standard' + +- stage: Build + dependsOn: SDLCheck + jobs: + - job: Build + pool: + vmImage: 'macos-latest' + + steps: + - task: NuGetToolInstaller@1 + displayName: Install NuGet Tool + + - task: Bash@3 + displayName: Run Unit and Integration Tests + inputs: + targetType: 'inline' + script: 'make prepare all check CONFIGURATION=$(buildConfiguration)' + + - task: EsrpCodeSigning@1 + displayName: Sign executable and dll files + condition: and(succeeded(), ne(variables['Build.Reason'], 'PullRequest')) + inputs: + ConnectedServiceName: 'CodeSigning-APEX' + FolderPath: '$(mdocPath)' + UseMinimatch: true + signConfigType: inlineSignParams + SessionTimeout: '60' + MaxConcurrency: '100' + MaxRetryAttempts: '5' + Pattern: | + *.dll + *.exe + inlineOperation: | + [ { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + "KeyCode": "CP-230012", + "OperationSetCode": "SigntoolSign", + "parameters": [ + { + "parameterName": "OpusName", + "parameterValue": "Microsoft" + }, + { + "parameterName": "OpusInfo", + "parameterValue": "http://www.microsoft.com" + }, + { + "parameterName": "PageHash", + "parameterValue": "/NPH" + }, + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + } + ], + "ToolName": "sign", + "ToolVersion": "1.0" }, { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - } - ], - "ToolName": "sign", - "ToolVersion": "1.0" - }, - { - "KeyCode": "CP-230012", - "OperationSetCode": "SigntoolVerify", - "Parameters": [ - { - "parameterName": "VerifyAll", - "parameterValue": "/all" + "KeyCode": "CP-230012", + "OperationSetCode": "SigntoolVerify", + "Parameters": [ + { + "parameterName": "VerifyAll", + "parameterValue": "/all" + } + ], + "ToolName": "sign", + "ToolVersion": "1.0" } - ], - "ToolName": "sign", - "ToolVersion": "1.0" - } - ] - -- task: ArchiveFiles@2 - displayName: Archive mdoc Files - inputs: - rootFolderOrFile: '$(mdocPath)' - includeRootFolder: false - archiveType: 'zip' - archiveFile: '$(Build.ArtifactStagingDirectory)/zips/mdoc-$(Build.BuildNumber).zip' - replaceExistingArchive: true - -- task: PublishBuildArtifacts@1 - displayName: 'Publish mdoc Artifact' - inputs: - PathtoPublish: '$(Build.ArtifactStagingDirectory)/zips' - ArtifactName: 'mdoc.Artifact' -- task: PowerShell@2 - name: 'mdocVersion' - displayName: 'Checking remote and local version of mdoc' - condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - filePath: 'mdoc/CheckNugetPublish.ps1' - -- task: NuGetCommand@2 - displayName: 'Create a NuGet package for mdoc' - condition: eq(variables['mdocVersion.NeedUpdate'], true) - inputs: - command: 'pack' - packagesToPack: 'mdoc/mdoc.nuspec' - -- task: EsrpCodeSigning@1 - displayName: 'Sign NuGet packages' - condition: eq(variables['mdocVersion.NeedUpdate'], true) - inputs: - ConnectedServiceName: 'CodeSigning-APEX' - FolderPath: '$(Build.ArtifactStagingDirectory)' - Pattern: '*.nupkg' - signConfigType: inlineSignParams - inlineOperation: | - [ - { - "KeyCode" : "CP-401405", - "OperationSetCode" : "NuGetSign", - "Parameters" : [ - { - "parameterName": "TimeStamp", - "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" - }, - { - "parameterName": "FileDigest", - "parameterValue": "/fd \"SHA256\"" - } - ], - "ToolName" : "sign", - "ToolVersion" : "1.0" - }, - { - "KeyCode" : "CP-401405", - "OperationSetCode" : "NuGetVerify", - "Parameters" : [], - "ToolName" : "sign", - "ToolVersion" : "1.0" - } - ] - SessionTimeout: 20 - -- task: NuGetCommand@2 - displayName: 'Publishing mdoc package to nuget.org' - condition: eq(variables['mdocVersion.NeedUpdate'], true) - inputs: - command: 'push' - packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg' - nuGetFeedType: 'external' - publishFeedCredentials: 'mdoc_nuget_org' \ No newline at end of file + ] + + - task: ArchiveFiles@2 + displayName: Archive mdoc Files + inputs: + rootFolderOrFile: '$(mdocPath)' + includeRootFolder: false + archiveType: 'zip' + archiveFile: '$(Build.ArtifactStagingDirectory)/zips/mdoc-$(Build.BuildNumber).zip' + replaceExistingArchive: true + + - task: PublishBuildArtifacts@1 + displayName: 'Publish mdoc Artifact' + inputs: + PathtoPublish: '$(Build.ArtifactStagingDirectory)/zips' + ArtifactName: 'mdoc.Artifact' + + - task: PowerShell@2 + name: 'mdocVersion' + displayName: 'Checking remote and local version of mdoc' + condition: and(succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + inputs: + filePath: 'mdoc/CheckNugetPublish.ps1' + + - task: NuGetCommand@2 + displayName: 'Create a NuGet package for mdoc' + condition: eq(variables['mdocVersion.NeedUpdate'], true) + inputs: + command: 'pack' + packagesToPack: 'mdoc/mdoc.nuspec' + + - task: EsrpCodeSigning@1 + displayName: 'Sign NuGet packages' + condition: eq(variables['mdocVersion.NeedUpdate'], true) + inputs: + ConnectedServiceName: 'CodeSigning-APEX' + FolderPath: '$(Build.ArtifactStagingDirectory)' + Pattern: '*.nupkg' + signConfigType: inlineSignParams + inlineOperation: | + [ + { + "KeyCode" : "CP-401405", + "OperationSetCode" : "NuGetSign", + "Parameters" : [ + { + "parameterName": "TimeStamp", + "parameterValue": "/tr \"http://rfc3161.gtm.corp.microsoft.com/TSS/HttpTspServer\" /td sha256" + }, + { + "parameterName": "FileDigest", + "parameterValue": "/fd \"SHA256\"" + } + ], + "ToolName" : "sign", + "ToolVersion" : "1.0" + }, + { + "KeyCode" : "CP-401405", + "OperationSetCode" : "NuGetVerify", + "Parameters" : [], + "ToolName" : "sign", + "ToolVersion" : "1.0" + } + ] + SessionTimeout: 20 + + - task: NuGetCommand@2 + displayName: 'Publishing mdoc package to nuget.org' + condition: eq(variables['mdocVersion.NeedUpdate'], true) + inputs: + command: 'push' + packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg' + nuGetFeedType: 'external' + publishFeedCredentials: 'modc_nuget_org' \ No newline at end of file From ed2dd14b4a6e75f448d2ae1d1eeb490d999a2c66 Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Tue, 10 Aug 2021 17:06:12 +0800 Subject: [PATCH 2/7] update typo --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index a35bc69aa..c76a3a79c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -226,4 +226,4 @@ stages: command: 'push' packagesToPush: '$(Build.ArtifactStagingDirectory)/**/*.nupkg;!$(Build.ArtifactStagingDirectory)/**/*.symbols.nupkg' nuGetFeedType: 'external' - publishFeedCredentials: 'modc_nuget_org' \ No newline at end of file + publishFeedCredentials: 'mdoc_nuget_org' \ No newline at end of file From bc8acea410a927b0ad9129882f09b26c869b2684 Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Wed, 11 Aug 2021 10:12:38 +0800 Subject: [PATCH 3/7] Update azure-pipelines.yml for Azure Pipelines test for BinSkim@3 --- azure-pipelines.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index c76a3a79c..dfaaad7a2 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -37,13 +37,12 @@ stages: solution: '$(solution)' configuration: '$(buildConfiguration)' - - task: BinSkim@4 + - task: BinSkim@3 displayName: Security - BinSkim inputs: InputType: 'Basic' Function: 'analyze' - TargetPattern: 'guardianGlob' - AnalyzeTargetGlob: '$(mdocPath)\**.dll;$(mdocPath)\**.exe;' + AnalyzeTarget: '$(mdocPath)\**.dll;$(mdocPath)\**.exe;' - task: SdtReport@1 displayName: Security - SdtReport From c0e70479b13ff69550bf4c4396a90c1475e633b6 Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Wed, 11 Aug 2021 10:30:00 +0800 Subject: [PATCH 4/7] Update azure-pipelines.yml for Azure Pipelines test --- azure-pipelines.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index dfaaad7a2..f6ff1763f 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,7 +42,9 @@ stages: inputs: InputType: 'Basic' Function: 'analyze' - AnalyzeTarget: '$(mdocPath)\**.dll;$(mdocPath)\**.exe;' + AnalyzeTarget: > + $(mdocPath)\**.dll; + $(mdocPath)\**.exe; - task: SdtReport@1 displayName: Security - SdtReport From 885bee000bf4d75414f085a80d5be67821e2752c Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Wed, 11 Aug 2021 11:32:31 +0800 Subject: [PATCH 5/7] Update azure-pipelines.yml for Azure Pipelines test --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index f6ff1763f..06b986a47 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,8 +43,8 @@ stages: InputType: 'Basic' Function: 'analyze' AnalyzeTarget: > - $(mdocPath)\**.dll; - $(mdocPath)\**.exe; + bin\Release\**.dll; + bin\Release\**.exe; - task: SdtReport@1 displayName: Security - SdtReport From b94820ace43ceccbd4fdb646fcc0aa4f42f6ee78 Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Wed, 11 Aug 2021 11:36:53 +0800 Subject: [PATCH 6/7] Update azure-pipelines.yml for Azure Pipelines test --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 06b986a47..e9668432a 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -43,8 +43,8 @@ stages: InputType: 'Basic' Function: 'analyze' AnalyzeTarget: > - bin\Release\**.dll; - bin\Release\**.exe; + bin\Release\*.dll; + bin\Release\*.exe; - task: SdtReport@1 displayName: Security - SdtReport From 46e11fb0c4b349de2327d526265083496dabd5ab Mon Sep 17 00:00:00 2001 From: Shuang Jiang Date: Wed, 11 Aug 2021 12:46:58 +0800 Subject: [PATCH 7/7] Update azure-pipelines.yml for Azure Pipelines fix binskim --- azure-pipelines.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e9668432a..aa21e2663 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -42,9 +42,7 @@ stages: inputs: InputType: 'Basic' Function: 'analyze' - AnalyzeTarget: > - bin\Release\*.dll; - bin\Release\*.exe; + AnalyzeTarget: '$(mdocPath)\*.dll;$(mdocPath)\*.exe;' - task: SdtReport@1 displayName: Security - SdtReport