From be4f499171a0ec999d218993d3823e73a0c008ce Mon Sep 17 00:00:00 2001 From: azure-sdk Date: Tue, 16 Jun 2020 01:34:56 +0000 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository --- .../TestResources/New-TestResources.ps1.md | 17 +++++++ eng/common/TestResources/README.md | 6 +++ .../templates/steps/create-pull-request.yml | 44 ++++++++++--------- .../steps/create-tags-and-git-release.yml | 8 +++- .../templates/steps/docs-metadata-release.yml | 24 +++++----- .../templates/steps/publish-blobs.yml | 12 +++-- eng/common/scripts/Submit-PullRequest.ps1 | 29 ++++++++---- .../scripts/copy-docs-to-blobstorage.ps1 | 2 - .../scripts/create-tags-and-git-release.ps1 | 2 - eng/common/scripts/git-branch-push.ps1 | 4 +- eng/common/scripts/update-docs-metadata.ps1 | 6 +-- 11 files changed, 96 insertions(+), 58 deletions(-) diff --git a/eng/common/TestResources/New-TestResources.ps1.md b/eng/common/TestResources/New-TestResources.ps1.md index 39efc1446258..c9a462aae3a8 100644 --- a/eng/common/TestResources/New-TestResources.ps1.md +++ b/eng/common/TestResources/New-TestResources.ps1.md @@ -412,6 +412,23 @@ Accept pipeline input: False Accept wildcard characters: False ``` +### -OutFile +save test environment settings into a test-resources.json.env file next to test-resources.json. +The file is protected via DPAPI. The environment file would be scoped to the current repository directory. +Note: Supported only on Windows. + +```yaml +Type: SwitchParameter +Parameter Sets: (All) +Aliases: + +Required: False +Position: Named +Default value: None +Accept pipeline input: False +Accept wildcard characters: False +``` + ### CommonParameters This cmdlet supports the common parameters: -Debug, -ErrorAction, -ErrorVariable, -InformationAction, -InformationVariable, -OutVariable, -OutBuffer, -PipelineVariable, -Verbose, -WarningAction, and -WarningVariable. For more information, see [about_CommonParameters](http://go.microsoft.com/fwlink/?LinkID=113216). diff --git a/eng/common/TestResources/README.md b/eng/common/TestResources/README.md index 89e61a4cb07d..3c0cd7c2207e 100644 --- a/eng/common/TestResources/README.md +++ b/eng/common/TestResources/README.md @@ -35,6 +35,12 @@ eng\common\TestResources\New-TestResources.ps1 ` -TestApplicationSecret (ConvertFrom-SecureString $sp.Secret -AsPlainText) ``` +If you are running this for a .NET project on Windows, the recommended method is to +add the `-OutFile` switch to the above command. This will save test environment settings +into a test-resources.json.env file next to test-resources.json. The file is protected via DPAPI. +The environment file would be scoped to the current repository directory and avoids the need to +set environment variables or restart your IDE to recognize them. + Along with some log messages, this will output environment variables based on your current shell like in the following example: diff --git a/eng/common/pipelines/templates/steps/create-pull-request.yml b/eng/common/pipelines/templates/steps/create-pull-request.yml index 46ff5a8f62f8..790de92bcbe5 100644 --- a/eng/common/pipelines/templates/steps/create-pull-request.yml +++ b/eng/common/pipelines/templates/steps/create-pull-request.yml @@ -35,27 +35,31 @@ steps: workingDirectory: ${{ parameters.WorkingDirectory }} ignoreLASTEXITCODE: true -- pwsh: | - ${{ parameters.ScriptDirectory }}/git-branch-push.ps1 ` - -PRBranchName "${{ parameters.PRBranchName }}" ` - -CommitMsg "${{ parameters.CommitMsg }}" ` - -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ parameters.RepoName }}.git" ` - -PushArgs "${{ parameters.PushArgs }}" - +- task: PowerShell@2 displayName: Push changes - workingDirectory: ${{ parameters.WorkingDirectory }} condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: ${{ parameters.WorkingDirectory }} + filePath: ${{ parameters.ScriptDirectory }}/git-branch-push.ps1 + arguments: > + -PRBranchName "${{ parameters.PRBranchName }}" + -CommitMsg "${{ parameters.CommitMsg }}" + -GitUrl "https://$(azuresdk-github-pat)@github.com/${{ parameters.PROwner }}/${{ parameters.RepoName }}.git" + -PushArgs "${{ parameters.PushArgs }}" -- pwsh: | - ${{ parameters.ScriptDirectory }}/Submit-PullRequest.ps1 ` - -RepoOwner "${{ parameters.RepoOwner }}" ` - -RepoName "${{ parameters.RepoName }}" ` - -BaseBranch "${{ parameters.BaseBranchName }}" ` - -PROwner "${{ parameters.PROwner }}" ` - -PRBranch "${{ parameters.PRBranchName }}" ` - -AuthToken "$(azuresdk-github-pat)" ` - -PRTitle "${{ parameters.PRTitle }}" - +- task: PowerShell@2 displayName: Create pull request - workingDirectory: ${{ parameters.WorkingDirectory }} - condition: and(succeeded(), eq(variables['HasChanges'], 'true')) \ No newline at end of file + condition: and(succeeded(), eq(variables['HasChanges'], 'true')) + inputs: + pwsh: true + workingDirectory: ${{ parameters.WorkingDirectory }} + filePath: ${{ parameters.ScriptDirectory }}/Submit-PullRequest.ps1 + arguments: > + -RepoOwner "${{ parameters.RepoOwner }}" + -RepoName "${{ parameters.RepoName }}" + -BaseBranch "${{ parameters.BaseBranchName }}" + -PROwner "${{ parameters.PROwner }}" + -PRBranch "${{ parameters.PRBranchName }}" + -AuthToken "$(azuresdk-github-pat)" + -PRTitle "${{ parameters.PRTitle }}" diff --git a/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml index a1fa15069dbe..8fb57243b0fd 100644 --- a/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml +++ b/eng/common/pipelines/templates/steps/create-tags-and-git-release.yml @@ -10,9 +10,13 @@ steps: - task: PowerShell@2 displayName: 'Verify Package Tags and Create Git Releases' inputs: - targetType: filePath filePath: ${{ parameters.ScriptDirectory }}/create-tags-and-git-release.ps1 - arguments: -artifactLocation ${{parameters.ArtifactLocation}} -packageRepository ${{parameters.PackageRepository}} -releaseSha ${{parameters.ReleaseSha}} -repoId ${{parameters.RepoId}} -workingDirectory '${{parameters.WorkingDirectory}}' + arguments: > + -artifactLocation ${{ parameters.ArtifactLocation }} + -packageRepository ${{ parameters.PackageRepository }} + -releaseSha ${{ parameters.ReleaseSha }} + -repoId ${{ parameters.RepoId }} + -workingDirectory '${{ parameters.WorkingDirectory }}' pwsh: true timeoutInMinutes: 5 env: diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index 89d12d4ac1d1..6ff9f84ff3ee 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -16,10 +16,10 @@ parameters: steps: - pwsh: | git clone https://github.com/${{ parameters.TargetDocRepoOwner }}/${{ parameters.TargetDocRepoName }} ${{ parameters.WorkingDirectory }}/repo - + try { Push-Location ${{ parameters.WorkingDirectory }}/repo - + Write-Host "git checkout smoke-test" git checkout smoke-test } finally { @@ -33,14 +33,14 @@ steps: inputs: targetType: filePath filePath: ${{ parameters.ScriptDirectory }}/update-docs-metadata.ps1 - arguments: > - -ArtifactLocation ${{parameters.ArtifactLocation}} - -Repository ${{parameters.PackageRepository}} - -ReleaseSHA ${{parameters.ReleaseSha}} - -RepoId ${{parameters.RepoId}} - -WorkDirectory '${{parameters.WorkingDirectory}}' - -DocRepoLocation "${{parameters.WorkingDirectory}}/repo" - -Language "${{parameters.Language}}" + arguments: > + -ArtifactLocation ${{ parameters.ArtifactLocation }} + -Repository ${{ parameters.PackageRepository }} + -ReleaseSHA ${{ parameters.ReleaseSha }} + -RepoId ${{ parameters.RepoId }} + -WorkDirectory "${{ parameters.WorkingDirectory }}" + -DocRepoLocation "${{ parameters.WorkingDirectory }}/repo" + -Language "${{parameters.Language}}" -DocRepoContentLocation ${{ parameters.DocRepoDestinationPath }} pwsh: true env: @@ -54,5 +54,5 @@ steps: CommitMsg: "Update readme content for ${{ parameters.ArtifactName }}" PRTitle: "Docs.MS Readme Update." BaseBranchName: smoke-test - WorkingDirectory: ${{parameters.WorkingDirectory}}/repo - ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}} + WorkingDirectory: ${{ parameters.WorkingDirectory }}/repo + ScriptDirectory: ${{ parameters.WorkingDirectory }}/${{ parameters.ScriptDirectory }} diff --git a/eng/common/pipelines/templates/steps/publish-blobs.yml b/eng/common/pipelines/templates/steps/publish-blobs.yml index ce0d9f708cca..682cc4d4f7c8 100644 --- a/eng/common/pipelines/templates/steps/publish-blobs.yml +++ b/eng/common/pipelines/templates/steps/publish-blobs.yml @@ -7,16 +7,20 @@ parameters: steps: - pwsh: | - Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" ` - -OutFile "azcopy.zip" | Wait-Process; Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" + Invoke-WebRequest -MaximumRetryCount 10 -Uri "https://aka.ms/downloadazcopy-v10-windows" -OutFile "azcopy.zip" | Wait-Process; + Expand-Archive -Path "azcopy.zip" -DestinationPath "$(Build.BinariesDirectory)/azcopy/" workingDirectory: $(Build.BinariesDirectory) displayName: Download and Extract azcopy Zip - task: Powershell@2 inputs: - targetType: 'filePath' filePath: ${{ parameters.ScriptPath }} - arguments: -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] -DocLocation "${{ parameters.FolderForUpload }}" -SASKey "${{ parameters.BlobSASKey }}" -Language "${{ parameters.TargetLanguage }}" -BlobName "${{ parameters.BlobName }}" + arguments: > + -AzCopy $(Resolve-Path "$(Build.BinariesDirectory)/azcopy/azcopy_windows_amd64_*/azcopy.exe")[0] + -DocLocation "${{ parameters.FolderForUpload }}" + -SASKey "${{ parameters.BlobSASKey }}" + -Language "${{ parameters.TargetLanguage }}" + -BlobName "${{ parameters.BlobName }}" pwsh: true workingDirectory: $(Pipeline.Workspace) displayName: Copy Docs to Blob diff --git a/eng/common/scripts/Submit-PullRequest.ps1 b/eng/common/scripts/Submit-PullRequest.ps1 index 2313a83c9f9a..ef2a6f545061 100644 --- a/eng/common/scripts/Submit-PullRequest.ps1 +++ b/eng/common/scripts/Submit-PullRequest.ps1 @@ -41,21 +41,25 @@ param( $PRBody = $PRTitle ) -Write-Host "> $PSCommandPath $args" +$headers = @{ + Authorization = "bearer $AuthToken" +} $query = "state=open&head=${PROwner}:${PRBranch}&base=${BaseBranch}" -$resp = Invoke-RestMethod "https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query" +try { + $resp = Invoke-RestMethod -Headers $headers "https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query" +} +catch { + Write-Error "Invoke-RestMethod [https://api.github.com/repos/$RepoOwner/$RepoName/pulls?$query] failed with exception:`n$_" + exit 1 +} $resp | Write-Verbose if ($resp.Count -gt 0) { Write-Host -f green "Pull request already exists $($resp[0].html_url)" } else { - $headers = @{ - Authorization = "bearer $AuthToken" - } - $data = @{ title = $PRTitle head = "${PROwner}:${PRBranch}" @@ -64,9 +68,16 @@ else { maintainer_can_modify = $true } - $resp = Invoke-RestMethod -Method POST -Headers $headers ` - https://api.github.com/repos/$RepoOwner/$RepoName/pulls ` - -Body ($data | ConvertTo-Json) + try { + $resp = Invoke-RestMethod -Method POST -Headers $headers ` + "https://api.github.com/repos/$RepoOwner/$RepoName/pulls" ` + -Body ($data | ConvertTo-Json) + } + catch { + Write-Error "Invoke-RestMethod [https://api.github.com/repos/$RepoOwner/$RepoName/pulls] failed with exception:`n$_" + exit 1 + } + $resp | Write-Verbose Write-Host -f green "Pull request created https://github.com/$RepoOwner/$RepoName/pull/$($resp.number)" } diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index 03e508c17cb5..0093c38e7f57 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -10,8 +10,6 @@ param ( $UploadLatest=1 ) -Write-Host "> $PSCommandPath $args" - $Language = $Language.ToLower() # Regex inspired but simplified from https://semver.org/#is-there-a-suggested-regular-expression-regex-to-check-a-semver-string diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index f87c90997839..1667cddd7e22 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -18,8 +18,6 @@ param ( [switch]$continueOnError = $false ) -Write-Host "> $PSCommandPath $args" - . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) $apiUrl = "https://api.github.com/repos/$repoId" diff --git a/eng/common/scripts/git-branch-push.ps1 b/eng/common/scripts/git-branch-push.ps1 index 9ff45f87ad67..cd28582efeda 100644 --- a/eng/common/scripts/git-branch-push.ps1 +++ b/eng/common/scripts/git-branch-push.ps1 @@ -28,8 +28,6 @@ param( [string] $PushArgs = "" ) -Write-Host "> $PSCommandPath $args" - # This is necessay because of the janky git command output writing to stderr. # Without explicitly setting the ErrorActionPreference to continue the script # would fail the first time git wrote command output. @@ -122,7 +120,7 @@ do } Write-Host "git add -A" - git add -A + git add -A if ($LASTEXITCODE -ne 0) { Write-Error "Unable to git add LASTEXITCODE=$($LASTEXITCODE), see command output above." diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index f88717c4f714..9eba32ae35ec 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -14,8 +14,6 @@ param ( $DocRepoContentLocation = "docs-ref-services/" # within the doc repo, where does our readme go? ) -Write-Host "> $PSCommandPath $args" - # import artifact parsing and semver handling . (Join-Path $PSScriptRoot artifact-metadata-parsing.ps1) @@ -58,7 +56,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ $pkgId = $pkgInfo.PackageId.Replace("@azure/", "") try { - $metadata = GetMetaData -lang $lang + $metadata = GetMetaData -lang $lang $service = $metadata | ? { $_.Package -eq $pkgId } @@ -99,7 +97,7 @@ $pkgs = VerifyPackages -pkgRepository $Repository ` if ($pkgs) { Write-Host "Given the visible artifacts, readmes will be copied for the following packages" - Write-Host ($pkgs | % { $_.PackageId }) + Write-Host ($pkgs | % { $_.PackageId }) foreach ($packageInfo in $pkgs) { # sync the doc repo