From eadfeb8bd1e094129ebe9ef8a9fbdedb5ed18add Mon Sep 17 00:00:00 2001 From: azure-sdk Date: Fri, 26 Jun 2020 23:57:26 +0000 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository --- .../templates/steps/verify-agent-os.yml | 1 - .../templates/steps/verify-changelog.yml | 25 ++++ eng/common/scripts/Verify-ChangeLog.ps1 | 22 ++++ .../scripts/artifact-metadata-parsing.ps1 | 34 ++--- .../scripts/create-tags-and-git-release.ps1 | 1 + .../scripts/modules/ChangeLog-Operations.psm1 | 116 ++++++++++++++++++ .../scripts/modules/common-manifest.psd1 | 2 +- eng/common/scripts/update-docs-metadata.ps1 | 18 +-- 8 files changed, 194 insertions(+), 25 deletions(-) create mode 100644 eng/common/pipelines/templates/steps/verify-changelog.yml create mode 100644 eng/common/scripts/Verify-ChangeLog.ps1 create mode 100644 eng/common/scripts/modules/ChangeLog-Operations.psm1 diff --git a/eng/common/pipelines/templates/steps/verify-agent-os.yml b/eng/common/pipelines/templates/steps/verify-agent-os.yml index 1786612ba56c..b221583bc6da 100644 --- a/eng/common/pipelines/templates/steps/verify-agent-os.yml +++ b/eng/common/pipelines/templates/steps/verify-agent-os.yml @@ -1,4 +1,3 @@ -# Template for all Python Scripts in this repository parameters: OSVmImage: $(OSVmImage) diff --git a/eng/common/pipelines/templates/steps/verify-changelog.yml b/eng/common/pipelines/templates/steps/verify-changelog.yml new file mode 100644 index 000000000000..f90238471d95 --- /dev/null +++ b/eng/common/pipelines/templates/steps/verify-changelog.yml @@ -0,0 +1,25 @@ +parameters: +- name: PackageName + type: string + default: 'not-specified' +- name: ServiceName + type: string + default: 'not-specified' +- name: ForRelease + type: boolean + default: false + +steps: + - task: Powershell@2 + inputs: + filePath: /eng/common/scripts/Verify-ChangeLog.ps1 + arguments: > + -PackageName ${{ parameters.PackageName }} + -ServiceName ${{ parameters.ServiceName }} + -RepoRoot $(Build.SourcesDirectory) + -RepoName $(Build.Repository.Name) + -ForRelease ${{ parameters.ForRelease }} + pwsh: true + workingDirectory: $(Pipeline.Workspace) + displayName: Verify ChangeLog / Release Notes + continueOnError: false \ No newline at end of file diff --git a/eng/common/scripts/Verify-ChangeLog.ps1 b/eng/common/scripts/Verify-ChangeLog.ps1 new file mode 100644 index 000000000000..1f9991bfd668 --- /dev/null +++ b/eng/common/scripts/Verify-ChangeLog.ps1 @@ -0,0 +1,22 @@ +# Wrapper Script for ChangeLog Verification +param ( + [Parameter(Mandatory=$true)] + [string]$PackageName, + [Parameter(Mandatory=$true)] + [string]$ServiceName, + [string]$RepoRoot, + [ValidateSet("net","java","js","python")] + [string]$Language, + [string]$RepoName, + [boolean]$ForRelease=$False +) + +Import-Module "${PSScriptRoot}/modules/common-manifest.psd1" + +if ([System.String]::IsNullOrEmpty($Language)) +{ + $Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1) +} + +$PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot +Confirm-ChangeLog -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgReadMePath -ForRelease $ForRelease \ No newline at end of file diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 2d2362f0d0d9..a4d510120167 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -1,3 +1,4 @@ +Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1" . (Join-Path $PSScriptRoot SemVer.ps1) $SDIST_PACKAGE_REGEX = "^(?.*)\-(?$([AzureEngSemanticVersion]::SEMVER_REGEX))" @@ -8,8 +9,8 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) { Write-Host "Creating release $($pkgInfo.Tag)" $releaseNotes = "" - if ($pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent -ne $null) { - $releaseNotes = $pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent + if ($pkgInfo.ReleaseNotes -ne $null) { + $releaseNotes = $pkgInfo.ReleaseNotes } $isPrerelease = $False @@ -96,7 +97,7 @@ function ParseMavenPackage($pkg, $workingDirectory) { $changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0] @@ -169,13 +170,15 @@ function ParseNPMPackage($pkg, $workingDirectory) { tar -xzf $pkg $packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json + $pkgId = $packageJSON.name + $pkgVersion = $packageJSON.version $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] + $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 if ($readmeContentLoc) { $readmeContent = Get-Content -Raw $readmeContentLoc } @@ -183,9 +186,6 @@ function ParseNPMPackage($pkg, $workingDirectory) { cd $origFolder Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - $pkgId = $packageJSON.name - $pkgVersion = $packageJSON.version - $resultObj = New-Object PSObject -Property @{ PackageId = $pkgId PackageVersion = $pkgVersion @@ -229,10 +229,12 @@ function ParseNugetPackage($pkg, $workingDirectory) { Copy-Item -Path $pkg -Destination $zipFileLocation Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content + $pkgId = $packageXML.package.metadata.id + $pkgVersion = $packageXML.package.metadata.version $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] @@ -241,8 +243,6 @@ function ParseNugetPackage($pkg, $workingDirectory) { } Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue - $pkgId = $packageXML.package.metadata.id - $pkgVersion = $packageXML.package.metadata.version return New-Object PSObject -Property @{ PackageId = $pkgId @@ -297,13 +297,15 @@ function ParsePyPIPackage($pkg, $workingDirectory) { $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } - $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0] + $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1 + if ($readmeContentLoc) { $readmeContent = Get-Content -Raw $readmeContentLoc } + Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue return New-Object PSObject -Property @{ @@ -321,10 +323,12 @@ function ParseCArtifact($pkg, $workingDirectory) { $releaseNotes = "" $readmeContent = "" + $pkgVersion = $packageInfo.version + $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0] if ($changeLogLoc) { - $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc + $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion } $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0] @@ -334,7 +338,7 @@ function ParseCArtifact($pkg, $workingDirectory) { return New-Object PSObject -Property @{ PackageId = '' - PackageVersion = $packageInfo.version + PackageVersion = $pkgVersion # Artifact info is always considered deployable for C becasue it is not # deployed anywhere. Dealing with duplicate tags happens downstream in # CheckArtifactShaAgainstTagsList diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index 1667cddd7e22..cec45536b704 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -6,6 +6,7 @@ param ( # used by VerifyPackages $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory) $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary) + [ValidateSet("Nuget","NPM","PyPI","Maven")] $packageRepository, # used to indicate destination against which we will check the existing version. # valid options: PyPI, Nuget, NPM, Maven, C # used by CreateTags diff --git a/eng/common/scripts/modules/ChangeLog-Operations.psm1 b/eng/common/scripts/modules/ChangeLog-Operations.psm1 new file mode 100644 index 000000000000..5279668412b7 --- /dev/null +++ b/eng/common/scripts/modules/ChangeLog-Operations.psm1 @@ -0,0 +1,116 @@ +# Common Changelog Operations + +$RELEASE_TITLE_REGEX = "(?^\#+.*(?\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)" + +# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG +function Get-ChangeLogEntries { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation + ) + + $changeLogEntries = @{} + if (!(Test-Path $ChangeLogLocation)) { + Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation + exit 1 + } + + try { + $contents = Get-Content $ChangeLogLocation + # walk the document, finding where the version specifiers are and creating lists + $changeLogEntry = $null + foreach ($line in $contents) { + if ($line -match $RELEASE_TITLE_REGEX) { + $changeLogEntry = [pscustomobject]@{ + ReleaseVersion = $matches["version"] + ReleaseStatus = $matches["releaseStatus"] + ReleaseTitle = $line + ReleaseContent = @() # Release content without the version title + } + $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry + } + else { + if ($changeLogEntry) { + $changeLogEntry.ReleaseContent += $line + } + } + } + } + catch { + Write-Host "Error parsing $ChangeLogLocation." + Write-Host $_.Exception.Message + } + return $changeLogEntries +} + +# Returns single changeLogEntry object containing the ChangeLog for a particular version +function Get-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation + + if ($changeLogEntries.ContainsKey($VersionString)) { + return $changeLogEntries[$VersionString] + } + Write-Error "Release Notes for the Specified version ${VersionString} was not found" + exit 1 +} + +#Returns the changelog for a particular version as string +function Get-ChangeLogEntryAsString { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString + ) + + $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + [string]$releaseTitle = $changeLogEntries.ReleaseTitle + [string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine + return $releaseTitle, $releaseContent -Join [Environment]::NewLine +} + +function Confirm-ChangeLogEntry { + param ( + [Parameter(Mandatory = $true)] + [String]$ChangeLogLocation, + [Parameter(Mandatory = $true)] + [String]$VersionString, + [boolean]$ForRelease = $false + ) + + $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString + + if ([System.String]::IsNullOrEmpty($changeLogEntries.ReleaseStatus)) { + Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation) + Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released" + exit 1 + } + + if ($ForRelease -eq $True) { + $CurrentDate = Get-Date -Format "yyyy-MM-dd" + if ($changeLogEntries.ReleaseStatus -ne "($CurrentDate)") { + Write-Host ("##[warning]Incorrect Date: Please use the current date in the Changelog '{0}' before releasing the package" -f $ChangeLogLocation) + exit 1 + } + + if ([System.String]::IsNullOrWhiteSpace($changeLogEntries.ReleaseContent)) { + Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation) + Write-Host "##[info]Please ensure there is a release notes entry before releasing the package." + exit 1 + } + } + + Write-Host ($changeLogEntries | Format-Table | Out-String) +} + +Export-ModuleMember -Function 'Get-ChangeLogEntries' +Export-ModuleMember -Function 'Get-ChangeLogEntry' +Export-ModuleMember -Function 'Get-ChangeLogEntryAsString' +Export-ModuleMember -Function 'Confirm-ChangeLogEntry' \ No newline at end of file diff --git a/eng/common/scripts/modules/common-manifest.psd1 b/eng/common/scripts/modules/common-manifest.psd1 index 419f046e6f67..43dee1c26e23 100644 --- a/eng/common/scripts/modules/common-manifest.psd1 +++ b/eng/common/scripts/modules/common-manifest.psd1 @@ -66,7 +66,7 @@ ScriptsToProcess = @("${PSScriptRoot}\..\SemVer.ps1") # FormatsToProcess = @() # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess -NestedModules = @("${PSScriptRoot}\Package-Properties.psm1") +NestedModules = @("${PSScriptRoot}\Package-Properties.psm1", "${PSScriptRoot}\ChangeLog-Operations.psm1") # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. # FunctionsToExport = @() diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index 9eba32ae35ec..a858078f5448 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -22,19 +22,19 @@ param ( function GetMetaData($lang){ switch ($lang) { "java" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/java-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv" break } ".net" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/dotnet-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv" break } "python" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/python-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv" break } "javascript" { - $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/js-packages.csv" + $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv" break } default { @@ -61,7 +61,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ $service = $metadata | ? { $_.Package -eq $pkgId } if ($service) { - $service = "$($service.Service)" + $service = "$($service.ServiceName)".ToLower().Replace(" ", "") } } catch { @@ -70,15 +70,17 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){ } $fileContent = $pkgInfo.ReadmeContent + $foundTitle = "" # only replace the version if the formatted header can be found $headerContentMatches = (Select-String -InputObject $pkgInfo.ReadmeContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET|C)') if ($headerContentMatches) { - $headerContentMatch = $headerContentMatches.Matches[0] - $header = "---`ntitle: $headerContentMatch`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n" - $fileContent = $pkgInfo.ReadmeContent -replace $headerContentMatch, "$headerContentMatch - Version $($pkgInfo.PackageVersion) `n" + $foundTitle = $headerContentMatches.Matches[0] + $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n" } + $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n" + if ($fileContent) { return "$header`n$fileContent" }