Skip to content

Commit

Permalink
Sync eng/common directory with azure-sdk-tools repository
Browse files Browse the repository at this point in the history
  • Loading branch information
azure-sdk committed Jun 26, 2020
1 parent 6ccdc67 commit eadfeb8
Show file tree
Hide file tree
Showing 8 changed files with 194 additions and 25 deletions.
1 change: 0 additions & 1 deletion eng/common/pipelines/templates/steps/verify-agent-os.yml
Original file line number Diff line number Diff line change
@@ -1,4 +1,3 @@
# Template for all Python Scripts in this repository
parameters:
OSVmImage: $(OSVmImage)

Expand Down
25 changes: 25 additions & 0 deletions eng/common/pipelines/templates/steps/verify-changelog.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
parameters:
- name: PackageName
type: string
default: 'not-specified'
- name: ServiceName
type: string
default: 'not-specified'
- name: ForRelease
type: boolean
default: false

steps:
- task: Powershell@2
inputs:
filePath: /eng/common/scripts/Verify-ChangeLog.ps1
arguments: >
-PackageName ${{ parameters.PackageName }}
-ServiceName ${{ parameters.ServiceName }}
-RepoRoot $(Build.SourcesDirectory)
-RepoName $(Build.Repository.Name)
-ForRelease ${{ parameters.ForRelease }}
pwsh: true
workingDirectory: $(Pipeline.Workspace)
displayName: Verify ChangeLog / Release Notes
continueOnError: false
22 changes: 22 additions & 0 deletions eng/common/scripts/Verify-ChangeLog.ps1
Original file line number Diff line number Diff line change
@@ -0,0 +1,22 @@
# Wrapper Script for ChangeLog Verification
param (
[Parameter(Mandatory=$true)]
[string]$PackageName,
[Parameter(Mandatory=$true)]
[string]$ServiceName,
[string]$RepoRoot,
[ValidateSet("net","java","js","python")]
[string]$Language,
[string]$RepoName,
[boolean]$ForRelease=$False
)

Import-Module "${PSScriptRoot}/modules/common-manifest.psd1"

if ([System.String]::IsNullOrEmpty($Language))
{
$Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1)
}

$PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
Confirm-ChangeLog -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgReadMePath -ForRelease $ForRelease
34 changes: 19 additions & 15 deletions eng/common/scripts/artifact-metadata-parsing.ps1
Original file line number Diff line number Diff line change
@@ -1,3 +1,4 @@
Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1"
. (Join-Path $PSScriptRoot SemVer.ps1)

$SDIST_PACKAGE_REGEX = "^(?<package>.*)\-(?<versionstring>$([AzureEngSemanticVersion]::SEMVER_REGEX))"
Expand All @@ -8,8 +9,8 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
Write-Host "Creating release $($pkgInfo.Tag)"

$releaseNotes = ""
if ($pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent -ne $null) {
$releaseNotes = $pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent
if ($pkgInfo.ReleaseNotes -ne $null) {
$releaseNotes = $pkgInfo.ReleaseNotes
}

$isPrerelease = $False
Expand Down Expand Up @@ -96,7 +97,7 @@ function ParseMavenPackage($pkg, $workingDirectory) {

$changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0]
if ($changeLogLoc) {
$releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
}

$readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0]
Expand Down Expand Up @@ -169,23 +170,22 @@ function ParseNPMPackage($pkg, $workingDirectory) {
tar -xzf $pkg

$packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json
$pkgId = $packageJSON.name
$pkgVersion = $packageJSON.version

$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
if ($changeLogLoc) {
$releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
}

$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1
if ($readmeContentLoc) {
$readmeContent = Get-Content -Raw $readmeContentLoc
}

cd $origFolder
Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue

$pkgId = $packageJSON.name
$pkgVersion = $packageJSON.version

$resultObj = New-Object PSObject -Property @{
PackageId = $pkgId
PackageVersion = $pkgVersion
Expand Down Expand Up @@ -229,10 +229,12 @@ function ParseNugetPackage($pkg, $workingDirectory) {
Copy-Item -Path $pkg -Destination $zipFileLocation
Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder
[xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content
$pkgId = $packageXML.package.metadata.id
$pkgVersion = $packageXML.package.metadata.version

$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
if ($changeLogLoc) {
$releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
}

$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
Expand All @@ -241,8 +243,6 @@ function ParseNugetPackage($pkg, $workingDirectory) {
}

Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue
$pkgId = $packageXML.package.metadata.id
$pkgVersion = $packageXML.package.metadata.version

return New-Object PSObject -Property @{
PackageId = $pkgId
Expand Down Expand Up @@ -297,13 +297,15 @@ function ParsePyPIPackage($pkg, $workingDirectory) {

$changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
if ($changeLogLoc) {
$releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
}

$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
$readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1

if ($readmeContentLoc) {
$readmeContent = Get-Content -Raw $readmeContentLoc
}

Remove-Item $workFolder -Force -Recurse -ErrorAction SilentlyContinue

return New-Object PSObject -Property @{
Expand All @@ -321,10 +323,12 @@ function ParseCArtifact($pkg, $workingDirectory) {
$releaseNotes = ""
$readmeContent = ""

$pkgVersion = $packageInfo.version

$changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
if ($changeLogLoc)
{
$releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
$releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
}

$readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
Expand All @@ -334,7 +338,7 @@ function ParseCArtifact($pkg, $workingDirectory) {

return New-Object PSObject -Property @{
PackageId = ''
PackageVersion = $packageInfo.version
PackageVersion = $pkgVersion
# Artifact info is always considered deployable for C becasue it is not
# deployed anywhere. Dealing with duplicate tags happens downstream in
# CheckArtifactShaAgainstTagsList
Expand Down
1 change: 1 addition & 0 deletions eng/common/scripts/create-tags-and-git-release.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@ param (
# used by VerifyPackages
$artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
$workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary)
[ValidateSet("Nuget","NPM","PyPI","Maven")]
$packageRepository, # used to indicate destination against which we will check the existing version.
# valid options: PyPI, Nuget, NPM, Maven, C
# used by CreateTags
Expand Down
116 changes: 116 additions & 0 deletions eng/common/scripts/modules/ChangeLog-Operations.psm1
Original file line number Diff line number Diff line change
@@ -0,0 +1,116 @@
# Common Changelog Operations

$RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"

# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
function Get-ChangeLogEntries {
param (
[Parameter(Mandatory = $true)]
[String]$ChangeLogLocation
)

$changeLogEntries = @{}
if (!(Test-Path $ChangeLogLocation)) {
Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation
exit 1
}

try {
$contents = Get-Content $ChangeLogLocation
# walk the document, finding where the version specifiers are and creating lists
$changeLogEntry = $null
foreach ($line in $contents) {
if ($line -match $RELEASE_TITLE_REGEX) {
$changeLogEntry = [pscustomobject]@{
ReleaseVersion = $matches["version"]
ReleaseStatus = $matches["releaseStatus"]
ReleaseTitle = $line
ReleaseContent = @() # Release content without the version title
}
$changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry
}
else {
if ($changeLogEntry) {
$changeLogEntry.ReleaseContent += $line
}
}
}
}
catch {
Write-Host "Error parsing $ChangeLogLocation."
Write-Host $_.Exception.Message
}
return $changeLogEntries
}

# Returns single changeLogEntry object containing the ChangeLog for a particular version
function Get-ChangeLogEntry {
param (
[Parameter(Mandatory = $true)]
[String]$ChangeLogLocation,
[Parameter(Mandatory = $true)]
[String]$VersionString
)

$changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation

if ($changeLogEntries.ContainsKey($VersionString)) {
return $changeLogEntries[$VersionString]
}
Write-Error "Release Notes for the Specified version ${VersionString} was not found"
exit 1
}

#Returns the changelog for a particular version as string
function Get-ChangeLogEntryAsString {
param (
[Parameter(Mandatory = $true)]
[String]$ChangeLogLocation,
[Parameter(Mandatory = $true)]
[String]$VersionString
)

$changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
[string]$releaseTitle = $changeLogEntries.ReleaseTitle
[string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine
return $releaseTitle, $releaseContent -Join [Environment]::NewLine
}

function Confirm-ChangeLogEntry {
param (
[Parameter(Mandatory = $true)]
[String]$ChangeLogLocation,
[Parameter(Mandatory = $true)]
[String]$VersionString,
[boolean]$ForRelease = $false
)

$changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString

if ([System.String]::IsNullOrEmpty($changeLogEntries.ReleaseStatus)) {
Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation)
Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released"
exit 1
}

if ($ForRelease -eq $True) {
$CurrentDate = Get-Date -Format "yyyy-MM-dd"
if ($changeLogEntries.ReleaseStatus -ne "($CurrentDate)") {
Write-Host ("##[warning]Incorrect Date: Please use the current date in the Changelog '{0}' before releasing the package" -f $ChangeLogLocation)
exit 1
}

if ([System.String]::IsNullOrWhiteSpace($changeLogEntries.ReleaseContent)) {
Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation)
Write-Host "##[info]Please ensure there is a release notes entry before releasing the package."
exit 1
}
}

Write-Host ($changeLogEntries | Format-Table | Out-String)
}

Export-ModuleMember -Function 'Get-ChangeLogEntries'
Export-ModuleMember -Function 'Get-ChangeLogEntry'
Export-ModuleMember -Function 'Get-ChangeLogEntryAsString'
Export-ModuleMember -Function 'Confirm-ChangeLogEntry'
2 changes: 1 addition & 1 deletion eng/common/scripts/modules/common-manifest.psd1
Original file line number Diff line number Diff line change
Expand Up @@ -66,7 +66,7 @@ ScriptsToProcess = @("${PSScriptRoot}\..\SemVer.ps1")
# FormatsToProcess = @()

# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess
NestedModules = @("${PSScriptRoot}\Package-Properties.psm1")
NestedModules = @("${PSScriptRoot}\Package-Properties.psm1", "${PSScriptRoot}\ChangeLog-Operations.psm1")

# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export.
# FunctionsToExport = @()
Expand Down
18 changes: 10 additions & 8 deletions eng/common/scripts/update-docs-metadata.ps1
Original file line number Diff line number Diff line change
Expand Up @@ -22,19 +22,19 @@ param (
function GetMetaData($lang){
switch ($lang) {
"java" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/java-packages.csv"
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
break
}
".net" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/dotnet-packages.csv"
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv"
break
}
"python" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/python-packages.csv"
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv"
break
}
"javascript" {
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/js-packages.csv"
$metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv"
break
}
default {
Expand All @@ -61,7 +61,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
$service = $metadata | ? { $_.Package -eq $pkgId }

if ($service) {
$service = "$($service.Service)"
$service = "$($service.ServiceName)".ToLower().Replace(" ", "")
}
}
catch {
Expand All @@ -70,15 +70,17 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
}

$fileContent = $pkgInfo.ReadmeContent
$foundTitle = ""

# only replace the version if the formatted header can be found
$headerContentMatches = (Select-String -InputObject $pkgInfo.ReadmeContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET|C)')
if ($headerContentMatches) {
$headerContentMatch = $headerContentMatches.Matches[0]
$header = "---`ntitle: $headerContentMatch`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"
$fileContent = $pkgInfo.ReadmeContent -replace $headerContentMatch, "$headerContentMatch - Version $($pkgInfo.PackageVersion) `n"
$foundTitle = $headerContentMatches.Matches[0]
$fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n"
}

$header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"

if ($fileContent) {
return "$header`n$fileContent"
}
Expand Down

0 comments on commit eadfeb8

Please sign in to comment.