Sync eng/common directory with azure-sdk-tools repository

eng/common/pipelines/templates/steps/verify-agent-os.yml

@@ -1,4 +1,3 @@
-# Template for all Python Scripts in this repository
 parameters: 
   OSVmImage: $(OSVmImage)
 

eng/common/pipelines/templates/steps/verify-changelog.yml

@@ -0,0 +1,25 @@
+parameters:
+- name: PackageName
+  type: string
+  default: 'not-specified'
+- name: ServiceName
+  type: string
+  default: 'not-specified'
+- name: ForRelease
+  type: boolean
+  default: false
+
+steps:
+  - task: Powershell@2
+    inputs:
+      filePath: /eng/common/scripts/Verify-ChangeLog.ps1
+      arguments: >
+        -PackageName ${{ parameters.PackageName }}
+        -ServiceName ${{ parameters.ServiceName }}
+        -RepoRoot $(Build.SourcesDirectory)
+        -RepoName $(Build.Repository.Name)
+        -ForRelease ${{ parameters.ForRelease }}
+      pwsh: true
+      workingDirectory: $(Pipeline.Workspace)
+    displayName: Verify ChangeLog / Release Notes
+    continueOnError: false

eng/common/scripts/Verify-ChangeLog.ps1

@@ -0,0 +1,22 @@
+# Wrapper Script for ChangeLog Verification
+param (
+    [Parameter(Mandatory=$true)]
+    [string]$PackageName,
+    [Parameter(Mandatory=$true)]
+    [string]$ServiceName,
+    [string]$RepoRoot,
+    [ValidateSet("net","java","js","python")]
+    [string]$Language,
+    [string]$RepoName,
+    [boolean]$ForRelease=$False
+)
+
+Import-Module "${PSScriptRoot}/modules/common-manifest.psd1"
+
+if ([System.String]::IsNullOrEmpty($Language))
+{
+    $Language = $RepoName.Substring($RepoName.LastIndexOf('-') + 1)
+}
+
+$PackageProp = Get-PkgProperties -PackageName $PackageName -ServiceName $ServiceName -Language $Language -RepoRoot $RepoRoot
+Confirm-ChangeLog -ChangeLogLocation $PackageProp.pkgChangeLogPath -VersionString $PackageProp.pkgReadMePath -ForRelease $ForRelease

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -1,3 +1,4 @@
+Import-Module "${PSScriptRoot}/modules/ChangeLog-Operations.psm1"
 . (Join-Path $PSScriptRoot SemVer.ps1)
 
 $SDIST_PACKAGE_REGEX = "^(?<package>.*)\-(?<versionstring>$([AzureEngSemanticVersion]::SEMVER_REGEX))"
@@ -8,8 +9,8 @@ function CreateReleases($pkgList, $releaseApiUrl, $releaseSha) {
     Write-Host "Creating release $($pkgInfo.Tag)"
 
     $releaseNotes = ""
-    if ($pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent -ne $null) {
-      $releaseNotes = $pkgInfo.ReleaseNotes[$pkgInfo.PackageVersion].ReleaseContent
+    if ($pkgInfo.ReleaseNotes -ne $null) {
+      $releaseNotes = $pkgInfo.ReleaseNotes
     }
 
     $isPrerelease = $False
@@ -96,7 +97,7 @@ function ParseMavenPackage($pkg, $workingDirectory) {
 
   $changeLogLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-changelog.md")[0]
   if ($changeLogLoc) {
-    $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
   }
 
   $readmeContentLoc = @(Get-ChildItem -Path $pkg.DirectoryName -Recurse -Include "$($pkg.Basename)-readme.md")[0]
@@ -169,23 +170,22 @@ function ParseNPMPackage($pkg, $workingDirectory) {
   tar -xzf $pkg
 
   $packageJSON = ResolvePkgJson -workFolder $workFolder | Get-Content | ConvertFrom-Json
+  $pkgId = $packageJSON.name
+  $pkgVersion = $packageJSON.version
 
   $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
   if ($changeLogLoc) {
-    $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
   }
 
-  $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
+  $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1
   if ($readmeContentLoc) {
     $readmeContent = Get-Content -Raw $readmeContentLoc
   }
 
   cd $origFolder
   Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
 
-  $pkgId = $packageJSON.name
-  $pkgVersion = $packageJSON.version
-
   $resultObj = New-Object PSObject -Property @{
     PackageId      = $pkgId
     PackageVersion = $pkgVersion
@@ -229,10 +229,12 @@ function ParseNugetPackage($pkg, $workingDirectory) {
   Copy-Item -Path $pkg -Destination $zipFileLocation
   Expand-Archive -Path $zipFileLocation -DestinationPath $workFolder
   [xml] $packageXML = Get-ChildItem -Path "$workFolder/*.nuspec" | Get-Content
+  $pkgId = $packageXML.package.metadata.id
+  $pkgVersion = $packageXML.package.metadata.version
 
   $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
   if ($changeLogLoc) {
-    $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
   }
 
   $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
@@ -241,8 +243,6 @@ function ParseNugetPackage($pkg, $workingDirectory) {
   }
 
   Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
-  $pkgId = $packageXML.package.metadata.id
-  $pkgVersion = $packageXML.package.metadata.version
 
   return New-Object PSObject -Property @{
     PackageId      = $pkgId
@@ -297,13 +297,15 @@ function ParsePyPIPackage($pkg, $workingDirectory) {
 
   $changeLogLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "CHANGELOG.md")[0]
   if ($changeLogLoc) {
-    $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
   }
 
-  $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md")[0]
+  $readmeContentLoc = @(Get-ChildItem -Path $workFolder -Recurse -Include "README.md") | Select-Object -Last 1
+
   if ($readmeContentLoc) {
     $readmeContent = Get-Content -Raw $readmeContentLoc
   }
+
   Remove-Item $workFolder -Force  -Recurse -ErrorAction SilentlyContinue
 
   return New-Object PSObject -Property @{
@@ -321,10 +323,12 @@ function ParseCArtifact($pkg, $workingDirectory) {
   $releaseNotes = ""
   $readmeContent = ""
 
+  $pkgVersion = $packageInfo.version
+
   $changeLogLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "CHANGELOG.md")[0]
   if ($changeLogLoc)
   {
-    $releaseNotes = &"${PSScriptRoot}/../Extract-ReleaseNotes.ps1" -ChangeLogLocation $changeLogLoc
+    $releaseNotes = Get-ChangeLogEntryAsString -ChangeLogLocation $changeLogLoc -VersionString $pkgVersion
   }
 
   $readmeContentLoc = @(Get-ChildItem -Path $packageArtifactLocation -Recurse -Include "README.md")[0]
@@ -334,7 +338,7 @@ function ParseCArtifact($pkg, $workingDirectory) {
 
   return New-Object PSObject -Property @{
     PackageId      = ''
-    PackageVersion = $packageInfo.version
+    PackageVersion = $pkgVersion
     # Artifact info is always considered deployable for C becasue it is not
     # deployed anywhere. Dealing with duplicate tags happens downstream in
     # CheckArtifactShaAgainstTagsList

eng/common/scripts/create-tags-and-git-release.ps1

@@ -6,6 +6,7 @@ param (
   # used by VerifyPackages
   $artifactLocation, # the root of the artifact folder. DevOps $(System.ArtifactsDirectory)
   $workingDirectory, # directory that package artifacts will be extracted into for examination (if necessary)
+  [ValidateSet("Nuget","NPM","PyPI","Maven")]
   $packageRepository, # used to indicate destination against which we will check the existing version.
   # valid options: PyPI, Nuget, NPM, Maven, C
   # used by CreateTags

eng/common/scripts/modules/ChangeLog-Operations.psm1

@@ -0,0 +1,116 @@
+# Common Changelog Operations
+
+$RELEASE_TITLE_REGEX = "(?<releaseNoteTitle>^\#+.*(?<version>\b\d+\.\d+\.\d+([^0-9\s][^\s:]+)?)(\s(?<releaseStatus>\(Unreleased\)|\(\d{4}-\d{2}-\d{2}\)))?)"
+
+# Returns a Collection of changeLogEntry object containing changelog info for all version present in the gived CHANGELOG
+function Get-ChangeLogEntries {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation
+  )
+
+  $changeLogEntries = @{}
+  if (!(Test-Path $ChangeLogLocation)) {
+    Write-Host "ChangeLog '{0}' was not found" -f $ChangeLogLocation
+    exit 1
+  }
+
+  try {
+    $contents = Get-Content $ChangeLogLocation
+    # walk the document, finding where the version specifiers are and creating lists
+    $changeLogEntry = $null
+    foreach ($line in $contents) {
+      if ($line -match $RELEASE_TITLE_REGEX) {
+        $changeLogEntry = [pscustomobject]@{ 
+          ReleaseVersion = $matches["version"]
+          ReleaseStatus  = $matches["releaseStatus"]
+          ReleaseTitle   = $line
+          ReleaseContent = @() # Release content without the version title
+        }
+        $changeLogEntries[$changeLogEntry.ReleaseVersion] = $changeLogEntry
+      }
+      else {
+        if ($changeLogEntry) {
+          $changeLogEntry.ReleaseContent += $line
+        }
+      }
+    }
+  }
+  catch {
+    Write-Host "Error parsing $ChangeLogLocation."
+    Write-Host $_.Exception.Message
+  }
+  return $changeLogEntries
+}
+
+# Returns single changeLogEntry object containing the ChangeLog for a particular version
+function Get-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+
+  $changeLogEntries = Get-ChangeLogEntries -ChangeLogLocation $ChangeLogLocation
+
+  if ($changeLogEntries.ContainsKey($VersionString)) {
+    return $changeLogEntries[$VersionString]
+  }
+  Write-Error "Release Notes for the Specified version ${VersionString} was not found"
+  exit 1
+}
+
+#Returns the changelog for a particular version as string
+function Get-ChangeLogEntryAsString {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString
+  )
+
+  $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+  [string]$releaseTitle = $changeLogEntries.ReleaseTitle
+  [string]$releaseContent = $changeLogEntries.ReleaseContent -Join [Environment]::NewLine
+  return $releaseTitle, $releaseContent -Join [Environment]::NewLine
+}
+
+function Confirm-ChangeLogEntry {
+  param (
+    [Parameter(Mandatory = $true)]
+    [String]$ChangeLogLocation,
+    [Parameter(Mandatory = $true)]
+    [String]$VersionString,
+    [boolean]$ForRelease = $false
+  )
+
+  $changeLogEntries = Get-ChangeLogEntry -ChangeLogLocation $ChangeLogLocation -VersionString $VersionString
+
+  if ([System.String]::IsNullOrEmpty($changeLogEntries.ReleaseStatus)) {
+    Write-Host ("##[error]Changelog '{0}' has wrong release note title" -f $ChangeLogLocation)
+    Write-Host "##[info]Ensure the release date is included i.e. (yyyy-MM-dd) or (Unreleased) if not yet released"
+    exit 1
+  }
+
+  if ($ForRelease -eq $True) {
+    $CurrentDate = Get-Date -Format "yyyy-MM-dd"
+    if ($changeLogEntries.ReleaseStatus -ne "($CurrentDate)") {
+      Write-Host ("##[warning]Incorrect Date: Please use the current date in the Changelog '{0}' before releasing the package" -f $ChangeLogLocation)
+      exit 1
+    }
+
+    if ([System.String]::IsNullOrWhiteSpace($changeLogEntries.ReleaseContent)) {
+      Write-Host ("##[error]Empty Release Notes for '{0}' in '{1}'" -f $VersionString, $ChangeLogLocation)
+      Write-Host "##[info]Please ensure there is a release notes entry before releasing the package."
+      exit 1
+    }
+  }
+
+  Write-Host ($changeLogEntries | Format-Table | Out-String)
+}
+
+Export-ModuleMember -Function 'Get-ChangeLogEntries'
+Export-ModuleMember -Function 'Get-ChangeLogEntry'
+Export-ModuleMember -Function 'Get-ChangeLogEntryAsString'
+Export-ModuleMember -Function 'Confirm-ChangeLogEntry'

eng/common/scripts/modules/common-manifest.psd1

@@ -66,7 +66,7 @@ ScriptsToProcess = @("${PSScriptRoot}\..\SemVer.ps1")
 # FormatsToProcess = @()
 
 # Modules to import as nested modules of the module specified in RootModule/ModuleToProcess
-NestedModules = @("${PSScriptRoot}\Package-Properties.psm1")
+NestedModules = @("${PSScriptRoot}\Package-Properties.psm1", "${PSScriptRoot}\ChangeLog-Operations.psm1")
 
 # Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export.
 # FunctionsToExport = @()

eng/common/scripts/update-docs-metadata.ps1

@@ -22,19 +22,19 @@ param (
 function GetMetaData($lang){
   switch ($lang) {
     "java" {
-      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/java-packages.csv"
+      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/java-packages.csv"
       break
     }
     ".net" {
-      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/dotnet-packages.csv"
+      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/dotnet-packages.csv"
       break
     }
     "python" {
-      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/python-packages.csv"
+      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/python-packages.csv"
       break
     }
     "javascript" {
-      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/allpackages/js-packages.csv"
+      $metadataUri = "https://raw.githubusercontent.com/Azure/azure-sdk/master/_data/releases/latest/js-packages.csv"
       break
     }
     default {
@@ -61,7 +61,7 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
       $service = $metadata | ? { $_.Package -eq $pkgId }
 
       if ($service) {
-        $service = "$($service.Service)"
+        $service = "$($service.ServiceName)".ToLower().Replace(" ", "")
       }
     }
     catch {
@@ -70,15 +70,17 @@ function GetAdjustedReadmeContent($pkgInfo, $lang){
     }
 
     $fileContent = $pkgInfo.ReadmeContent
+    $foundTitle = ""
 
     # only replace the version if the formatted header can be found
     $headerContentMatches = (Select-String -InputObject $pkgInfo.ReadmeContent -Pattern 'Azure .+? (client|plugin|shared) library for (JavaScript|Java|Python|\.NET|C)')
     if ($headerContentMatches) {
-      $headerContentMatch = $headerContentMatches.Matches[0]
-      $header = "---`ntitle: $headerContentMatch`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"
-      $fileContent = $pkgInfo.ReadmeContent -replace $headerContentMatch, "$headerContentMatch - Version $($pkgInfo.PackageVersion) `n"
+      $foundTitle = $headerContentMatches.Matches[0]
+      $fileContent = $pkgInfo.ReadmeContent -replace $foundTitle, "$foundTitle - Version $($pkgInfo.PackageVersion) `n"
     }
 
+    $header = "---`ntitle: $foundTitle`nkeywords: Azure, $lang, SDK, API, $($pkgInfo.PackageId), $service`nauthor: maggiepint`nms.author: magpint`nms.date: $date`nms.topic: article`nms.prod: azure`nms.technology: azure`nms.devlang: $lang`nms.service: $service`n---`n"
+
     if ($fileContent) {
       return "$header`n$fileContent"
     }