Sync eng/common directory with azure-sdk-tools repository

eng/common/TestResources/remove-test-resources.yml

@@ -32,4 +32,5 @@ steps:
         -Force `
         -Verbose
     displayName: Remove test resources
+    condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed())
     continueOnError: true

eng/common/pipelines/templates/steps/docs-metadata-release.yml

@@ -56,6 +56,3 @@ steps:
     BaseBranchName: smoke-test
     WorkingDirectory: ${{parameters.WorkingDirectory}}/repo
     ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}}
-
-
-

eng/common/scripts/artifact-metadata-parsing.ps1

@@ -358,7 +358,7 @@ function GetExistingTags($apiUrl) {
 }
 
 # Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases
-function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $exitOnError = $True) {
+function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha,  $continueOnError = $false) {
   $pkgList = [array]@()
   $ParsePkgInfoFn = ""
   $packagePattern = ""
@@ -404,7 +404,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
         continue
       }
 
-      if ($parsedPackage.Deployable -ne $True -and $exitOnError) {
+      if ($parsedPackage.Deployable -ne $True -and !$continueOnError) {
         Write-Host "Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version $($parsedPackage.PackageVersion) has already been deployed to the target repository."
         Write-Host "Maybe a pkg version wasn't updated properly?"
         exit(1)
@@ -430,8 +430,8 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
 
   $intersect = $results | % { $_.Tag } | ? { $existingTags -contains $_ }
 
-  if ($intersect.Length -gt 0 -and $exitOnError) {
-    CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -exitOnError $exitOnError
+  if ($intersect.Length -gt 0 -and !$continueOnError) {
+    CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -continueOnError $continueOnError
 
     # all the tags are clean. remove them from the list of releases we will publish.
     $results = $results | ? { -not ($intersect -contains $_.Tag ) }
@@ -443,7 +443,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a
 # given a set of tags that we want to release, we need to ensure that if they already DO exist.
 # if they DO exist, quietly exit if the commit sha of the artifact matches that of the tag
 # if the commit sha does not match, exit with error and report both problem shas
-function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $exitOnError) {
+function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $continueOnError) {
   $headers = @{
     "Content-Type"  = "application/json"
     "Authorization" = "token $($env:GH_TOKEN)"
@@ -465,7 +465,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap
     }
   }
 
-  if ($unmatchedTags.Length -gt 0 -and $exitOnError) {
+  if ($unmatchedTags.Length -gt 0 -and !$continueOnError) {
     Write-Host "Tags already existing with different SHA versions. Exiting."
     exit(1)
   }

eng/common/scripts/copy-docs-to-blobstorage.ps1

@@ -304,6 +304,17 @@ if ($Language -eq "java")
             jar -xf "$($Item.FullName)"
             Set-Location $CurrentLocation
 
+            # If javadocs are produced for a library with source, there will always be an
+            # index.html. If this file doesn't exist in the UnjarredDocumentationPath then
+            # this is a sourceless library which means there are no javadocs and nothing
+            # should be uploaded to blob storage.
+            $IndexHtml = Join-Path -Path $UnjarredDocumentationPath -ChildPath "index.html"
+            if (!(Test-Path -path $IndexHtml))
+            {
+                Write-Host "$($PkgName) does not have an index.html file, skippping."
+                continue
+            }
+
             # Get the POM file for the artifact we're processing
             $PomFile = $Item.FullName.Substring(0,$Item.FullName.LastIndexOf(("-javadoc.jar"))) + ".pom"
             Write-Host "PomFile $($PomFile)"

eng/common/scripts/create-tags-and-git-release.ps1

@@ -15,7 +15,7 @@ param (
   $repoOwner = "", # the owning organization of the repository. EG "Azure"
   $repoName = "", # the name of the repository. EG "azure-sdk-for-java"
   $repoId = "$repoOwner/$repoName", # full repo id. EG azure/azure-sdk-for-net  DevOps: $(Build.Repository.Id),
-  [switch]$forceCreate = $false
+  [switch]$continueOnError = $false
 )
 
 Write-Host "> $PSCommandPath $args"
@@ -26,7 +26,7 @@ $apiUrl = "https://api.github.com/repos/$repoId"
 Write-Host "Using API URL $apiUrl"
 
 # VERIFY PACKAGES
-$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha
+$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError
 
 if ($pkgList) {
   Write-Host "Given the visible artifacts, github releases will be created for the following:"

eng/common/scripts/modules/Package-Properties.psm1

@@ -0,0 +1,252 @@
+# Helper functions for retireving useful information from azure-sdk-for-* repo
+# Example Use : Import-Module .\eng\common\scripts\modules
+class PackageProps
+{
+    [string]$pkgName
+    [AzureEngSemanticVersion]$pkgVersion
+    [string]$pkgDirectoryPath
+    [string]$pkgServiceName
+    [string]$pkgReadMePath
+    [string]$pkgChangeLogPath
+
+    PackageProps(
+        [string]$pkgName,
+        [string]$pkgVersion,
+        [string]$pkgDirectoryPath,
+        [string]$pkgServiceName
+    )
+    {
+        $this.pkgName = $pkgName
+        $this.pkgVersion = [AzureEngSemanticVersion]::ParseVersionString($pkgVersion)
+        if ($this.pkgVersion -eq $null)
+        {
+            Write-Error "Invalid version in $pkgDirectoryPath"
+        }
+        $this.pkgDirectoryPath = $pkgDirectoryPath
+        $this.pkgServiceName = $pkgServiceName
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "README.md"))
+        {
+            $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md"
+        } 
+        else
+        {
+            $this.pkgReadMePath = $null
+        }
+
+        if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md"))
+        {
+            $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md"
+        } 
+        else
+        {
+            $this.pkgChangeLogPath = $null
+        }
+    }
+}
+
+Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser
+
+function Extract-PkgProps ($pkgPath, $serviceName, $pkgName, $lang)
+{
+    if ($lang -eq "net")
+    {
+        return Extract-DotNetPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName 
+    }
+    if ($lang -eq "java")
+    {
+        return Extract-JavaPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName 
+    }
+    if ($lang -eq "js")
+    {
+        return Extract-JsPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName 
+    }
+    if ($lang -eq "python")
+    {
+        return Extract-PythonPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName 
+    }
+}
+
+function Extract-DotNetPkgProps ($pkgPath, $serviceName, $pkgName)
+{
+    $projectPath = Join-Path $pkgPath "src" "$pkgName.csproj"
+    if (Test-Path $projectPath)
+    {
+        $projectData = New-Object -TypeName XML
+        $projectData.load($projectPath)
+        $pkgVersion = Select-XML -Xml $projectData -XPath '/Project/PropertyGroup/Version'
+        return [PackageProps]::new($pkgName, $pkgVersion, $pkgPath, $serviceName)
+    } 
+    else 
+    {
+        return $null
+    }
+}
+
+function Extract-JsPkgProps ($pkgPath, $serviceName, $pkgName)
+{
+    $projectPath = Join-Path $pkgPath "package.json"
+    if (Test-Path $projectPath)
+    {
+        $projectJson = Get-Content $projectPath | ConvertFrom-Json
+        $jsStylePkgName = $pkgName.replace("azure-", "@azure/")
+        if ($projectJson.name -eq "$jsStylePkgName")
+        {
+            return [PackageProps]::new($projectJson.name, $projectJson.version, $pkgPath, $serviceName)
+        }
+    }
+    return $null
+}
+
+function Extract-PythonPkgProps ($pkgPath, $serviceName, $pkgName)
+{
+    $pkgName = $pkgName.Replace('_', '-')
+
+    if (Test-Path (Join-Path $pkgPath "setup.py"))
+    {
+        $setupLocation = $pkgPath.Replace('\','/')
+        pushd $RepoRoot
+        $setupProps = (python -c "import scripts.devops_tasks.common_tasks; obj=scripts.devops_tasks.common_tasks.parse_setup('$setupLocation'); print('{0},{1}'.format(obj[0], obj[1]));") -split ","
+        popd
+        if (($setupProps -ne $null) -and ($setupProps[0] -eq $pkgName))
+        {
+            return [PackageProps]::new($setupProps[0], $setupProps[1], $pkgPath, $serviceName)
+        }
+    }
+    return $null
+}
+
+function Extract-JavaPkgProps ($pkgPath, $serviceName, $pkgName)
+{
+    $projectPath = Join-Path $pkgPath "pom.xml"
+
+    if (Test-Path $projectPath)
+    {
+        $projectData = New-Object -TypeName XML
+        $projectData.load($projectPath)
+        $projectPkgName = $projectData.project.artifactId
+        $pkgVersion = $projectData.project.version
+
+        if ($projectPkgName -eq $pkgName)
+        {
+            return [PackageProps]::new($pkgName, $pkgVersion.ToString(), $pkgPath, $serviceName)
+        }
+    }
+    return $null
+}
+
+# Takes package name and service Name
+# Returns important properties of the package as related to the language repo
+# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+# Note: python is required for parsing python package properties.
+function Get-PkgProperties
+{
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [string]$PackageName,
+        [Parameter(Mandatory=$true)]
+        [string]$ServiceName,
+        [Parameter(Mandatory=$true)]
+        [ValidateSet("net","java","js","python")]
+        [string]$Language,
+        [string]$RepoRoot="${PSScriptRoot}/../../../.."
+    )
+
+    $pkgDirectoryName = $null
+    $pkgDirectoryPath = $null
+    $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName
+    if (!(Test-Path $serviceDirectoryPath))
+    {
+        Write-Error "Service Directory $ServiceName does not exist"
+        exit 1
+    }
+
+    $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory
+
+    foreach ($directory in $directoriesPresent)
+    {
+        $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name
+        $pkgProps = Extract-PkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName -lang $Language
+        if ($pkgProps -ne $null)
+        {
+            return $pkgProps
+        }
+    }
+    Write-Error "Failed to retrive Properties for $PackageName"
+}
+
+# Takes ServiceName, Language, and Repo Root Directory
+# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified
+# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath }
+function Get-AllPkgProperties
+{
+    Param
+    (
+        [Parameter(Mandatory=$true)]
+        [ValidateSet("net","java","js","python")]
+        [string]$Language,
+        [string]$RepoRoot="${PSScriptRoot}/../../../..",
+        [string]$ServiceName=$null
+    )
+
+    $pkgPropsResult = @()
+
+    if ([string]::IsNullOrEmpty($ServiceName))
+    {
+        $searchDir = Join-Path $RepoRoot "sdk"
+        foreach ($dir in (Get-ChildItem $searchDir -Directory))
+        {
+            $serviceDir = Join-Path $searchDir $dir.Name
+
+            if (Test-Path (Join-Path $serviceDir "ci.yml"))
+            {
+                $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+                if ($activePkgList -ne $null)
+                {
+                    $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -language $Language -repoRoot $RepoRoot -pkgPropsResult $pkgPropsResult
+                }
+            }
+        }
+    } 
+    else
+    {
+        $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName
+        if (Test-Path (Join-Path $serviceDir "ci.yml"))
+        {
+            $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml")
+            if ($activePkgList -ne $null)
+            {
+                $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -language $Language -repoRoot $RepoRoot -pkgPropsResult $pkgPropsResult
+            }
+        }
+    }
+
+    return $pkgPropsResult
+}
+
+function Operate-OnPackages ($activePkgList, $serviceName, $language, $repoRoot, [Array]$pkgPropsResult)
+{
+    foreach ($pkg in $activePkgList)
+    {
+        $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName -Language $language -RepoRoot $repoRoot
+        $pkgPropsResult += $pkgProps
+    }
+    return $pkgPropsResult
+}
+
+function Get-PkgListFromYml ($ciYmlPath)
+{
+    $ciYmlContent = Get-Content $ciYmlPath -Raw
+    $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered
+    $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"]
+
+    if ($artifactsInCI -eq $null)
+    {
+        Write-Error "Failed to retrive package names in ci $ciYmlPath"
+    }
+    return $artifactsInCI
+}
+
+Export-ModuleMember -Function 'Get-PkgProperties'
+Export-ModuleMember -Function 'Get-AllPkgProperties'