From c6107651588f31457baee757d39c882699aa9159 Mon Sep 17 00:00:00 2001 From: azure-sdk Date: Wed, 20 May 2020 23:51:24 +0000 Subject: [PATCH] Sync eng/common directory with azure-sdk-tools repository --- .../TestResources/remove-test-resources.yml | 1 + .../templates/steps/docs-metadata-release.yml | 3 - .../scripts/artifact-metadata-parsing.ps1 | 12 +- .../scripts/copy-docs-to-blobstorage.ps1 | 11 + .../scripts/create-tags-and-git-release.ps1 | 4 +- .../scripts/modules/Package-Properties.psm1 | 252 ++++++++++++++++++ .../scripts/modules/common-manifest.psd1 | 123 +++++++++ eng/common/scripts/update-docs-metadata.ps1 | 2 +- 8 files changed, 396 insertions(+), 12 deletions(-) create mode 100644 eng/common/scripts/modules/Package-Properties.psm1 create mode 100644 eng/common/scripts/modules/common-manifest.psd1 diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index e66933b60cc6..1565e7d4bd0f 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -32,4 +32,5 @@ steps: -Force ` -Verbose displayName: Remove test resources + condition: and(ne(variables['AZURE_RESOURCEGROUP_NAME'], ''), succeededOrFailed()) continueOnError: true diff --git a/eng/common/pipelines/templates/steps/docs-metadata-release.yml b/eng/common/pipelines/templates/steps/docs-metadata-release.yml index 3ffdd5ec60a6..89d12d4ac1d1 100644 --- a/eng/common/pipelines/templates/steps/docs-metadata-release.yml +++ b/eng/common/pipelines/templates/steps/docs-metadata-release.yml @@ -56,6 +56,3 @@ steps: BaseBranchName: smoke-test WorkingDirectory: ${{parameters.WorkingDirectory}}/repo ScriptDirectory: ${{parameters.WorkingDirectory}}/${{parameters.ScriptDirectory}} - - - diff --git a/eng/common/scripts/artifact-metadata-parsing.ps1 b/eng/common/scripts/artifact-metadata-parsing.ps1 index 51ee8eb56b84..66e090ed6f6d 100644 --- a/eng/common/scripts/artifact-metadata-parsing.ps1 +++ b/eng/common/scripts/artifact-metadata-parsing.ps1 @@ -358,7 +358,7 @@ function GetExistingTags($apiUrl) { } # Walk across all build artifacts, check them against the appropriate repository, return a list of tags/releases -function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $exitOnError = $True) { +function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $apiUrl, $releaseSha, $continueOnError = $false) { $pkgList = [array]@() $ParsePkgInfoFn = "" $packagePattern = "" @@ -404,7 +404,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a continue } - if ($parsedPackage.Deployable -ne $True -and $exitOnError) { + if ($parsedPackage.Deployable -ne $True -and !$continueOnError) { Write-Host "Package $($parsedPackage.PackageId) is marked with version $($parsedPackage.PackageVersion), the version $($parsedPackage.PackageVersion) has already been deployed to the target repository." Write-Host "Maybe a pkg version wasn't updated properly?" exit(1) @@ -430,8 +430,8 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a $intersect = $results | % { $_.Tag } | ? { $existingTags -contains $_ } - if ($intersect.Length -gt 0 -and $exitOnError) { - CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -exitOnError $exitOnError + if ($intersect.Length -gt 0 -and !$continueOnError) { + CheckArtifactShaAgainstTagsList -priorExistingTagList $intersect -releaseSha $releaseSha -apiUrl $apiUrl -continueOnError $continueOnError # all the tags are clean. remove them from the list of releases we will publish. $results = $results | ? { -not ($intersect -contains $_.Tag ) } @@ -443,7 +443,7 @@ function VerifyPackages($pkgRepository, $artifactLocation, $workingDirectory, $a # given a set of tags that we want to release, we need to ensure that if they already DO exist. # if they DO exist, quietly exit if the commit sha of the artifact matches that of the tag # if the commit sha does not match, exit with error and report both problem shas -function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $exitOnError) { +function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $apiUrl, $continueOnError) { $headers = @{ "Content-Type" = "application/json" "Authorization" = "token $($env:GH_TOKEN)" @@ -465,7 +465,7 @@ function CheckArtifactShaAgainstTagsList($priorExistingTagList, $releaseSha, $ap } } - if ($unmatchedTags.Length -gt 0 -and $exitOnError) { + if ($unmatchedTags.Length -gt 0 -and !$continueOnError) { Write-Host "Tags already existing with different SHA versions. Exiting." exit(1) } diff --git a/eng/common/scripts/copy-docs-to-blobstorage.ps1 b/eng/common/scripts/copy-docs-to-blobstorage.ps1 index 3d097abab8b2..920bc392fa24 100644 --- a/eng/common/scripts/copy-docs-to-blobstorage.ps1 +++ b/eng/common/scripts/copy-docs-to-blobstorage.ps1 @@ -304,6 +304,17 @@ if ($Language -eq "java") jar -xf "$($Item.FullName)" Set-Location $CurrentLocation + # If javadocs are produced for a library with source, there will always be an + # index.html. If this file doesn't exist in the UnjarredDocumentationPath then + # this is a sourceless library which means there are no javadocs and nothing + # should be uploaded to blob storage. + $IndexHtml = Join-Path -Path $UnjarredDocumentationPath -ChildPath "index.html" + if (!(Test-Path -path $IndexHtml)) + { + Write-Host "$($PkgName) does not have an index.html file, skippping." + continue + } + # Get the POM file for the artifact we're processing $PomFile = $Item.FullName.Substring(0,$Item.FullName.LastIndexOf(("-javadoc.jar"))) + ".pom" Write-Host "PomFile $($PomFile)" diff --git a/eng/common/scripts/create-tags-and-git-release.ps1 b/eng/common/scripts/create-tags-and-git-release.ps1 index 56e3f22a4274..f87c90997839 100644 --- a/eng/common/scripts/create-tags-and-git-release.ps1 +++ b/eng/common/scripts/create-tags-and-git-release.ps1 @@ -15,7 +15,7 @@ param ( $repoOwner = "", # the owning organization of the repository. EG "Azure" $repoName = "", # the name of the repository. EG "azure-sdk-for-java" $repoId = "$repoOwner/$repoName", # full repo id. EG azure/azure-sdk-for-net DevOps: $(Build.Repository.Id), - [switch]$forceCreate = $false + [switch]$continueOnError = $false ) Write-Host "> $PSCommandPath $args" @@ -26,7 +26,7 @@ $apiUrl = "https://api.github.com/repos/$repoId" Write-Host "Using API URL $apiUrl" # VERIFY PACKAGES -$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha +$pkgList = VerifyPackages -pkgRepository $packageRepository -artifactLocation $artifactLocation -workingDirectory $workingDirectory -apiUrl $apiUrl -releaseSha $releaseSha -continueOnError $continueOnError if ($pkgList) { Write-Host "Given the visible artifacts, github releases will be created for the following:" diff --git a/eng/common/scripts/modules/Package-Properties.psm1 b/eng/common/scripts/modules/Package-Properties.psm1 new file mode 100644 index 000000000000..294f6609dab8 --- /dev/null +++ b/eng/common/scripts/modules/Package-Properties.psm1 @@ -0,0 +1,252 @@ +# Helper functions for retireving useful information from azure-sdk-for-* repo +# Example Use : Import-Module .\eng\common\scripts\modules +class PackageProps +{ + [string]$pkgName + [AzureEngSemanticVersion]$pkgVersion + [string]$pkgDirectoryPath + [string]$pkgServiceName + [string]$pkgReadMePath + [string]$pkgChangeLogPath + + PackageProps( + [string]$pkgName, + [string]$pkgVersion, + [string]$pkgDirectoryPath, + [string]$pkgServiceName + ) + { + $this.pkgName = $pkgName + $this.pkgVersion = [AzureEngSemanticVersion]::ParseVersionString($pkgVersion) + if ($this.pkgVersion -eq $null) + { + Write-Error "Invalid version in $pkgDirectoryPath" + } + $this.pkgDirectoryPath = $pkgDirectoryPath + $this.pkgServiceName = $pkgServiceName + + if (Test-Path (Join-Path $pkgDirectoryPath "README.md")) + { + $this.pkgReadMePath = Join-Path $pkgDirectoryPath "README.md" + } + else + { + $this.pkgReadMePath = $null + } + + if (Test-Path (Join-Path $pkgDirectoryPath "CHANGELOG.md")) + { + $this.pkgChangeLogPath = Join-Path $pkgDirectoryPath "CHANGELOG.md" + } + else + { + $this.pkgChangeLogPath = $null + } + } +} + +Install-Module -Name powershell-yaml -RequiredVersion 0.4.1 -Force -Scope CurrentUser + +function Extract-PkgProps ($pkgPath, $serviceName, $pkgName, $lang) +{ + if ($lang -eq "net") + { + return Extract-DotNetPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName + } + if ($lang -eq "java") + { + return Extract-JavaPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName + } + if ($lang -eq "js") + { + return Extract-JsPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName + } + if ($lang -eq "python") + { + return Extract-PythonPkgProps -pkgPath $pkgPath -serviceName $serviceName -pkgName $pkgName + } +} + +function Extract-DotNetPkgProps ($pkgPath, $serviceName, $pkgName) +{ + $projectPath = Join-Path $pkgPath "src" "$pkgName.csproj" + if (Test-Path $projectPath) + { + $projectData = New-Object -TypeName XML + $projectData.load($projectPath) + $pkgVersion = Select-XML -Xml $projectData -XPath '/Project/PropertyGroup/Version' + return [PackageProps]::new($pkgName, $pkgVersion, $pkgPath, $serviceName) + } + else + { + return $null + } +} + +function Extract-JsPkgProps ($pkgPath, $serviceName, $pkgName) +{ + $projectPath = Join-Path $pkgPath "package.json" + if (Test-Path $projectPath) + { + $projectJson = Get-Content $projectPath | ConvertFrom-Json + $jsStylePkgName = $pkgName.replace("azure-", "@azure/") + if ($projectJson.name -eq "$jsStylePkgName") + { + return [PackageProps]::new($projectJson.name, $projectJson.version, $pkgPath, $serviceName) + } + } + return $null +} + +function Extract-PythonPkgProps ($pkgPath, $serviceName, $pkgName) +{ + $pkgName = $pkgName.Replace('_', '-') + + if (Test-Path (Join-Path $pkgPath "setup.py")) + { + $setupLocation = $pkgPath.Replace('\','/') + pushd $RepoRoot + $setupProps = (python -c "import scripts.devops_tasks.common_tasks; obj=scripts.devops_tasks.common_tasks.parse_setup('$setupLocation'); print('{0},{1}'.format(obj[0], obj[1]));") -split "," + popd + if (($setupProps -ne $null) -and ($setupProps[0] -eq $pkgName)) + { + return [PackageProps]::new($setupProps[0], $setupProps[1], $pkgPath, $serviceName) + } + } + return $null +} + +function Extract-JavaPkgProps ($pkgPath, $serviceName, $pkgName) +{ + $projectPath = Join-Path $pkgPath "pom.xml" + + if (Test-Path $projectPath) + { + $projectData = New-Object -TypeName XML + $projectData.load($projectPath) + $projectPkgName = $projectData.project.artifactId + $pkgVersion = $projectData.project.version + + if ($projectPkgName -eq $pkgName) + { + return [PackageProps]::new($pkgName, $pkgVersion.ToString(), $pkgPath, $serviceName) + } + } + return $null +} + +# Takes package name and service Name +# Returns important properties of the package as related to the language repo +# Returns a PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +# Note: python is required for parsing python package properties. +function Get-PkgProperties +{ + Param + ( + [Parameter(Mandatory=$true)] + [string]$PackageName, + [Parameter(Mandatory=$true)] + [string]$ServiceName, + [Parameter(Mandatory=$true)] + [ValidateSet("net","java","js","python")] + [string]$Language, + [string]$RepoRoot="${PSScriptRoot}/../../../.." + ) + + $pkgDirectoryName = $null + $pkgDirectoryPath = $null + $serviceDirectoryPath = Join-Path $RepoRoot "sdk" $ServiceName + if (!(Test-Path $serviceDirectoryPath)) + { + Write-Error "Service Directory $ServiceName does not exist" + exit 1 + } + + $directoriesPresent = Get-ChildItem $serviceDirectoryPath -Directory + + foreach ($directory in $directoriesPresent) + { + $pkgDirectoryPath = Join-Path $serviceDirectoryPath $directory.Name + $pkgProps = Extract-PkgProps -pkgPath $pkgDirectoryPath -serviceName $ServiceName -pkgName $PackageName -lang $Language + if ($pkgProps -ne $null) + { + return $pkgProps + } + } + Write-Error "Failed to retrive Properties for $PackageName" +} + +# Takes ServiceName, Language, and Repo Root Directory +# Returns important properties for each package in the specified service, or entire repo if the serviceName is not specified +# Returns an Table of service key to array values of PS Object with properties @ { pkgName, pkgVersion, pkgDirectoryPath, pkgReadMePath, pkgChangeLogPath } +function Get-AllPkgProperties +{ + Param + ( + [Parameter(Mandatory=$true)] + [ValidateSet("net","java","js","python")] + [string]$Language, + [string]$RepoRoot="${PSScriptRoot}/../../../..", + [string]$ServiceName=$null + ) + + $pkgPropsResult = @() + + if ([string]::IsNullOrEmpty($ServiceName)) + { + $searchDir = Join-Path $RepoRoot "sdk" + foreach ($dir in (Get-ChildItem $searchDir -Directory)) + { + $serviceDir = Join-Path $searchDir $dir.Name + + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $dir.Name -language $Language -repoRoot $RepoRoot -pkgPropsResult $pkgPropsResult + } + } + } + } + else + { + $serviceDir = Join-Path $RepoRoot "sdk" $ServiceName + if (Test-Path (Join-Path $serviceDir "ci.yml")) + { + $activePkgList = Get-PkgListFromYml -ciYmlPath (Join-Path $serviceDir "ci.yml") + if ($activePkgList -ne $null) + { + $pkgPropsResult = Operate-OnPackages -activePkgList $activePkgList -serviceName $ServiceName -language $Language -repoRoot $RepoRoot -pkgPropsResult $pkgPropsResult + } + } + } + + return $pkgPropsResult +} + +function Operate-OnPackages ($activePkgList, $serviceName, $language, $repoRoot, [Array]$pkgPropsResult) +{ + foreach ($pkg in $activePkgList) + { + $pkgProps = Get-PkgProperties -PackageName $pkg["name"] -ServiceName $serviceName -Language $language -RepoRoot $repoRoot + $pkgPropsResult += $pkgProps + } + return $pkgPropsResult +} + +function Get-PkgListFromYml ($ciYmlPath) +{ + $ciYmlContent = Get-Content $ciYmlPath -Raw + $ciYmlObj = ConvertFrom-Yaml $ciYmlContent -Ordered + $artifactsInCI = $ciYmlObj["stages"][0]["parameters"]["Artifacts"] + + if ($artifactsInCI -eq $null) + { + Write-Error "Failed to retrive package names in ci $ciYmlPath" + } + return $artifactsInCI +} + +Export-ModuleMember -Function 'Get-PkgProperties' +Export-ModuleMember -Function 'Get-AllPkgProperties' \ No newline at end of file diff --git a/eng/common/scripts/modules/common-manifest.psd1 b/eng/common/scripts/modules/common-manifest.psd1 new file mode 100644 index 000000000000..419f046e6f67 --- /dev/null +++ b/eng/common/scripts/modules/common-manifest.psd1 @@ -0,0 +1,123 @@ +# +# Module manifest for module 'Common Modules' +# +# Generated by: azure-sdk +# +# Generated on: 5/19/2020 +# + +@{ + +# Script module or binary module file associated with this manifest. +# RootModule = '' + +# Version number of this module. +ModuleVersion = '1.0' + +# Supported PSEditions +# CompatiblePSEditions = @() + +# ID used to uniquely identify this module +GUID = '552cdcff-1f53-4f68-87a6-54490af6e94a' + +# Author of this module +Author = 'azure-sdk' + +# Company or vendor of this module +CompanyName = 'Unknown' + +# Copyright statement for this module +Copyright = '(c) azure-sdk. All rights reserved.' + +# Description of the functionality provided by this module +# Description = '' + +# Minimum version of the PowerShell engine required by this module +# PowerShellVersion = '' + +# Name of the PowerShell host required by this module +# PowerShellHostName = '' + +# Minimum version of the PowerShell host required by this module +# PowerShellHostVersion = '' + +# Minimum version of Microsoft .NET Framework required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# DotNetFrameworkVersion = '' + +# Minimum version of the common language runtime (CLR) required by this module. This prerequisite is valid for the PowerShell Desktop edition only. +# CLRVersion = '' + +# Processor architecture (None, X86, Amd64) required by this module +# ProcessorArchitecture = '' + +# Modules that must be imported into the global environment prior to importing this module +# RequiredModules = @() + +# Assemblies that must be loaded prior to importing this module +RequiredAssemblies = @() + +# Script files (.ps1) that are run in the caller's environment prior to importing this module. +ScriptsToProcess = @("${PSScriptRoot}\..\SemVer.ps1") + +# Type files (.ps1xml) to be loaded when importing this module +# TypesToProcess = @() + +# Format files (.ps1xml) to be loaded when importing this module +# FormatsToProcess = @() + +# Modules to import as nested modules of the module specified in RootModule/ModuleToProcess +NestedModules = @("${PSScriptRoot}\Package-Properties.psm1") + +# Functions to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no functions to export. +# FunctionsToExport = @() + +# Cmdlets to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no cmdlets to export. +CmdletsToExport = @() + +# Variables to export from this module +VariablesToExport = '*' + +# Aliases to export from this module, for best performance, do not use wildcards and do not delete the entry, use an empty array if there are no aliases to export. +AliasesToExport = @() + +# DSC resources to export from this module +# DscResourcesToExport = @() + +# List of all modules packaged with this module +# ModuleList = @() + +# List of all files packaged with this module +# FileList = @() + +# Private data to pass to the module specified in RootModule/ModuleToProcess. This may also contain a PSData hashtable with additional module metadata used by PowerShell. +PrivateData = @{ + + PSData = @{ + + # Tags applied to this module. These help with module discovery in online galleries. + # Tags = @() + + # A URL to the license for this module. + # LicenseUri = '' + + # A URL to the main website for this project. + # ProjectUri = '' + + # A URL to an icon representing this module. + # IconUri = '' + + # ReleaseNotes of this module + # ReleaseNotes = '' + + } # End of PSData hashtable + +} # End of PrivateData hashtable + +# HelpInfo URI of this module +# HelpInfoURI = '' + +# Default prefix for commands exported from this module. Override the default prefix using Import-Module -Prefix. +# DefaultCommandPrefix = '' + +} + diff --git a/eng/common/scripts/update-docs-metadata.ps1 b/eng/common/scripts/update-docs-metadata.ps1 index a162cbf8514e..b4cc2ac7849e 100644 --- a/eng/common/scripts/update-docs-metadata.ps1 +++ b/eng/common/scripts/update-docs-metadata.ps1 @@ -86,7 +86,7 @@ $pkgs = VerifyPackages -pkgRepository $Repository ` -workingDirectory $WorkDirectory ` -apiUrl $apiUrl ` -releaseSha $ReleaseSHA ` - -exitOnError $False + -continueOnError $True if ($pkgs) { Write-Host "Given the visible artifacts, readmes will be copied for the following packages"