From 6e621933b4a9e480042f38e61c5edb17a98d692c Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Thu, 27 May 2021 19:04:17 -0400 Subject: [PATCH] Enable Caching of PS Modules (#21425) - Remove copied AzPowershell utilities - Add latest AZ module path already on hosted agents to PSModulePath - Rename setup-az-modules template setup-environments to reflect what is is doing - Add support for Caching the current user PS Module folder - Add support for install-module if not already present in module folder - Organize the live test clean-up script to be in the standard location Co-authored-by: Wes Haggard --- .../AzurePowerShellV4/Utility.ps1 | 153 ------------------ eng/common/TestResources/Import-AzModules.ps1 | 11 -- .../TestResources/deploy-test-resources.yml | 6 +- .../TestResources/remove-test-resources.yml | 2 +- ...-az-modules.yml => setup-environments.yml} | 4 +- .../templates/steps/cache-ps-modules.yml | 10 ++ .../scripts/Helpers/PSModule-Helpers.ps1 | 97 +++++++++++ eng/common/scripts/Import-AzModules.ps1 | 8 + 8 files changed, 122 insertions(+), 169 deletions(-) delete mode 100644 eng/common/TestResources/AzurePowerShellV4/Utility.ps1 delete mode 100644 eng/common/TestResources/Import-AzModules.ps1 rename eng/common/TestResources/{setup-az-modules.yml => setup-environments.yml} (92%) create mode 100644 eng/common/pipelines/templates/steps/cache-ps-modules.yml create mode 100644 eng/common/scripts/Helpers/PSModule-Helpers.ps1 create mode 100644 eng/common/scripts/Import-AzModules.ps1 diff --git a/eng/common/TestResources/AzurePowerShellV4/Utility.ps1 b/eng/common/TestResources/AzurePowerShellV4/Utility.ps1 deleted file mode 100644 index 0bac797fb14ba..0000000000000 --- a/eng/common/TestResources/AzurePowerShellV4/Utility.ps1 +++ /dev/null @@ -1,153 +0,0 @@ -# Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/a1502bbe67561f5bec8402f32c997406f798a019/Tasks/AzurePowerShellV4/Utility.ps1 - -function Get-SavedModulePath { - [CmdletBinding()] - param([string] $azurePowerShellVersion) - $savedModulePath = $($env:SystemDrive + "\Modules\az_" + $azurePowerShellVersion) - Write-Verbose "The value of the module path is: $savedModulePath" - return $savedModulePath -} - -function Get-SavedModulePathLinux { - [CmdletBinding()] - param([string] $azurePowerShellVersion) - $savedModulePath = $("/usr/share/az_" + $azurePowerShellVersion) - Write-Verbose "The value of the module path is: $savedModulePath" - return $savedModulePath -} - -function Update-PSModulePathForHostedAgent { - [CmdletBinding()] - param([string] $targetAzurePs) - try { - if ($targetAzurePs) { - $hostedAgentAzModulePath = Get-SavedModulePath -azurePowerShellVersion $targetAzurePs - } - else { - $hostedAgentAzModulePath = Get-LatestModule -patternToMatch "^az_[0-9]+\.[0-9]+\.[0-9]+$" -patternToExtract "[0-9]+\.[0-9]+\.[0-9]+$" - } - $env:PSModulePath = $hostedAgentAzModulePath + ";" + $env:PSModulePath - $env:PSModulePath = $env:PSModulePath.TrimStart(';') - } finally { - Write-Verbose "The updated value of the PSModulePath is: $($env:PSModulePath)" - } -} - -function Update-PSModulePathForHostedAgentLinux { - [CmdletBinding()] - param([string] $targetAzurePs) - try { - if ($targetAzurePs) { - $hostedAgentAzModulePath = Get-SavedModulePathLinux -azurePowerShellVersion $targetAzurePs - if(!(Test-Path $hostedAgentAzModulePath)) { - Write-Verbose "No module path found with this name" - throw ("Could not find the module path with given version.") - } - } - else { - $hostedAgentAzModulePath = Get-LatestModuleLinux -patternToMatch "^az_[0-9]+\.[0-9]+\.[0-9]+$" -patternToExtract "[0-9]+\.[0-9]+\.[0-9]+$" - } - $env:PSModulePath = $hostedAgentAzModulePath + ":" + $env:PSModulePath - $env:PSModulePath = $env:PSModulePath.TrimStart(':') - } finally { - Write-Verbose "The updated value of the PSModulePath is: $($env:PSModulePath)" - } -} - -function Get-LatestModule { - [CmdletBinding()] - param([string] $patternToMatch, - [string] $patternToExtract) - - $resultFolder = "" - $regexToMatch = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToMatch - $regexToExtract = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToExtract - $maxVersion = [version] "0.0.0" - $modulePath = $env:SystemDrive + "\Modules"; - - try { - if (-not (Test-Path -Path $modulePath)) { - return $resultFolder - } - - $moduleFolders = Get-ChildItem -Directory -Path $modulePath | Where-Object { $regexToMatch.IsMatch($_.Name) } - foreach ($moduleFolder in $moduleFolders) { - $moduleVersion = [version] $($regexToExtract.Match($moduleFolder.Name).Groups[0].Value) - if($moduleVersion -gt $maxVersion) { - $modulePath = [System.IO.Path]::Combine($moduleFolder.FullName,"Az\$moduleVersion\Az.psm1") - - if(Test-Path -LiteralPath $modulePath -PathType Leaf) { - $maxVersion = $moduleVersion - $resultFolder = $moduleFolder.FullName - } else { - Write-Verbose "A folder matching the module folder pattern was found at $($moduleFolder.FullName) but didn't contain a valid module file" - } - } - } - } - catch { - Write-Verbose "Attempting to find the Latest Module Folder failed with the error: $($_.Exception.Message)" - $resultFolder = "" - } - Write-Verbose "Latest module folder detected: $resultFolder" - return $resultFolder -} - -function Get-LatestModuleLinux { - [CmdletBinding()] - param([string] $patternToMatch, - [string] $patternToExtract) - - $resultFolder = "" - $regexToMatch = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToMatch - $regexToExtract = New-Object -TypeName System.Text.RegularExpressions.Regex -ArgumentList $patternToExtract - $maxVersion = [version] "0.0.0" - - try { - $moduleFolders = Get-ChildItem -Directory -Path $("/usr/share") | Where-Object { $regexToMatch.IsMatch($_.Name) } - foreach ($moduleFolder in $moduleFolders) { - $moduleVersion = [version] $($regexToExtract.Match($moduleFolder.Name).Groups[0].Value) - if($moduleVersion -gt $maxVersion) { - $modulePath = [System.IO.Path]::Combine($moduleFolder.FullName,"Az/$moduleVersion/Az.psm1") - - if(Test-Path -LiteralPath $modulePath -PathType Leaf) { - $maxVersion = $moduleVersion - $resultFolder = $moduleFolder.FullName - } else { - Write-Verbose "A folder matching the module folder pattern was found at $($moduleFolder.FullName) but didn't contain a valid module file" - } - } - } - } - catch { - Write-Verbose "Attempting to find the Latest Module Folder failed with the error: $($_.Exception.Message)" - $resultFolder = "" - } - Write-Verbose "Latest module folder detected: $resultFolder" - return $resultFolder -} - -function CleanUp-PSModulePathForHostedAgent { - # Clean up PSModulePath for hosted agent - $azureRMModulePath = "C:\Modules\azurerm_2.1.0" - $azureModulePath = "C:\Modules\azure_2.1.0" - $azPSModulePath = $env:PSModulePath - - if ($azPSModulePath.split(";") -contains $azureRMModulePath) { - $azPSModulePath = (($azPSModulePath).Split(";") | ? { $_ -ne $azureRMModulePath }) -join ";" - write-verbose "$azureRMModulePath removed. Restart the prompt for the changes to take effect." - } - else { - write-verbose "$azureRMModulePath is not present in $azPSModulePath" - } - - if ($azPSModulePath.split(";") -contains $azureModulePath) { - $azPSModulePath = (($azPSModulePath).Split(";") | ? { $_ -ne $azureModulePath }) -join ";" - write-verbose "$azureModulePath removed. Restart the prompt for the changes to take effect." - } - else { - write-verbose "$azureModulePath is not present in $azPSModulePath" - } - - $env:PSModulePath = $azPSModulePath -} diff --git a/eng/common/TestResources/Import-AzModules.ps1 b/eng/common/TestResources/Import-AzModules.ps1 deleted file mode 100644 index 3cd7b3124489f..0000000000000 --- a/eng/common/TestResources/Import-AzModules.ps1 +++ /dev/null @@ -1,11 +0,0 @@ -. "$PSScriptRoot/AzurePowerShellV4/Utility.ps1" - -if ($IsWindows) { - # Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/9cc8e1b3ee37dc023c81290de1dd522b77faccf7/Tasks/AzurePowerShellV4/AzurePowerShell.ps1#L57-L58 - CleanUp-PSModulePathForHostedAgent - Update-PSModulePathForHostedAgent -} -else { - # Copied from https://github.com/microsoft/azure-pipelines-tasks/blob/9cc8e1b3ee37dc023c81290de1dd522b77faccf7/Tasks/AzurePowerShellV4/InitializeAz.ps1#L16 - Update-PSModulePathForHostedAgentLinux -} diff --git a/eng/common/TestResources/deploy-test-resources.yml b/eng/common/TestResources/deploy-test-resources.yml index a9d693b99420e..ac05e14c79343 100644 --- a/eng/common/TestResources/deploy-test-resources.yml +++ b/eng/common/TestResources/deploy-test-resources.yml @@ -33,10 +33,12 @@ parameters: steps: - - template: /eng/common/TestResources/setup-az-modules.yml + - template: /eng/common/pipelines/templates/steps/cache-ps-modules.yml + + - template: /eng/common/TestResources/setup-environments.yml - pwsh: | - eng/common/TestResources/Import-AzModules.ps1 + eng/common/scripts/Import-AzModules.ps1 $subscriptionConfiguration = @' ${{ parameters.SubscriptionConfiguration }} diff --git a/eng/common/TestResources/remove-test-resources.yml b/eng/common/TestResources/remove-test-resources.yml index a0871a2ff7adb..0c059f51a0328 100644 --- a/eng/common/TestResources/remove-test-resources.yml +++ b/eng/common/TestResources/remove-test-resources.yml @@ -21,7 +21,7 @@ parameters: steps: - pwsh: | - eng/common/TestResources/Import-AzModules.ps1 + eng/common/scripts/Import-AzModules.ps1 $subscriptionConfiguration = @" ${{ parameters.SubscriptionConfiguration }} diff --git a/eng/common/TestResources/setup-az-modules.yml b/eng/common/TestResources/setup-environments.yml similarity index 92% rename from eng/common/TestResources/setup-az-modules.yml rename to eng/common/TestResources/setup-environments.yml index b5369f433b902..ef400bcb1d127 100644 --- a/eng/common/TestResources/setup-az-modules.yml +++ b/eng/common/TestResources/setup-environments.yml @@ -21,12 +21,12 @@ steps: condition: contains(variables['OSVmImage'], 'mac') - task: Powershell@2 + displayName: Register Dogfood environment inputs: - displayName: Register Dogfood environment targetType: inline pwsh: true script: | - eng/common/TestResources/Import-AzModules.ps1 + eng/common/scripts/Import-AzModules.ps1 $environmentSpec = @" $(env-config-dogfood) diff --git a/eng/common/pipelines/templates/steps/cache-ps-modules.yml b/eng/common/pipelines/templates/steps/cache-ps-modules.yml new file mode 100644 index 0000000000000..563dba40294ff --- /dev/null +++ b/eng/common/pipelines/templates/steps/cache-ps-modules.yml @@ -0,0 +1,10 @@ +steps: +- pwsh: | + . ./eng/common/scripts/Helpers/PSModule-Helpers.ps1 + Write-Host "##vso[task.setvariable variable=CachedPSModulePath]$global:CurrentUserModulePath" + displayName: Set PS Modules Cache Directory +- task: Cache@2 + inputs: + key: 'PSModulePath | $(CacheSalt) | $(Agent.OS) | $(Build.SourcesDirectory)/eng/common/scripts/Import-AzModules.ps1' + path: $(CachedPSModulePath) + displayName: Cache PS Modules \ No newline at end of file diff --git a/eng/common/scripts/Helpers/PSModule-Helpers.ps1 b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 new file mode 100644 index 0000000000000..96f34ff71f4a5 --- /dev/null +++ b/eng/common/scripts/Helpers/PSModule-Helpers.ps1 @@ -0,0 +1,97 @@ +$DefaultPSRepositoryUrl = "https://www.powershellgallery.com/api/v2" +$global:CurrentUserModulePath = "" + +function Update-PSModulePath() +{ + # Information on PSModulePath taken from docs + # https://docs.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_psmodulepath + + # Information on Az custom module paths on hosted agents taken from + # https://github.com/microsoft/azure-pipelines-tasks/blob/c9771bc064cd60f47587c68e5c871b7cd13f0f28/Tasks/AzurePowerShellV5/Utility.ps1 + + if ($IsWindows) { + $hostedAgentModulePath = $env:SystemDrive + "\Modules" + $moduleSeperator = ";" + } else { + $hostedAgentModulePath = "/usr/share" + $moduleSeperator = ":" + } + $modulePaths = $env:PSModulePath -split $moduleSeperator + + # Remove any hosted agent paths (needed to remove old default azure/azurerm paths which cause conflicts) + $modulePaths = $modulePaths.Where({ !$_.StartsWith($hostedAgentModulePath) }) + + # Add any "az_" paths from the agent which is the lastest set of azure modules + $AzModuleCachPath = (Get-ChildItem "$hostedAgentModulePath/az_*" -Attributes Directory) -join $moduleSeperator + if ($AzModuleCachPath -and $env.PSModulePath -notcontains $AzModuleCachPath) { + $modulePaths += $AzModuleCachPath + } + + $env:PSModulePath = $modulePaths -join $moduleSeperator + + # Find the path that is under user home directory + $homeDirectories = $modulePaths.Where({ $_.StartsWith($home) }) + if ($homeDirectories.Count -gt 0) { + $global:CurrentUserModulePath = $homeDirectories[0] + if ($homeDirectories.Count -gt 1) { + Write-Verbose "Found more then one module path starting with $home so selecting the first one $global:CurrentUserModulePath" + } + + # In some cases the directory might not exist so we need to create it otherwise caching an empty directory will fail + if (!(Test-Path $global:CurrentUserModulePath)) { + New-Item $global:CurrentUserModulePath -ItemType Directory > $null + } + } + else { + Write-Error "Did not find a module path starting with $home to set up a user module path in $env:PSModulePath" + } +} + +# If we want to use another default repository other then PSGallery we can update the default parameters +function Install-ModuleIfNotInstalled($moduleName, $version, $repositoryUrl = $DefaultPSRepositoryUrl) +{ + # Check installed modules + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + } + + if ($modules.Count -eq 0) + { + $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) + if ($repositories.Count -eq 0) + { + Register-PSRepository -Name $repositoryUrl -SourceLocation $repositoryUrl -InstallationPolicy Trusted + $repositories = (Get-PSRepository).Where({ $_.SourceLocation -eq $repositoryUrl }) + if ($repositories.Count -eq 0) { + Write-Error "Failed to registory package repository $repositoryUrl." + return + } + } + $repository = $repositories[0] + + if ($repository.InstallationPolicy -ne "Trusted") { + Set-PSRepository -Name $repository.Name -InstallationPolicy "Trusted" + } + + Write-Host "Installing module $moduleName with min version $version from $repositoryUrl" + # Install under CurrentUser scope so that the end up under $CurrentUserModulePath for caching + Install-Module $moduleName -MinimumVersion $version -Repository $repository.Name -Scope CurrentUser -Force + + # Ensure module installed + $modules = (Get-Module -ListAvailable $moduleName) + if ($version -as [Version]) { + $modules = $modules.Where({ [Version]$_.Version -ge [Version]$version }) + } + + if ($modules.Count -eq 0) { + Write-Error "Failed to install module $moduleName with version $version" + return + } + } + + Write-Host "Using module $($modules[0].Name) with version $($modules[0].Version)." + return $modules[0] +} + +Update-PSModulePath \ No newline at end of file diff --git a/eng/common/scripts/Import-AzModules.ps1 b/eng/common/scripts/Import-AzModules.ps1 new file mode 100644 index 0000000000000..6e97ba970289d --- /dev/null +++ b/eng/common/scripts/Import-AzModules.ps1 @@ -0,0 +1,8 @@ +[CmdletBinding()] +param ( + [string]$AzModuleVersion = "5.7.0" # Current version cached on agents +) + +. (Join-Path $PSScriptRoot Helpers PSModule-Helpers.ps1) + +Install-ModuleIfNotInstalled "Az" $AzModuleVersion | Import-Module \ No newline at end of file