diff --git a/generic/Run/HelperFunctions.ps1 b/generic/Run/HelperFunctions.ps1 index 6ef6fb1..c0793e1 100644 --- a/generic/Run/HelperFunctions.ps1 +++ b/generic/Run/HelperFunctions.ps1 @@ -667,122 +667,199 @@ function Download-Artifacts { [switch] $includePlatform, [switch] $force, [switch] $forceRedirection, - [string] $basePath = 'c:\dl' + [string] $basePath = 'c:\dl', + [int] $timeout = 300 ) - if (-not (Test-Path $basePath)) { New-Item $basePath -ItemType Directory | Out-Null } - - do { - $redir = $false - $appUri = [Uri]::new($artifactUrl) - - $appArtifactPath = Join-Path $basePath $appUri.AbsolutePath - $exists = Test-Path $appArtifactPath - if ($exists -and $force) { - Remove-Item $appArtifactPath -Recurse -Force - $exists = $false - } - if ($exists -and $forceRedirection) { - $appManifestPath = Join-Path $appArtifactPath "manifest.json" - $appManifest = Get-Content $appManifestPath | ConvertFrom-Json - if ($appManifest.PSObject.Properties.name -eq "applicationUrl") { + try { + $wipFileInitialized = $false; + $wipPlatformFileInitialized = $false; + do { + $redir = $false + $appUri = [Uri]::new($artifactUrl) + $appArtifactPath = Join-Path $basePath $appUri.AbsolutePath + + $wipFile = Join-Path $appArtifactPath '_inprocess.txt' + $readyFile = Join-Path $appArtifactPath '_ready.txt' + Test-RunningArtifactPreparationWithWait -InProcessFile $wipFile -Timeout $timeout + if (-not $wipFileInitialized) { + $lockFile = New-Item $wipFile -ItemType File -Force + $lockFileStream = $lockFile.OpenWrite() + $wipFileInitialized = $true + } + $exists = Test-Path $readyFile + if ($exists -and $force) { Remove-Item $appArtifactPath -Recurse -Force $exists = $false } - } - if (-not $exists) { - Write-Host "Downloading application artifact $($appUri.AbsolutePath)" - $appZip = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString()).zip" - try { - Download-File -sourceUrl $artifactUrl -destinationFile $appZip + if ($exists -and $forceRedirection) { + $appManifestPath = Join-Path $appArtifactPath "manifest.json" + $appManifest = Get-Content $appManifestPath | ConvertFrom-Json + if ($appManifest.PSObject.Properties.name -eq "applicationUrl") { + Remove-Item $appArtifactPath -Recurse -Force + $exists = $false + } } - catch { - if ($artifactUrl.Contains('.azureedge.net/')) { - $artifactUrl = $artifactUrl.Replace('.azureedge.net/','.blob.core.windows.net/') - Write-Host "Retrying download..." + if (-not $exists) { + Write-Host "Downloading application artifact $($appUri.AbsolutePath)" + $appZip = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString()).zip" + try { Download-File -sourceUrl $artifactUrl -destinationFile $appZip } + catch { + if ($artifactUrl.Contains('.azureedge.net/')) { + $artifactUrl = $artifactUrl.Replace('.azureedge.net/','.blob.core.windows.net/') + Write-Host "Retrying download..." + Download-File -sourceUrl $artifactUrl -destinationFile $appZip + } + } + Write-Host "Unpacking application artifact" + Expand-Archive -Path $appZip -DestinationPath $appArtifactPath -Force + Remove-Item -path $appZip -force + } else { + Write-Host "Reusing existing artifact cache." + } + try { [System.IO.File]::WriteAllText((Join-Path $appArtifactPath 'lastused'), "$([datetime]::UtcNow.Ticks)") } catch {} + $appManifestPath = Join-Path $appArtifactPath "manifest.json" + $appManifest = Get-Content $appManifestPath | ConvertFrom-Json + if ($appManifest.PSObject.Properties.name -eq "applicationUrl") { + $redir = $true + $artifactUrl = $appManifest.ApplicationUrl + if ($artifactUrl -notlike 'https://*') { + $artifactUrl = "https://$($appUri.Host)/$artifactUrl$($appUri.Query)" + } } - Write-Host "Unpacking application artifact" - Expand-Archive -Path $appZip -DestinationPath $appArtifactPath -Force - Remove-Item -path $appZip -force - } - try { [System.IO.File]::WriteAllText((Join-Path $appArtifactPath 'lastused'), "$([datetime]::UtcNow.Ticks)") } catch {} - $appManifestPath = Join-Path $appArtifactPath "manifest.json" - $appManifest = Get-Content $appManifestPath | ConvertFrom-Json + } while ($redir) + + $appArtifactPath - if ($appManifest.PSObject.Properties.name -eq "applicationUrl") { - $redir = $true - $artifactUrl = $appManifest.ApplicationUrl - if ($artifactUrl -notlike 'https://*') { - $artifactUrl = "https://$($appUri.Host)/$artifactUrl$($appUri.Query)" + if ($includePlatform) { + if ($appManifest.PSObject.Properties.name -eq "platformUrl") { + $platformUrl = $appManifest.platformUrl + } + else { + $platformUrl = "$($appUri.AbsolutePath.Substring(0,$appUri.AbsolutePath.LastIndexOf('/')))/platform".TrimStart('/') } - } - } while ($redir) + if ($platformUrl -notlike 'https://*') { + $platformUrl = "https://$($appUri.Host.TrimEnd('/'))/$platformUrl$($appUri.Query)" + } + $platformUri = [Uri]::new($platformUrl) + + $platformArtifactPath = Join-Path $basePath $platformUri.AbsolutePath + $wipPlatformFile = Join-Path $platformArtifactPath '_inprocess.txt' + $readyPlatformFile = Join-Path $platformArtifactPath '_ready.txt' + Test-RunningArtifactPreparationWithWait -InProcessFile $wipPlatformFile -Timeout $timeout + if (-not $wipPlatformFileInitialized) { + $lockPlatformFile = New-Item $wipPlatformFile -ItemType File -Force + $lockPlatformFileStream = $lockPlatformFile.OpenWrite() + $wipPlatformFileInitialized = $true + } + $exists = Test-Path $readyPlatformFile + if ($exists -and $force) { + Remove-Item $platformArtifactPath -Recurse -Force + $exists = $false + } + if (-not $exists) { + Write-Host "Downloading platform artifact $($platformUri.AbsolutePath)" + $platformZip = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString()).zip" + try { + Download-File -sourceUrl $platformUrl -destinationFile $platformZip + } + catch { + if ($platformUrl.Contains('.azureedge.net/')) { + $platformUrl = $platformUrl.Replace('.azureedge.net/','.blob.core.windows.net/') + Write-Host "Retrying download..." + Download-File -sourceUrl $platformUrl -destinationFile $platformZip + } + } + Write-Host "Unpacking platform artifact" + Expand-Archive -Path $platformZip -DestinationPath $platformArtifactPath -Force + Remove-Item -path $platformZip -force + + $prerequisiteComponentsFile = Join-Path $platformArtifactPath "Prerequisite Components.json" + if (Test-Path $prerequisiteComponentsFile) { + $prerequisiteComponents = Get-Content $prerequisiteComponentsFile | ConvertFrom-Json + Write-Host "Downloading Prerequisite Components" + $prerequisiteComponents.PSObject.Properties | % { + $path = Join-Path $platformArtifactPath $_.Name + if (-not (Test-Path $path)) { + $dirName = [System.IO.Path]::GetDirectoryName($path) + $filename = [System.IO.Path]::GetFileName($path) + if (-not (Test-Path $dirName)) { + New-Item -Path $dirName -ItemType Directory | Out-Null + } + $url = $_.Value + Download-File -sourceUrl $url -destinationFile $path + } + } + } + } else { + Write-Host "Reusing existing platform artifact cache." + } + try { [System.IO.File]::WriteAllText((Join-Path $platformArtifactPath 'lastused'), "$([datetime]::UtcNow.Ticks)") } catch {} - $appArtifactPath + if ($wipPlatformFileInitialized) { + New-Item $readyPlatformFile -ItemType File -Force | Out-Null + } - if ($includePlatform) { - if ($appManifest.PSObject.Properties.name -eq "platformUrl") { - $platformUrl = $appManifest.platformUrl + $platformArtifactPath } - else { - $platformUrl = "$($appUri.AbsolutePath.Substring(0,$appUri.AbsolutePath.LastIndexOf('/')))/platform".TrimStart('/') + + if ($wipFileInitialized) { + New-Item $readyFile -ItemType File -Force | Out-Null } - - if ($platformUrl -notlike 'https://*') { - $platformUrl = "https://$($appUri.Host.TrimEnd('/'))/$platformUrl$($appUri.Query)" + + } finally { + + if ($wipFileInitialized) { + $lockFileStream.Close() + Remove-Item $lockFile -Force | Out-Null + $wipFileInitialized = $false } - $platformUri = [Uri]::new($platformUrl) - - $platformArtifactPath = Join-Path $basePath $platformUri.AbsolutePath - $exists = Test-Path $platformArtifactPath - if ($exists -and $force) { - Remove-Item $platformArtifactPath -Recurse -Force - $exists = $false + + if ($wipPlatformFileInitialized) { + $lockPlatformFileStream.Close() + Remove-Item $lockPlatformFile -Force | Out-Null + $wipPlatformFileInitialized = $false } - if (-not $exists) { - Write-Host "Downloading platform artifact $($platformUri.AbsolutePath)" - $platformZip = Join-Path ([System.IO.Path]::GetTempPath()) "$([Guid]::NewGuid().ToString()).zip" - try { - Download-File -sourceUrl $platformUrl -destinationFile $platformZip - } - catch { - if ($platformUrl.Contains('.azureedge.net/')) { - $platformUrl = $platformUrl.Replace('.azureedge.net/','.blob.core.windows.net/') - Write-Host "Retrying download..." - Download-File -sourceUrl $platformUrl -destinationFile $platformZip - } + + } +} + +function Test-RunningArtifactPreparationWithWait { + param ( + [Parameter(Mandatory = $true)] + [string] $InProcessFile, + [Parameter(Mandatory = $true)] + [int] $Timeout + ) + if (Test-Path $InProcessFile) { + $fileLocked = $false + $waitingAnotherProcessMsgIssued = $false + $lockCheckStartDT = Get-Date + + do { + try { + (Get-Date | Out-File -FilePath $InProcessFile -Append) + $fileLocked = $false + } catch { + $fileLocked = $true } - Write-Host "Unpacking platform artifact" - Expand-Archive -Path $platformZip -DestinationPath $platformArtifactPath -Force - Remove-Item -path $platformZip -force - - $prerequisiteComponentsFile = Join-Path $platformArtifactPath "Prerequisite Components.json" - if (Test-Path $prerequisiteComponentsFile) { - $prerequisiteComponents = Get-Content $prerequisiteComponentsFile | ConvertFrom-Json - Write-Host "Downloading Prerequisite Components" - $prerequisiteComponents.PSObject.Properties | % { - $path = Join-Path $platformArtifactPath $_.Name - if (-not (Test-Path $path)) { - $dirName = [System.IO.Path]::GetDirectoryName($path) - $filename = [System.IO.Path]::GetFileName($path) - if (-not (Test-Path $dirName)) { - New-Item -Path $dirName -ItemType Directory | Out-Null - } - $url = $_.Value - Download-File -sourceUrl $url -destinationFile $path - } + if ($fileLocked) { + if (-not $waitingAnotherProcessMsgIssued) { + Write-Host "Waiting for another process running artifact download and extraction ..." + $waitingAnotherProcessMsgIssued = $true } + Start-Sleep -Seconds 5 } + } while (($fileLocked) -or (((Get-Date) - $lockCheckStartDT).Second > $Timeout)) + if ($fileLocked) { + Write-Error "Artifacts are being downloaded and extracted somewhere else and timeout=$Timeout seconds has passed." } - try { [System.IO.File]::WriteAllText((Join-Path $platformArtifactPath 'lastused'), "$([datetime]::UtcNow.Ticks)") } catch {} - - $platformArtifactPath } }