From e41f19715307deb9691906a2b74b93e0bbc6604c Mon Sep 17 00:00:00 2001 From: "dotnet-maestro[bot]" Date: Mon, 29 Apr 2019 12:52:58 +0000 Subject: [PATCH] Update dependencies from https://github.com/dotnet/arcade build 20190426.3 (#320) - Microsoft.DotNet.Arcade.Sdk - 1.0.0-beta.19226.3 --- eng/Version.Details.xml | 4 +- eng/common/SourceLinkValidation.ps1 | 184 ++++++++++++++++++++++++++++ global.json | 2 +- 3 files changed, 187 insertions(+), 3 deletions(-) create mode 100644 eng/common/SourceLinkValidation.ps1 diff --git a/eng/Version.Details.xml b/eng/Version.Details.xml index 8550113fae..2eb743ef9f 100644 --- a/eng/Version.Details.xml +++ b/eng/Version.Details.xml @@ -3,9 +3,9 @@ - + https://github.com/dotnet/arcade - d37270268a65592cae630f1b979b70f74d4614dd + 7bec23ce3da545d97f53f99abce457a2e252aa58 diff --git a/eng/common/SourceLinkValidation.ps1 b/eng/common/SourceLinkValidation.ps1 new file mode 100644 index 0000000000..cb2d28cb99 --- /dev/null +++ b/eng/common/SourceLinkValidation.ps1 @@ -0,0 +1,184 @@ +param( + [Parameter(Mandatory=$true)][string] $InputPath, # Full path to directory where Symbols.NuGet packages to be checked are stored + [Parameter(Mandatory=$true)][string] $ExtractPath, # Full path to directory where the packages will be extracted during validation + [Parameter(Mandatory=$true)][string] $SourceLinkToolPath, # Full path to directory where dotnet SourceLink CLI was installed + [Parameter(Mandatory=$true)][string] $GHRepoName, # GitHub name of the repo including the Org. E.g., dotnet/arcade + [Parameter(Mandatory=$true)][string] $GHCommit # GitHub commit SHA used to build the packages +) + +# Cache/HashMap (File -> Exist flag) used to consult whether a file exist +# in the repository at a specific commit point. This is populated by inserting +# all files present in the repo at a specific commit point. +$global:RepoFiles = @{} + +$ValidatePackage = { + param( + [string] $PackagePath # Full path to a Symbols.NuGet package + ) + + # Ensure input file exist + if (!(Test-Path $PackagePath)) { + throw "Input file does not exist: $PackagePath" + } + + # Extensions for which we'll look for SourceLink information + # For now we'll only care about Portable & Embedded PDBs + $RelevantExtensions = @(".dll", ".exe", ".pdb") + + Write-Host -NoNewLine "Validating" ([System.IO.Path]::GetFileName($PackagePath)) "... " + + $PackageId = [System.IO.Path]::GetFileNameWithoutExtension($PackagePath) + $ExtractPath = Join-Path -Path $using:ExtractPath -ChildPath $PackageId + $FailedFiles = 0 + + Add-Type -AssemblyName System.IO.Compression.FileSystem + + [System.IO.Directory]::CreateDirectory($ExtractPath); + + $zip = [System.IO.Compression.ZipFile]::OpenRead($PackagePath) + + $zip.Entries | + Where-Object {$RelevantExtensions -contains [System.IO.Path]::GetExtension($_.Name)} | + ForEach-Object { + $FileName = $_.FullName + $Extension = [System.IO.Path]::GetExtension($_.Name) + $FakeName = -Join((New-Guid), $Extension) + $TargetFile = Join-Path -Path $ExtractPath -ChildPath $FakeName + + # We ignore resource DLLs + if ($FileName.EndsWith(".resources.dll")) { + return + } + + [System.IO.Compression.ZipFileExtensions]::ExtractToFile($_, $TargetFile, $true) + + $ValidateFile = { + param( + [string] $FullPath, # Full path to the module that has to be checked + [string] $RealPath, + [ref] $FailedFiles + ) + + # Makes easier to reference `sourcelink cli` + Push-Location $using:SourceLinkToolPath + + $SourceLinkInfos = .\sourcelink.exe print-urls $FullPath | Out-String + + if ($LASTEXITCODE -eq 0 -and -not ([string]::IsNullOrEmpty($SourceLinkInfos))) { + $NumFailedLinks = 0 + + # We only care about Http addresses + $Matches = (Select-String '(http[s]?)(:\/\/)([^\s,]+)' -Input $SourceLinkInfos -AllMatches).Matches + + if ($Matches.Count -ne 0) { + $Matches.Value | + ForEach-Object { + $Link = $_ + $CommitUrl = -Join("https://raw.githubusercontent.com/", $using:GHRepoName, "/", $using:GHCommit, "/") + $FilePath = $Link.Replace($CommitUrl, "") + $Status = 200 + $Cache = $using:RepoFiles + + if ( !($Cache.ContainsKey($FilePath)) ) { + try { + $Uri = $Link -as [System.URI] + + # Only GitHub links are valid + if ($Uri.AbsoluteURI -ne $null -and $Uri.Host -match "github") { + $Status = (Invoke-WebRequest -Uri $Link -UseBasicParsing -Method HEAD -TimeoutSec 5).StatusCode + } + else { + $Status = 0 + } + } + catch { + $Status = 0 + } + } + + if ($Status -ne 200) { + if ($NumFailedLinks -eq 0) { + if ($FailedFiles.Value -eq 0) { + Write-Host + } + + Write-Host "`tFile $RealPath has broken links:" + } + + Write-Host "`t`tFailed to retrieve $Link" + + $NumFailedLinks++ + } + } + } + + if ($NumFailedLinks -ne 0) { + $FailedFiles.value++ + $global:LASTEXITCODE = 1 + } + } + + Pop-Location + } + + &$ValidateFile $TargetFile $FileName ([ref]$FailedFiles) + } + + $zip.Dispose() + + if ($FailedFiles -eq 0) { + Write-Host "Passed." + } +} + +function ValidateSourceLinkLinks { + if (!($GHRepoName -Match "^[^\s\/]+/[^\s\/]+$")) { + Write-Host "GHRepoName should be in the format /" + $global:LASTEXITCODE = 1 + return + } + + if (!($GHCommit -Match "^[0-9a-fA-F]{40}$")) { + Write-Host "GHCommit should be a 40 chars hexadecimal string" + $global:LASTEXITCODE = 1 + return + } + + $RepoTreeURL = -Join("https://api.github.com/repos/", $GHRepoName, "/git/trees/", $GHCommit, "?recursive=1") + $CodeExtensions = @(".cs", ".vb", ".fs", ".fsi", ".fsx", ".fsscript") + + try { + # Retrieve the list of files in the repo at that particular commit point and store them in the RepoFiles hash + $Data = Invoke-WebRequest $RepoTreeURL | ConvertFrom-Json | Select-Object -ExpandProperty tree + + foreach ($file in $Data) { + $Extension = [System.IO.Path]::GetExtension($file.path) + + if ($CodeExtensions.Contains($Extension)) { + $RepoFiles[$file.path] = 1 + } + } + } + catch { + Write-Host "Problems downloading the list of files from the repo. Url used: $RepoTreeURL" + $global:LASTEXITCODE = 1 + return + } + + if (Test-Path $ExtractPath) { + Remove-Item $ExtractPath -Force -Recurse -ErrorAction SilentlyContinue + } + + # Process each NuGet package in parallel + $Jobs = @() + Get-ChildItem "$InputPath\*.symbols.nupkg" | + ForEach-Object { + $Jobs += Start-Job -ScriptBlock $ValidatePackage -ArgumentList $_.FullName + } + + foreach ($Job in $Jobs) { + Wait-Job -Id $Job.Id | Receive-Job + } +} + +Measure-Command { ValidateSourceLinkLinks } diff --git a/global.json b/global.json index 6f02347c57..1cba26362d 100644 --- a/global.json +++ b/global.json @@ -7,6 +7,6 @@ "xcopy-msbuild": "15.9.0-alpha" }, "msbuild-sdks": { - "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19225.5" + "Microsoft.DotNet.Arcade.Sdk": "1.0.0-beta.19226.3" } }