From 6a9d4a5fff926bc17a68fe985db03c356d7eedc3 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 15 Jul 2022 14:33:10 -0400 Subject: [PATCH 1/3] Add full clone fallback to sparse checkout --- .../templates/steps/sparse-checkout.yml | 34 +++++++++++++++++++ 1 file changed, 34 insertions(+) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index a3b553b3a7..4522666475 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -23,6 +23,25 @@ steps: # Define this inline, because of the chicken/egg problem with loading a script when nothing # has been checked out yet. script: | + function Clone([Hashtable]$repository) + { + Write-Warning "Sparse checkout failed, falling back to full clone" + if (Test-Path .git) { + Write-Warning "Deleting existing git repository" + Write-Host "Remove-Item -Force -Recurse ./*" + Remove-Item -Force -Recurse ./* + } + + Write-Host "git clone https://github.com/$($repository.Name) ." + git clone https://github.com/$($repository.Name) . + Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" + # This will use the default branch if repo.Commitish is empty + git -c advice.detachedHead=false checkout $($repository.Commitish) + if ($LASTEXITCODE) { + exit $LASTEXITCODE + } + } + function SparseCheckout([Array]$paths, [Hashtable]$repository) { $dir = $repository.WorkingDirectory @@ -40,14 +59,23 @@ steps: Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + if ($LASTEXITCODE) { + return (Clone $repository && Pop-Location) + } Write-Host "git sparse-checkout init" git sparse-checkout init + if ($LASTEXITCODE) { + return (Clone $repository && Pop-Location) + } # Set non-cone mode otherwise path filters will not work in git >= 2.37.0 # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'" git sparse-checkout set --no-cone '/*' '!/*/' '/eng' + if ($LASTEXITCODE) { + return (Clone $repository && Pop-Location) + } } # Prevent wildcard expansion in Invoke-Expression (e.g. for checkout path '/*') @@ -55,6 +83,9 @@ steps: $gitsparsecmd = "git sparse-checkout add $quotedPaths" Write-Host $gitsparsecmd Invoke-Expression -Command $gitsparsecmd + if ($LASTEXITCODE) { + return (Clone $repository && Pop-Location) + } Write-Host "Set sparse checkout paths to:" Get-Content .git/info/sparse-checkout @@ -64,6 +95,9 @@ steps: Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" # This will use the default branch if repo.Commitish is empty git -c advice.detachedHead=false checkout $($repository.Commitish) + if ($LASTEXITCODE) { + return (Clone $repository && Pop-Location) + } } else { Write-Host "Skipping checkout as repo has already been initialized" } From 9513e1918b25e8ec3036d297ff3f5576c9723a24 Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Fri, 22 Jul 2022 05:32:41 -0400 Subject: [PATCH 2/3] Improve clone handling and overrides for sparse checkout --- .../templates/steps/sparse-checkout.yml | 120 ++++++++++-------- 1 file changed, 68 insertions(+), 52 deletions(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 4522666475..3b664154a1 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -25,7 +25,6 @@ steps: script: | function Clone([Hashtable]$repository) { - Write-Warning "Sparse checkout failed, falling back to full clone" if (Test-Path .git) { Write-Warning "Deleting existing git repository" Write-Host "Remove-Item -Force -Recurse ./*" @@ -34,6 +33,9 @@ steps: Write-Host "git clone https://github.com/$($repository.Name) ." git clone https://github.com/$($repository.Name) . + if ($LASTEXITCODE) { + exit $LASTEXITCODE + } Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" # This will use the default branch if repo.Commitish is empty git -c advice.detachedHead=false checkout $($repository.Commitish) @@ -44,65 +46,56 @@ steps: function SparseCheckout([Array]$paths, [Hashtable]$repository) { - $dir = $repository.WorkingDirectory - if (!$dir) { - $dir = "./$($repository.Name)" - } - New-Item $dir -ItemType Directory -Force - Push-Location $dir - - if (Test-Path .git/info/sparse-checkout) { - $hasInitialized = $true - Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step." - } else { - Write-Host "Repository $($repository.Name) is being initialized." - - Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." - git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . - if ($LASTEXITCODE) { - return (Clone $repository && Pop-Location) - } + if (Test-Path .git/info/sparse-checkout) { + $hasInitialized = $true + Write-Host "Repository $($repository.Name) has already been initialized. Skipping this step." + } else { + Write-Host "Repository $($repository.Name) is being initialized." - Write-Host "git sparse-checkout init" - git sparse-checkout init - if ($LASTEXITCODE) { - return (Clone $repository && Pop-Location) - } + Write-Host "git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) ." + git clone --no-checkout --filter=tree:0 https://github.com/$($repository.Name) . + if ($LASTEXITCODE) { + throw + } - # Set non-cone mode otherwise path filters will not work in git >= 2.37.0 - # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits - Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'" - git sparse-checkout set --no-cone '/*' '!/*/' '/eng' - if ($LASTEXITCODE) { - return (Clone $repository && Pop-Location) - } + Write-Host "git sparse-checkout init" + git sparse-checkout init + if ($LASTEXITCODE) { + throw } - # Prevent wildcard expansion in Invoke-Expression (e.g. for checkout path '/*') - $quotedPaths = $paths | ForEach-Object { "'$_'" } - $gitsparsecmd = "git sparse-checkout add $quotedPaths" - Write-Host $gitsparsecmd - Invoke-Expression -Command $gitsparsecmd + # Set non-cone mode otherwise path filters will not work in git >= 2.37.0 + # See https://github.blog/2022-06-27-highlights-from-git-2-37/#tidbits + Write-Host "git sparse-checkout set --no-cone '/*' '!/*/' '/eng'" + git sparse-checkout set --no-cone '/*' '!/*/' '/eng' if ($LASTEXITCODE) { - return (Clone $repository && Pop-Location) + throw } + } + + # Prevent wildcard expansion in Invoke-Expression (e.g. for checkout path '/*') + $quotedPaths = $paths | ForEach-Object { "'$_'" } + $gitsparsecmd = "git sparse-checkout add $quotedPaths" + Write-Host $gitsparsecmd + Invoke-Expression -Command $gitsparsecmd + if ($LASTEXITCODE) { + throw + } - Write-Host "Set sparse checkout paths to:" - Get-Content .git/info/sparse-checkout + Write-Host "Set sparse checkout paths to:" + Get-Content .git/info/sparse-checkout - # sparse-checkout commands after initial checkout will auto-checkout again - if (!$hasInitialized) { - Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" - # This will use the default branch if repo.Commitish is empty - git -c advice.detachedHead=false checkout $($repository.Commitish) - if ($LASTEXITCODE) { - return (Clone $repository && Pop-Location) - } - } else { - Write-Host "Skipping checkout as repo has already been initialized" + # sparse-checkout commands after initial checkout will auto-checkout again + if (!$hasInitialized) { + Write-Host "git -c advice.detachedHead=false checkout $($repository.Commitish)" + # This will use the default branch if repo.Commitish is empty + git -c advice.detachedHead=false checkout $($repository.Commitish) + if ($LASTEXITCODE) { + throw } - - Pop-Location + } else { + Write-Host "Skipping checkout as repo has already been initialized" + } } # Paths may be sourced as a yaml object literal OR a dynamically generated variable json string. @@ -111,7 +104,30 @@ steps: # Replace windows backslash paths, as Azure Pipelines default directories are sometimes formatted like 'D:\a\1\s' $repositories = '${{ convertToJson(parameters.Repositories) }}' -replace '\\', '/' | ConvertFrom-Json -AsHashtable foreach ($repo in $Repositories) { - SparseCheckout $paths $repo + $dir = $repo.WorkingDirectory + if (!$dir) { + $dir = "./$($repo.Name)" + } + New-Item $dir -ItemType Directory -Force + Push-Location $dir + + try { + # Enable global override if there are sparse checkout issues + if ('$(DisableSparseCheckout)' -ne 'true') { + try { + SparseCheckout $paths $repo + } catch { + # Fallback to full clone if sparse checkout is not working properly + Write-Warning "Sparse checkout failed, falling back to full clone" + Clone $repo + } + } else { + Write-Warning "Sparse checkout disabled, performing full clone" + Clone $repo + } + } finally { + Pop-Location + } } pwsh: true workingDirectory: $(System.DefaultWorkingDirectory) From 5cb47bd3b385869ee39c8664bc9961dd832abe7b Mon Sep 17 00:00:00 2001 From: Ben Broderick Phillips Date: Mon, 25 Jul 2022 15:50:09 -0400 Subject: [PATCH 3/3] Use SkipSparseCheckout variable name --- eng/common/pipelines/templates/steps/sparse-checkout.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/sparse-checkout.yml b/eng/common/pipelines/templates/steps/sparse-checkout.yml index 3b664154a1..efc8c6ed92 100644 --- a/eng/common/pipelines/templates/steps/sparse-checkout.yml +++ b/eng/common/pipelines/templates/steps/sparse-checkout.yml @@ -113,7 +113,7 @@ steps: try { # Enable global override if there are sparse checkout issues - if ('$(DisableSparseCheckout)' -ne 'true') { + if ('$(SkipSparseCheckout)' -ne 'true') { try { SparseCheckout $paths $repo } catch {