Skip to content

Enable parallel runs of platyPS per version #4171

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 18, 2019
Merged

Enable parallel runs of platyPS per version #4171

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
3 commits
Select commit Hold shift + click to select a range
f6a0f78
make building versions of docs in parallel
SteveL-MSFT Apr 17, 2019
22b5b5b
fix typo in .gitignore
SteveL-MSFT Apr 18, 2019
3ee2f75
install ThreadJob if not available
SteveL-MSFT Apr 18, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .gitignore
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ _site/
Tools/NuGet/
_site/
maml/
updateablehelp/
updatablehelp/


.openpublishing.build.mdproj
Expand Down
211 changes: 125 additions & 86 deletions build.ps1
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,37 +5,7 @@ param(

# Turning off the progress display, by default
$global:ProgressPreference = 'SilentlyContinue'
if($ShowProgress){$ProgressPreference = 'Continue'}

function Get-ContentWithoutHeader {
param(
$path
)

$doc = Get-Content $path -Encoding UTF8
$start = $end = -1

# search the first 30 lines for the Yaml header
# no yaml header in our docset will ever be that long

for ($x = 0; $x -lt 30; $x++) {
if ($doc[$x] -eq '---') {
if ($start -eq -1) {
$start = $x
} else {
if ($end -eq -1) {
$end = $x+1
break
}
}
}
}
if ($end -gt $start) {
Write-Output ($doc[$end..$($doc.count)] -join "`r`n")
} else {
Write-Output ($doc -join "`r`n")
}
}
if ($ShowProgress) { $ProgressPreference = 'Continue' }

[Net.ServicePointManager]::SecurityProtocol = [Net.ServicePointManager]::SecurityProtocol -bor [Net.SecurityProtocolType]::Tls12

Expand All @@ -50,86 +20,155 @@ Invoke-WebRequest -Uri $pandocSourceURL -OutFile $pandocZipPath
Expand-Archive -Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion-windows.zip") -DestinationPath $pandocDestinationPath -Force
$pandocExePath = Join-Path (Join-Path $pandocDestinationPath "pandoc-$panDocVersion") "pandoc.exe"

# Install ThreadJob if not available
$threadJob = Get-Module ThreadJob -ListAvailable
if ($null -eq $threadjob) {
Install-Module ThreadJob -RequiredVersion 1.1.2 -Scope CurrentUser -Force
}

# Find the reference folder path w.r.t the script
$ReferenceDocset = Join-Path $PSScriptRoot 'reference'

# Variable to collect any errors in during processing
$allErrors = @()

# Go through all the directories in the reference folder
$jobs = [System.Collections.Generic.List[object]]::new()
Get-ChildItem $ReferenceDocset -Directory -Exclude 'docs-conceptual', 'mapping', 'bread' | ForEach-Object -Process {
$Version = $_.Name
Write-Verbose -Verbose "Version = $Version"
$job = Start-ThreadJob -Name $_.Name -ArgumentList @($SkipCabs,$pandocExePath,$PSScriptRoot,$_) -ScriptBlock {
param($SkipCabs, $pandocExePath, $WorkingDirectory, $DocSet)

$tempDir = [System.IO.Path]::GetTempPath()
$workingDir = Join-Path $tempDir $DocSet.Name
$workingDir = New-Item -ItemType Directory -Path $workingDir -Force
Set-Location $WorkingDir

function Get-ContentWithoutHeader {
param(
$path
)

$doc = Get-Content $path -Encoding UTF8
$start = $end = -1

# search the first 30 lines for the Yaml header
# no yaml header in our docset will ever be that long

for ($x = 0; $x -lt 30; $x++) {
if ($doc[$x] -eq '---') {
if ($start -eq -1) {
$start = $x
}
else {
if ($end -eq -1) {
$end = $x + 1
break
}
}
}
}
if ($end -gt $start) {
Write-Output ($doc[$end..$($doc.count)] -join "`r`n")
}
else {
Write-Output ($doc -join "`r`n")
}
}

$VersionFolder = $_.FullName
Write-Verbose -Verbose "VersionFolder = $VersionFolder"
$Version = $DocSet.Name
Write-Verbose -Verbose "Version = $Version"

# For each of the directories, go through each module folder
Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process {
$ModuleName = $_.Name
Write-Verbose -Verbose "ModuleName = $ModuleName"
$VersionFolder = $DocSet.FullName
Write-Verbose -Verbose "VersionFolder = $VersionFolder"

$ModulePath = Join-Path $VersionFolder $ModuleName
Write-Verbose -Verbose "ModulePath = $ModulePath"
# For each of the directories, go through each module folder
Get-ChildItem $VersionFolder -Directory | ForEach-Object -Process {
$ModuleName = $_.Name
Write-Verbose -Verbose "ModuleName = $ModuleName"

$LandingPage = Join-Path $ModulePath "$ModuleName.md"
Write-Verbose -Verbose "LandingPage = $LandingPage"
$ModulePath = Join-Path $VersionFolder $ModuleName
Write-Verbose -Verbose "ModulePath = $ModulePath"

$MamlOutputFolder = Join-Path "$PSScriptRoot\maml" "$Version\$ModuleName"
Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder"
$LandingPage = Join-Path $ModulePath "$ModuleName.md"
Write-Verbose -Verbose "LandingPage = $LandingPage"

$CabOutputFolder = Join-Path "$PSScriptRoot\updatablehelp" "$Version\$ModuleName"
Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder"
$MamlOutputFolder = Join-Path "$WorkingDirectory\maml" "$Version\$ModuleName"
Write-Verbose -Verbose "MamlOutputFolder = $MamlOutputFolder"

if (-not (Test-Path $MamlOutputFolder)) {
New-Item $MamlOutputFolder -ItemType Directory -Force > $null
}
$CabOutputFolder = Join-Path "$WorkingDirectory\updatablehelp" "$Version\$ModuleName"
Write-Verbose -Verbose "CabOutputFolder = $CabOutputFolder"

# Process the about topics if any
$AboutFolder = Join-Path $ModulePath "About"

if (Test-Path $AboutFolder) {
Write-Verbose -Verbose "AboutFolder = $AboutFolder"
Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object {
$aboutFileFullName = $_.FullName
$aboutFileOutputName = "$($_.BaseName).help.txt"
$aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName

$pandocArgs = @(
"--from=gfm",
"--to=plain+multiline_tables+inline_code_attributes",
"--columns=75",
"--output=$aboutFileOutputFullName",
"--quiet"
)

Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs
if (-not (Test-Path $MamlOutputFolder)) {
New-Item $MamlOutputFolder -ItemType Directory -Force > $null
}
}

try {
# For each module, create a single maml help file
# Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors
New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop
# Process the about topics if any
$AboutFolder = Join-Path $ModulePath "About"

if (Test-Path $AboutFolder) {
Write-Verbose -Verbose "AboutFolder = $AboutFolder"
Get-ChildItem "$aboutfolder/about_*.md" | ForEach-Object {
$aboutFileFullName = $_.FullName
$aboutFileOutputName = "$($_.BaseName).help.txt"
$aboutFileOutputFullName = Join-Path $MamlOutputFolder $aboutFileOutputName

$pandocArgs = @(
"--from=gfm",
"--to=plain+multiline_tables+inline_code_attributes",
"--columns=75",
"--output=$aboutFileOutputFullName",
"--quiet"
)

Get-ContentWithoutHeader $aboutFileFullName | & $pandocExePath $pandocArgs
}
}

try {
# For each module, create a single maml help file
# Adding warningaction=stop to throw errors for all warnings, erroraction=stop to make them terminating errors
New-ExternalHelp -Path $ModulePath -OutputPath $MamlOutputFolder -Force -WarningAction Stop -ErrorAction Stop

# For each module, create update-help help files (cab and helpinfo.xml files)
if (-not $SkipCabs) {
$cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder
# For each module, create update-help help files (cab and helpinfo.xml files)
if (-not $SkipCabs) {
$cabInfo = New-ExternalHelpCab -CabFilesFolder $MamlOutputFolder -LandingPagePath $LandingPage -OutputFolder $CabOutputFolder

# Only output the cab fileinfo object
if ($cabInfo.Count -eq 8) {$cabInfo[-1].FullName}
# Only output the cab fileinfo object
if ($cabInfo.Count -eq 8) { $cabInfo[-1].FullName }
}
}
catch {
Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_
}
}
catch {
$allErrors += $_
Write-Error -Message "PlatyPS failure: $ModuleName -- $Version" -Exception $_

Remove-Item $workingDir -Force -ErrorAction SilentlyContinue
}
Write-Verbose -Verbose "Started job for $($_.Name)"
$jobs += $job
}

$null = $jobs | Wait-Job

# Variable to collect any errors in during processing
$allErrors = [System.Collections.Generic.List[string]]::new()
foreach ($job in $jobs) {
Write-Verbose -Verbose "$($job.Name) output:"
if ($job.Verbose.Count -gt 0) {
foreach ($verboseMessage in $job.Verbose) {
Write-Verbose -Verbose $verboseMessage
}
}

if ($job.State -eq "Failed") {
$allErrors += "$($job.Name) failed due to unhandled exception"
}

if ($job.Error.Count -gt 0) {
$allErrors += "$($job.Name) failed with errors:"
$allErrors += $job.Error.ReadAll()
}
}

# If the above block, produced any errors, throw and fail the job
if ($allErrors) {
if ($allErrors.Count -gt 0) {
$allErrors
throw "There are errors during platyPS run!`nPlease fix your markdown to comply with the schema: https://github.com/PowerShell/platyPS/blob/master/platyPS.schema.md"
}