From 219785f8743e2b7890b3cc7092485fb0d502eff2 Mon Sep 17 00:00:00 2001 From: Daniel Jurek <djurek@microsoft.com> Date: Tue, 15 Feb 2022 11:19:07 -0800 Subject: [PATCH] Add logic to generate docs.ms ToC --- eng/common/scripts/Update-DocsMsToc.ps1 | 313 ++++++++++++++++++++++++ eng/common/scripts/common.ps1 | 6 +- 2 files changed, 318 insertions(+), 1 deletion(-) create mode 100644 eng/common/scripts/Update-DocsMsToc.ps1 diff --git a/eng/common/scripts/Update-DocsMsToc.ps1 b/eng/common/scripts/Update-DocsMsToc.ps1 new file mode 100644 index 000000000000..47edd401b97f --- /dev/null +++ b/eng/common/scripts/Update-DocsMsToc.ps1 @@ -0,0 +1,313 @@ +<# +.SYNOPSIS +Update unified ToC file for publishing reference docs on docs.microsoft.com + +.DESCRIPTION +Given a doc repo location and a location to output the ToC generate a Unified +Table of Contents: + +* Get list of packages onboarded to docs.microsoft.com (domain specific) +* Get metadata for onboarded packages from metadata CSV +* Build a sorted list of services +* Add ToC nodes for the service +* Add "Core" packages to the bottom of the ToC under "Other" + +ToC node layout: +* Service (service level overview page) + * Client Package 1 (package level overview page) + * Client Package 2 (package level overview page) + ... + * Management + * Management Package 1 + * Management Package 2 + ... + +.PARAMETER DocRepoLocation +Location of the documentation repo. This repo may be sparsely checked out +depending on the requirements for the domain + +.PARAMETER OutputLocation +Output location for unified reference yml file + +#> + +param( + [Parameter(Mandatory = $true)] + [string] $DocRepoLocation, + + [Parameter(Mandatory = $true)] + [string] $OutputLocation +) +Set-StrictMode -Version 3 +. $PSScriptRoot/common.ps1 +. $PSScriptRoot/Helpers/PSModule-Helpers.ps1 + +Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module + +function GetClientPackageNode($clientPackage) { + $packageInfo = &$GetDocsMsTocDataFn ` + -packageMetadata $clientPackage ` + -docRepoLocation $DocRepoLocation + + return [PSCustomObject]@{ + name = $packageInfo.PackageTocHeader + href = $packageInfo.PackageLevelReadmeHref + # This is always one package and it must be an array + children = $packageInfo.TocChildren + }; +} + +function GetPackageKey($pkg) { + $pkgKey = $pkg.Package + $groupId = $null + + if ($pkg.PSObject.Members.Name -contains "GroupId") { + $groupId = $pkg.GroupId + } + + if ($groupId) { + $pkgKey = "${groupId}:${pkgKey}" + } + + return $pkgKey +} + +function GetPackageLookup($packageList) { + $packageLookup = @{} + + foreach ($pkg in $packageList) { + $pkgKey = GetPackageKey $pkg + + # We want to prefer updating non-hidden packages but if there is only + # a hidden entry then we will return that + if (!$packageLookup.ContainsKey($pkgKey) -or $packageLookup[$pkgKey].Hide -eq "true") { + $packageLookup[$pkgKey] = $pkg + } + else { + # Warn if there are more then one non-hidden package + if ($pkg.Hide -ne "true") { + Write-Host "Found more than one package entry for $($pkg.Package) selecting the first non-hidden one." + } + } + + if ($pkg.PSObject.Members.Name -contains "GroupId" -and ($pkg.New -eq "true") -and $pkg.Package) { + $pkgKey = $pkg.Package + if (!$packageLookup.ContainsKey($pkgKey)) { + $packageLookup[$pkgKey] = $pkg + } + else { + $packageValue = $packageLookup[$pkgKey] + Write-Host "Found more than one package entry for $($packageValue.Package) selecting the first one with groupId $($packageValue.GroupId), skipping $($pkg.GroupId)" + } + } + } + + return $packageLookup +} + +$onboardedPackages = &$GetOnboardedDocsMsPackagesFn ` + -DocRepoLocation $DocRepoLocation + +# This criteria is different from criteria used in `Update-DocsMsPackages.ps1` +# because we need to generate ToCs for packages which are not necessarily "New" +# in the metadata AND onboard legacy packages (which `Update-DocsMsPackages.ps1` +# does not do) +$metadata = (Get-CSVMetadata).Where({ + $_.Package ` + -and $onboardedPackages.ContainsKey($_.Package) ` + -and $_.Hide -ne 'true' + }) + +$fileMetadata = @() +foreach ($metadataFile in Get-ChildItem "$DocRepoLocation/metadata/*/*.json" -Recurse) { + $fileContent = Get-Content $metadataFile -Raw + $metadataEntry = ConvertFrom-Json $fileContent + + if ($metadataEntry) { + $fileMetadata += $metadataEntry + } +} + +# Add file metadata information to package metadata from metadata CSV. Because +# metadata can exist for packages in both preview and GA there may be more than +# one file metadata entry. If that is the case keep the first entry found. We +# only use the `DirectoryPath` property from the json file metadata at this time +for ($i = 0; $i -lt $metadata.Count; $i++) { + foreach ($fileEntry in $fileMetadata) { + if ($fileEntry.Name -eq $metadata[$i].Package) { + if ($metadata[$i].PSObject.Members.Name -contains "FileMetadata") { + Write-Host "File metadata already added for $($metadata[$i].Package). Keeping the first entry found." + continue + } + + Add-Member ` + -InputObject $metadata[$i] ` + -MemberType NoteProperty ` + -Name FileMetadata ` + -Value $fileEntry + } + } +} + +$packagesForToc = @{} +foreach ($metadataEntry in (GetPackageLookup $metadata).Values) { + if (!$metadataEntry.ServiceName) { + LogWarning "Empty ServiceName for package `"$($metadataEntry.Package)`". Skipping." + continue + } + $packagesForToc[$metadataEntry.Package] = $metadataEntry +} + +# Get unique service names and sort alphabetically to act as the service nodes +# in the ToC +$services = @{} +foreach ($package in $packagesForToc.Values) { + if ($package.ServiceName -eq 'Other') { + # Skip packages under the service category "Other". Those will be handled + # later + continue + } + if (!$services.ContainsKey($package.ServiceName)) { + $services[$package.ServiceName] = $true + } +} +$serviceNameList = $services.Keys | Sort-Object + + +$toc = @() +foreach ($service in $serviceNameList) { + Write-Host "Building service: $service" + + $packageItems = @() + + # Client packages get individual entries + $clientPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('client' -eq $_.Type) }) + $clientPackages = $clientPackages | Sort-Object -Property Package + if ($clientPackages) { + foreach ($clientPackage in $clientPackages) { + $packageItems += GetClientPackageNode -clientPackage $clientPackage + } + } + + + # All management packages go under a single `Management` header in the ToC + $mgmtPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('mgmt' -eq $_.Type) }) + $mgmtPackages = $mgmtPackages | Sort-Object -Property Package + if ($mgmtPackages) { + $children = &$GetDocsMsTocChildrenForManagementPackagesFn ` + -packageMetadata $mgmtPackages ` + -docRepoLocation $DocRepoLocation + + $packageItems += [PSCustomObject]@{ + name = 'Management' + # There could be multiple packages, ensure this is treated as an array + # even if it is a single package + children = @($children) + }; + } + + $uncategorizedPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and !(@('client', 'mgmt') -contains $_.Type) }) + if ($uncategorizedPackages) { + foreach ($package in $uncategorizedPackages) { + LogWarning "Uncategorized package for service: $service - $($package.Package). Package not onboarded." + } + } + + $serviceReadmeBaseName = $service.ToLower().Replace(' ', '-') + $serviceTocEntry = [PSCustomObject]@{ + name = $service; + href = "~/docs-ref-services/{moniker}/$serviceReadmeBaseName.md" + landingPageType = 'Service' + items = @($packageItems) + } + + $toc += $serviceTocEntry +} + +# Core packages belong under the "Other" node in the ToC +$otherPackageItems = New-Object -TypeName System.Collections.Generic.List[PSCustomObject] +$otherPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq 'Other' }) +$otherPackages = $otherPackages | Sort-Object -Property DisplayName + +if ($otherPackages) { + foreach ($otherPackage in $otherPackages) { + $segments = $otherPackage.DisplayName.Split('-').ForEach({ $_.Trim() }) + + + if ($segments.Count -gt 1) { + $currentNode = $otherPackageItems + + # Iterate up to the penultimate item in the array so that the final item + # in the array can be added as a leaf node. Since the array always has at + # least two elements this iteration will cover at least the first element. + # e.g. @(0, 1)[0..0] => 0 + foreach ($segment in $segments[0..($segments.Count - 2)]) { + $matchingNode = $currentNode.Where({ $_.name -eq $segment }) + + # ToC nodes can be "branches" which contain 0 or more branch + # or leaf nodes in an "items" field OR they can be leaf nodes which have + # a "children" field which can only contain package names or namespaces. + # A node cannot contain both "items" and "children". If a node already + # has a "children" field then it is a leaf node and cannot take + # additional branch nodes. + # Children are added using the `GetClientPackageNode` function + if ($matchingNode -and $matchingNode.PSObject.Members.Name -contains "children") { + LogWarning "Cannot create nested entry for package $($otherPackage.Package) because Segment `"$segment`" in the DisplayName $($otherPackage.DisplayName) is already a leaf node. Excluding package: $($otherPackage.Package)" + $currentNode = $null + break + } + + if ($matchingNode) { + $currentNode = $matchingNode[0].items + } + else { + $newNode = [PSCustomObject]@{ + name = $segment + landingPageType = 'Service' + items = New-Object -TypeName System.Collections.Generic.List[PSCustomObject] + } + $currentNode.Add($newNode) + $currentNode = $newNode.items + } + } + + if ($null -ne $currentNode) { + $otherPackage.DisplayName = $segments[$segments.Count - 1] + $currentNode.Add((GetClientPackageNode $otherPackage)) + } + + } + else { + $otherPackageItems.Add((GetClientPackageNode $otherPackage)) + } + } +} + +$toc += [PSCustomObject]@{ + name = 'Other'; + landingPageType = 'Service'; + items = $otherPackageItems + @( + [PSCustomObject]@{ + name = "Uncategorized Packages"; + landingPageType = 'Service'; + # All onboarded packages which have not been placed in the ToC will be + # handled by the docs system here. In this case the list would consist of + # packages whose ServiceName field is empty in the metadata. + children = @('**'); + } + ) +} + +$output = @([PSCustomObject]@{ + name = 'Reference'; + landingPageType = 'Root'; + expanded = $false; + items = $toc + }) + +if (Test-Path "Function:$UpdateDocsMsTocFn") { + $output = &$UpdateDocsMsTocFn -toc $output +} + +$outputYaml = ConvertTo-Yaml $output +Set-Content -Path $OutputLocation -Value $outputYaml diff --git a/eng/common/scripts/common.ps1 b/eng/common/scripts/common.ps1 index 0f2dfe2f76f5..c01f9c9f53ac 100644 --- a/eng/common/scripts/common.ps1 +++ b/eng/common/scripts/common.ps1 @@ -45,4 +45,8 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex" $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview" $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}" -$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" \ No newline at end of file +$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages" +$GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages" +$GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData" +$GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages" +$UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"