Add logic to generate docs.ms ToC (#2759)

Originally proposed here -- Azure/azure-sdk-for-js#19915

eng/common/scripts/Update-DocsMsToc.ps1

@@ -0,0 +1,313 @@
+<#
+.SYNOPSIS
+Update unified ToC file for publishing reference docs on docs.microsoft.com
+
+.DESCRIPTION
+Given a doc repo location and a location to output the ToC generate a Unified
+Table of Contents:
+
+* Get list of packages onboarded to docs.microsoft.com (domain specific)
+* Get metadata for onboarded packages from metadata CSV
+* Build a sorted list of services
+* Add ToC nodes for the service
+* Add "Core" packages to the bottom of the ToC under "Other"
+
+ToC node layout:
+* Service (service level overview page)
+  * Client Package 1 (package level overview page)
+  * Client Package 2 (package level overview page)
+  ...
+  * Management
+    * Management Package 1
+    * Management Package 2
+    ...
+
+.PARAMETER DocRepoLocation
+Location of the documentation repo. This repo may be sparsely checked out
+depending on the requirements for the domain
+
+.PARAMETER OutputLocation
+Output location for unified reference yml file
+
+#>
+
+param(
+  [Parameter(Mandatory = $true)]
+  [string] $DocRepoLocation,
+
+  [Parameter(Mandatory = $true)]
+  [string] $OutputLocation
+)
+Set-StrictMode -Version 3
+. $PSScriptRoot/common.ps1
+. $PSScriptRoot/Helpers/PSModule-Helpers.ps1
+
+Install-ModuleIfNotInstalled "powershell-yaml" "0.4.1" | Import-Module
+
+function GetClientPackageNode($clientPackage) {
+  $packageInfo = &$GetDocsMsTocDataFn `
+    -packageMetadata $clientPackage `
+    -docRepoLocation $DocRepoLocation
+
+  return [PSCustomObject]@{
+    name     = $packageInfo.PackageTocHeader
+    href     = $packageInfo.PackageLevelReadmeHref
+    # This is always one package and it must be an array
+    children = $packageInfo.TocChildren
+  };
+}
+
+function GetPackageKey($pkg) {
+  $pkgKey = $pkg.Package
+  $groupId = $null
+
+  if ($pkg.PSObject.Members.Name -contains "GroupId") {
+    $groupId = $pkg.GroupId
+  }
+
+  if ($groupId) {
+    $pkgKey = "${groupId}:${pkgKey}"
+  }
+
+  return $pkgKey
+}
+
+function GetPackageLookup($packageList) {
+  $packageLookup = @{}
+
+  foreach ($pkg in $packageList) {
+    $pkgKey = GetPackageKey $pkg
+
+    # We want to prefer updating non-hidden packages but if there is only
+    # a hidden entry then we will return that
+    if (!$packageLookup.ContainsKey($pkgKey) -or $packageLookup[$pkgKey].Hide -eq "true") {
+      $packageLookup[$pkgKey] = $pkg
+    }
+    else {
+      # Warn if there are more then one non-hidden package
+      if ($pkg.Hide -ne "true") {
+        Write-Host "Found more than one package entry for $($pkg.Package) selecting the first non-hidden one."
+      }
+    }
+
+    if ($pkg.PSObject.Members.Name -contains "GroupId" -and ($pkg.New -eq "true") -and $pkg.Package) {
+      $pkgKey = $pkg.Package
+      if (!$packageLookup.ContainsKey($pkgKey)) {
+        $packageLookup[$pkgKey] = $pkg
+      }
+      else {
+        $packageValue = $packageLookup[$pkgKey]
+        Write-Host "Found more than one package entry for $($packageValue.Package) selecting the first one with groupId $($packageValue.GroupId), skipping $($pkg.GroupId)"
+      }
+    }
+  }
+
+  return $packageLookup
+}
+
+$onboardedPackages = &$GetOnboardedDocsMsPackagesFn `
+  -DocRepoLocation $DocRepoLocation
+
+# This criteria is different from criteria used in `Update-DocsMsPackages.ps1`
+# because we need to generate ToCs for packages which are not necessarily "New"
+# in the metadata AND onboard legacy packages (which `Update-DocsMsPackages.ps1`
+# does not do)
+$metadata = (Get-CSVMetadata).Where({
+    $_.Package `
+      -and $onboardedPackages.ContainsKey($_.Package) `
+      -and $_.Hide -ne 'true'
+  })
+
+$fileMetadata = @()
+foreach ($metadataFile in Get-ChildItem "$DocRepoLocation/metadata/*/*.json" -Recurse) {
+  $fileContent = Get-Content $metadataFile -Raw
+  $metadataEntry = ConvertFrom-Json $fileContent
+
+  if ($metadataEntry) {
+    $fileMetadata += $metadataEntry
+  }
+}
+
+# Add file metadata information to package metadata from metadata CSV. Because
+# metadata can exist for packages in both preview and GA there may be more than
+# one file metadata entry. If that is the case keep the first entry found. We
+# only use the `DirectoryPath` property from the json file metadata at this time
+for ($i = 0; $i -lt $metadata.Count; $i++) {
+  foreach ($fileEntry in $fileMetadata) {
+    if ($fileEntry.Name -eq $metadata[$i].Package) {
+      if ($metadata[$i].PSObject.Members.Name -contains "FileMetadata") {
+        Write-Host "File metadata already added for $($metadata[$i].Package). Keeping the first entry found."
+        continue
+      }
+
+      Add-Member `
+        -InputObject $metadata[$i] `
+        -MemberType NoteProperty `
+        -Name FileMetadata `
+        -Value $fileEntry
+    }
+  }
+}
+
+$packagesForToc = @{}
+foreach ($metadataEntry in (GetPackageLookup $metadata).Values) {
+  if (!$metadataEntry.ServiceName) {
+    LogWarning "Empty ServiceName for package `"$($metadataEntry.Package)`". Skipping."
+    continue
+  }
+  $packagesForToc[$metadataEntry.Package] = $metadataEntry
+}
+
+# Get unique service names and sort alphabetically to act as the service nodes
+# in the ToC
+$services = @{}
+foreach ($package in $packagesForToc.Values) {
+  if ($package.ServiceName -eq 'Other') {
+    # Skip packages under the service category "Other". Those will be handled
+    # later
+    continue
+  }
+  if (!$services.ContainsKey($package.ServiceName)) {
+    $services[$package.ServiceName] = $true
+  }
+}
+$serviceNameList = $services.Keys | Sort-Object
+
+
+$toc = @()
+foreach ($service in $serviceNameList) {
+  Write-Host "Building service: $service"
+
+  $packageItems = @()
+
+  # Client packages get individual entries
+  $clientPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('client' -eq $_.Type) })
+  $clientPackages = $clientPackages | Sort-Object -Property Package
+  if ($clientPackages) {
+    foreach ($clientPackage in $clientPackages) {
+      $packageItems += GetClientPackageNode -clientPackage $clientPackage
+    }
+  }
+
+
+  # All management packages go under a single `Management` header in the ToC
+  $mgmtPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and ('mgmt' -eq $_.Type) })
+  $mgmtPackages = $mgmtPackages | Sort-Object -Property Package
+  if ($mgmtPackages) {
+    $children = &$GetDocsMsTocChildrenForManagementPackagesFn `
+      -packageMetadata $mgmtPackages `
+      -docRepoLocation $DocRepoLocation
+
+    $packageItems += [PSCustomObject]@{
+      name     = 'Management'
+      # There could be multiple packages, ensure this is treated as an array
+      # even if it is a single package
+      children = @($children)
+    };
+  }
+
+  $uncategorizedPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq $service -and !(@('client', 'mgmt') -contains $_.Type) })
+  if ($uncategorizedPackages) {
+    foreach ($package in $uncategorizedPackages) {
+      LogWarning "Uncategorized package for service: $service - $($package.Package). Package not onboarded."
+    }
+  }
+
+  $serviceReadmeBaseName = $service.ToLower().Replace(' ', '-')
+  $serviceTocEntry = [PSCustomObject]@{
+    name            = $service;
+    href            = "~/docs-ref-services/{moniker}/$serviceReadmeBaseName.md"
+    landingPageType = 'Service'
+    items           = @($packageItems)
+  }
+
+  $toc += $serviceTocEntry
+}
+
+# Core packages belong under the "Other" node in the ToC
+$otherPackageItems = New-Object -TypeName System.Collections.Generic.List[PSCustomObject]
+$otherPackages = $packagesForToc.Values.Where({ $_.ServiceName -eq 'Other' })
+$otherPackages = $otherPackages | Sort-Object -Property DisplayName
+
+if ($otherPackages) {
+  foreach ($otherPackage in $otherPackages) {
+    $segments = $otherPackage.DisplayName.Split('-').ForEach({ $_.Trim() })
+
+
+    if ($segments.Count -gt 1) {
+      $currentNode = $otherPackageItems
+
+      # Iterate up to the penultimate item in the array so that the final item
+      # in the array can be added as a leaf node. Since the array always has at
+      # least two elements this iteration will cover at least the first element.
+      # e.g. @(0, 1)[0..0] => 0
+      foreach ($segment in $segments[0..($segments.Count - 2)]) {
+        $matchingNode = $currentNode.Where({ $_.name -eq $segment })
+
+        # ToC nodes can be "branches" which contain 0 or more branch
+        # or leaf nodes in an "items" field OR they can be leaf nodes which have
+        # a "children" field which can only contain package names or namespaces.
+        # A node cannot contain both "items" and "children". If a node already
+        # has a "children" field then it is a leaf node and cannot take
+        # additional branch nodes.
+        # Children are added using the `GetClientPackageNode` function
+        if ($matchingNode -and $matchingNode.PSObject.Members.Name -contains "children") {
+          LogWarning "Cannot create nested entry for package $($otherPackage.Package) because Segment `"$segment`" in the DisplayName $($otherPackage.DisplayName) is already a leaf node. Excluding package: $($otherPackage.Package)"
+          $currentNode = $null
+          break
+        }
+
+        if ($matchingNode) {
+          $currentNode = $matchingNode[0].items
+        }
+        else {
+          $newNode = [PSCustomObject]@{
+            name            = $segment
+            landingPageType = 'Service'
+            items           = New-Object -TypeName System.Collections.Generic.List[PSCustomObject]
+          }
+          $currentNode.Add($newNode)
+          $currentNode = $newNode.items
+        }
+      }
+
+      if ($null -ne $currentNode) {
+        $otherPackage.DisplayName = $segments[$segments.Count - 1]
+        $currentNode.Add((GetClientPackageNode $otherPackage))
+      }
+
+    }
+    else {
+      $otherPackageItems.Add((GetClientPackageNode $otherPackage))
+    }
+  }
+}
+
+$toc += [PSCustomObject]@{
+  name            = 'Other';
+  landingPageType = 'Service';
+  items           = $otherPackageItems + @(
+    [PSCustomObject]@{
+      name            = "Uncategorized Packages";
+      landingPageType = 'Service';
+      # All onboarded packages which have not been placed in the ToC will be
+      # handled by the docs system here. In this case the list would consist of
+      # packages whose ServiceName field is empty in the metadata.
+      children        = @('**');
+    }
+  )
+}
+
+$output = @([PSCustomObject]@{
+    name            = 'Reference';
+    landingPageType = 'Root';
+    expanded        = $false;
+    items           = $toc
+  })
+
+if (Test-Path "Function:$UpdateDocsMsTocFn") {
+  $output = &$UpdateDocsMsTocFn -toc $output
+}
+
+$outputYaml = ConvertTo-Yaml $output
+Set-Content -Path $OutputLocation -Value $outputYaml

eng/common/scripts/common.ps1

@@ -45,4 +45,8 @@ $GetDocsMsDevLanguageSpecificPackageInfoFn = "Get-${Language}-DocsMsDevLanguageS
 $GetGithubIoDocIndexFn = "Get-${Language}-GithubIoDocIndex"
 $FindArtifactForApiReviewFn = "Find-${Language}-Artifacts-For-Apireview"
 $TestProxyTrustCertFn = "Import-Dev-Cert-${Language}"
-$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
+$ValidateDocsMsPackagesFn = "Validate-${Language}-DocMsPackages"
+$GetOnboardedDocsMsPackagesFn = "Get-${Language}-OnboardedDocsMsPackages"
+$GetDocsMsTocDataFn = "Get-${Language}-DocsMsTocData"
+$GetDocsMsTocChildrenForManagementPackagesFn = "Get-${Language}-DocsMsTocChildrenForManagementPackages"
+$UpdateDocsMsTocFn = "Get-${Language}-UpdatedDocsMsToc"