Onboard packages for docs in daily docindex run (Azure#22476)

* Add onboarding (with test)

* Try faster checkout

* Use $(DefaultBranch)

* Review feedback: Add comments to parsing logic, more cleanup in releas pipeline

* Update eng/scripts/Language-Settings.ps1

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

Co-authored-by: Wes Haggard <weshaggard@users.noreply.github.com>

eng/pipelines/docindex.yml

@@ -2,3 +2,45 @@ trigger: none
 
 jobs:
   - template: /eng/common/pipelines/templates/jobs/docindex.yml
+
+  - job: UpdateDocsMsBuildConfig
+    pool:
+      vmImage: ubuntu-20.04
+    variables:
+      DocRepoLocation: $(Pipeline.Workspace)/docs
+      DocRepoOwner: Azure
+      DocRepoName: azure-docs-sdk-dotnet
+    steps:
+      # Checkout the eng folder from the SDK repo
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          SkipDefaultCheckout: true
+
+      # Sync docs repo
+      - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
+        parameters:
+          Paths:
+            - bundlepackages
+          Repositories:
+            - Name: $(DocRepoOwner)/$(DocRepoName)
+              WorkingDirectory: $(DocRepoLocation)
+
+      # Call update docs ci script to onboard packages
+      - task: Powershell@2
+        inputs:
+          pwsh: true
+          filePath: eng/common/scripts/Update-DocsMsPackages.ps1
+          arguments: -DocRepoLocation $(DocRepoLocation)
+        displayName: Update Docs Onboarding
+
+      - template: /eng/common/pipelines/templates/steps/set-default-branch.yml
+
+      - template: /eng/common/pipelines/templates/steps/git-push-changes.yml
+        parameters:
+          BaseRepoBranch: $(DefaultBranch)
+          BaseRepoOwner: $(DocRepoOwner)
+          CommitMsg: "Update docs CI configuration"
+          TargetRepoName: $(DocRepoName)
+          TargetRepoOwner: $(DocRepoOwner)
+          WorkingDirectory: $(DocRepoLocation)
+

eng/pipelines/templates/stages/archetype-net-release.yml

@@ -151,13 +151,7 @@ stages:
                   deploy:
                     steps:
                       - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml
-                        parameters:
-                          paths:
-                            - .github/CODEOWNERS
-                      - template: /eng/common/pipelines/templates/steps/get-pr-owners.yml
-                        parameters:
-                          TargetVariable: "OwningGHUser"
-                          ServiceDirectory: ${{parameters.ServiceDirectory}}
+
                       - template: /eng/common/pipelines/templates/steps/docs-metadata-release.yml
                         parameters:
                           ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}-signed/${{artifact.name}}
@@ -171,7 +165,6 @@ stages:
                           ArtifactName: ${{artifact.name}}
                           Language: '.net'
                           DocRepoDestinationPath: 'api/overview/azure/'
-                          GHReviewersVariable: 'OwningGHUser'
                           CIConfigs: $(CIConfigs)
                           OnboardingBranch: 'onboarding'
                           CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}'

eng/scripts/Language-Settings.ps1

@@ -245,3 +245,201 @@ function GetExistingPackageVersions ($PackageName, $GroupId=$null)
     return $null
   }
 }
+
+function Get-dotnet-DocsMsMetadataForPackage($PackageInfo) {
+  $suffix = ''
+  $parsedVersion = [AzureEngSemanticVersion]::ParseVersionString($PackageInfo.Version)
+  if ($parsedVersion.IsPrerelease) { 
+    $suffix = '-pre'
+  }
+
+  New-Object PSObject -Property @{ 
+    DocsMsReadMeName = $PackageInfo.Name.ToLower() -replace "." , ""
+    LatestReadMeLocation = 'api/overview/azure'
+    PreviewReadMeLocation = 'api/overview/azure'
+    Suffix = $suffix
+  }
+}
+
+
+# Details on CSV schema:
+# https://review.docs.microsoft.com/en-us/help/onboard/admin/reference/dotnet/documenting-nuget?branch=master#set-up-the-ci-job
+# 
+# PowerShell's included Import-Csv cmdlet is not sufficient for parsing this 
+# format because it does not easily handle rows whose number of columns is 
+# greater than the number of columns in the first row. We must manually parse
+# this CSV file.
+function Get-DocsCiConfig($configPath) {
+  Write-Host "Loading csv from $configPath"
+  $output = @()
+  foreach ($row in Get-Content $configPath) {
+      # CSV format: 
+      # {package_moniker_base_string},{package_ID},{version_1},{version_2},...,{version_N}
+      #
+      # The {package_ID} field can contain optional properties denoted by square
+      # brackets of the format: [key=value;key=value;...]
+
+      # Split the rows by the comma
+      $fields = $row.Split(',')
+
+      # If the {package_ID} field contains optional properties inside square
+      # brackets, parse those properties into key value pairs. In the case of 
+      # duplicate keys, the last one wins.
+      $rawProperties = ''
+      $packageProperties = [ordered]@{}
+      if ($fields[1] -match '\[(.*)\]') { 
+          $rawProperties = $Matches[1] 
+          foreach ($propertyExpression in $rawProperties.Split(';')) {
+              $propertyParts = $propertyExpression.Split('=')
+              $packageProperties[$propertyParts[0]] = $propertyParts[1]
+          }
+      }
+
+      # Matches the "Package.Name" from the {package_ID} field. Possible
+      # formats:
+      # [key=value;key=value]Package.Name
+      # Package.Name
+      $packageName = ''
+      if ($fields[1] -match '(\[.*\])?(.*)') { 
+          $packageName = $Matches[2] 
+      } else { 
+          Write-Error "Could not find package id in row: $row" 
+      }
+
+      # Remaining entries in the row are versions, add them to the package 
+      # properties
+      $outputVersions = @()
+      if ($fields[2]) {
+        $outputVersions = $fields[2..($fields.Count - 1)]
+      }
+
+      # Example row: 
+      # packagemoniker,[key1=value1;key2=value2]Package.Name,1.0.0,1.2.3-beta.1
+      $output += [PSCustomObject]@{
+          Id = $fields[0];                  # packagemoniker
+          Name = $packageName;              # Package.Name
+          Properties = $packageProperties;  # @{key1='value1'; key2='value2'}
+          Versions = $outputVersions        # @('1.0.0', '1.2.3-beta.1')
+      }
+  }
+
+  return $output
+}
+
+function Get-DocsCiLine ($item) { 
+  $line = ''
+  if ($item.Properties.Count) {
+    $propertyPairs = @()
+    foreach ($key in $item.Properties.Keys) { 
+      $propertyPairs += "$key=$($item.Properties[$key])"
+    }
+    $packageProperties = $propertyPairs -join ';'
+
+    $line = "$($item.Id),[$packageProperties]$($item.Name)"
+  } else { 
+    $line = "$($item.Id),$($item.Name)"
+  }
+
+  if ($item.Versions) {
+    $joinedVersions = $item.Versions -join ','
+    $line += ",$joinedVersions"
+  }
+
+  return $line
+}
+
+function Update-dotnet-DocsMsPackages($DocsRepoLocation, $DocsMetadata) {
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'bundlepackages/azure-dotnet-preview.csv') `
+    'preview' `
+    $DocsMetadata 
+
+  UpdateDocsMsPackages `
+    (Join-Path $DocsRepoLocation 'bundlepackages/azure-dotnet.csv') `
+    'latest' `
+    $DocsMetadata
+}
+
+function UpdateDocsMsPackages($DocConfigFile, $Mode, $DocsMetadata) {
+  Write-Host "Updating configuration: $DocConfigFile with mode: $Mode"
+  $packageConfig = Get-DocsCiConfig $DocConfigFile
+
+  $outputPackages = @()
+  foreach ($package in $packageConfig) {
+    # Do not filter by GA/Preview status because we want differentiate between
+    # tracked and non-tracked packages
+    $matchingPublishedPackageArray = $DocsMetadata.Where({ $_.Package -eq $package.Name })
+
+    # If this package does not match any published packages keep it in the list.
+    # This handles packages which are not tracked in metadata but still need to
+    # be built in Docs CI.
+    if ($matchingPublishedPackageArray.Count -eq 0) {
+      Write-Host "Keep non-tracked preview package: $($package.Name)"
+      $outputPackages += $package
+      continue
+    }
+
+    if ($matchingPublishedPackageArray.Count -gt 1) { 
+      LogWarning "Found more than one matching published package in metadata for $($package.Name); only updating first entry"
+    }
+    $matchingPublishedPackage = $matchingPublishedPackageArray[0]
+
+    if ($Mode -eq 'preview' -and !$matchingPublishedPackage.VersionPreview.Trim()) { 
+      # If we are in preview mode and the package does not have a superseding
+      # preview version, remove the package from the list. 
+      Write-Host "Remove superseded preview package: $($package.Name)"
+      continue
+    }
+
+    Write-Host "Keep tracked package: $($package.Name)"
+    $package.Versions = @($matchingPublishedPackage.VersionGA.Trim())
+    if ($Mode -eq 'preview') {
+      $package.Versions = @($matchingPublishedPackage.VersionPreview.Trim())
+    }
+    $outputPackages += $package
+  }
+
+  $outputPackagesHash = @{}
+  foreach ($package in $outputPackages) {
+    $outputPackagesHash[$package.Name] = $true
+  }
+
+  $remainingPackages = @() 
+  if ($Mode -eq 'preview') { 
+    $remainingPackages = $DocsMetadata.Where({ 
+      $_.VersionPreview.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  } else { 
+    $remainingPackages = $DocsMetadata.Where({ 
+      $_.VersionGA.Trim() -and !$outputPackagesHash.ContainsKey($_.Package)
+    })
+  }
+
+  # Add packages that exist in the metadata but are not onboarded in docs config
+  # TODO: tfm='netstandard2.0' is a temporary workaround for 
+  # https://github.com/Azure/azure-sdk-for-net/issues/22494
+  $newPackageProperties = [ordered]@{ tfm = 'netstandard2.0' }
+  if ($Mode -eq 'preview') {
+    $newPackageProperties['isPrerelease'] = 'true'
+  }
+
+  foreach ($package in $remainingPackages) {
+    Write-Host "Add new package from metadata: $($package.Package)"
+    $versions = @($package.VersionGA.Trim())
+    if ($Mode -eq 'preview') {
+      $versions = @($package.VersionPreview.Trim())
+    }
+    $outputPackages += [PSCustomObject]@{
+      Id = $package.Package.Replace('.', '').ToLower();
+      Name = $package.Package;
+      Properties = $newPackageProperties;
+      Versions = $versions
+    }
+  }
+
+  $outputLines = @() 
+  foreach ($package in $outputPackages) { 
+    $outputLines += Get-DocsCiLine $package
+  }
+  Set-Content -Path $DocConfigFile -Value $outputLines
+}