From 57c52324197d71eeba373ac05f447667d901203b Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Fri, 22 Nov 2024 12:10:23 -0800 Subject: [PATCH 1/2] Update the docIndex default for manual runs --- eng/pipelines/docindex.yml | 255 ++++++++++++++++++++----------------- 1 file changed, 135 insertions(+), 120 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index 94beaf7ff9e9..b354be079a46 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -56,127 +56,142 @@ jobs: - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Move deprecated packages to legacy moniker - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Generate Service Level Readme for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - displayName: Generate ToC for main branch - condition: and(succeeded(), or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.MainUpdate'], 'true'))) - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/set-default-branch.yml parameters: WorkingDirectory: $(DocRepoLocation) - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DefaultBranch) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - - # Prepare daily docs CI - - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml - parameters: - DailyBranchVariableName: DailyDocsBranchName - - pwsh: | - $ErrorActionPreference = "Continue" - git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null - $LASTEXITCODE = 0 # This ignores any error from git checkout - git status - displayName: Checkout daily branch if it exists - workingDirectory: $(DocRepoLocation) - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsPackages.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Update Docs Onboarding for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 - arguments: -DocRepoLocation $(DocRepoLocation) - displayName: Generate Service Level Readme for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Update-DocsMsToc.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml - displayName: Generate ToC for Daily docs - - - task: Powershell@2 - inputs: - pwsh: true - filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 - arguments: >- - -DocRepoLocation $(DocRepoLocation) - displayName: Verify Required Docs Json Members - - - template: /eng/common/pipelines/templates/steps/git-push-changes.yml - parameters: - BaseRepoBranch: $(DailyDocsBranchName) - BaseRepoOwner: $(DocRepoOwner) - CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" - TargetRepoName: $(DocRepoName) - TargetRepoOwner: $(DocRepoOwner) - WorkingDirectory: $(DocRepoLocation) - ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts - PushArgs: -f - - - task: AzureCLI@2 - displayName: Queue Docs CI build - inputs: - azureSubscription: msdocs-apidrop-connection - scriptType: pscore - scriptLocation: inlineScript - inlineScript: | - $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv - $buildParamJson = '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' - eng/common/scripts/Queue-Pipeline.ps1 ` - -Organization "apidrop" ` - -Project "Content%20CI" ` - -DefinitionId 3452 ` - -BuildParametersJson $buildParamJson ` - -BearerToken $accessToken + # Updating main is the default reason to kick off manual builds of the docIndex run. + # Manual builds of docIndex are typically done to update Main when certain updates are + # made. These updates include new libraries and CSV updates in azure-sdk which, depending + # on the column(s) updated can cause anything from package deprecation to service level + # readme or ToC updates. + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true')) }}: + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Move deprecated packages to legacy moniker + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for main branch + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Generate Service Level Readme for main branch + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for main branch + condition: succeeded() + + # Push changes to docs repo + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DefaultBranch) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update docs CI configuration Build: $(System.CollectionUri)$(System.TeamProject)/_build/results?buildId=$(Build.BuildId)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + + # The scenario for running a Manual build is normally only for the main updates. The daily build + # should really only get kicked off for scheduled runs. + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true')) }}: + # Prepare daily docs CI + - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml + parameters: + DailyBranchVariableName: DailyDocsBranchName + - pwsh: | + $ErrorActionPreference = "Continue" + git checkout "origin/$(DailyDocsBranchName)" 2>&1 | Out-Null + $LASTEXITCODE = 0 # This ignores any error from git checkout + git status + displayName: Checkout daily branch if it exists + workingDirectory: $(DocRepoLocation) + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Verify-RequiredDocsJsonMembers.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + displayName: Verify Required Docs Json Members + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsPackages.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Update Docs Onboarding for Daily docs + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 + arguments: -DocRepoLocation $(DocRepoLocation) + displayName: Generate Service Level Readme for Daily docs + condition: succeeded() + + - task: Powershell@2 + inputs: + pwsh: true + filePath: eng/common/scripts/Update-DocsMsToc.ps1 + arguments: >- + -DocRepoLocation $(DocRepoLocation) + -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml + displayName: Generate ToC for Daily docs + condition: succeeded() + + - template: /eng/common/pipelines/templates/steps/git-push-changes.yml + parameters: + BaseRepoBranch: $(DailyDocsBranchName) + BaseRepoOwner: $(DocRepoOwner) + CommitMsg: "Update targeting packages based on release metadata. (Daily docs)" + TargetRepoName: $(DocRepoName) + TargetRepoOwner: $(DocRepoOwner) + WorkingDirectory: $(DocRepoLocation) + ScriptDirectory: $(Build.SourcesDirectory)/eng/common/scripts + PushArgs: -f + + - task: AzureCLI@2 + displayName: Queue Docs CI build + inputs: + azureSubscription: msdocs-apidrop-connection + scriptType: pscore + scriptLocation: inlineScript + inlineScript: | + $accessToken = az account get-access-token --resource "499b84ac-1321-427f-aa17-267ca6975798" --query "accessToken" --output tsv + $buildParamJson = '{"params":"{ \"target_repo\": { \"url\": \"https://github.com/MicrosoftDocs/azure-docs-sdk-node\", \"branch\": \"$(DailyDocsBranchName)\", \"folder\": \"./\" }, \"source_repos\": [] }"}' + eng/common/scripts/Queue-Pipeline.ps1 ` + -Organization "apidrop" ` + -Project "Content%20CI" ` + -DefinitionId 3452 ` + -BuildParametersJson $buildParamJson ` + -BearerToken $accessToken + condition: succeeded() From 38464cb5e7b6169b325693f8acc53dd6cbd7b5c6 Mon Sep 17 00:00:00 2001 From: James Suplizio Date: Mon, 25 Nov 2024 12:24:18 -0800 Subject: [PATCH 2/2] Change the variables to parameters --- eng/pipelines/docindex.yml | 27 ++++++++++++++++----------- 1 file changed, 16 insertions(+), 11 deletions(-) diff --git a/eng/pipelines/docindex.yml b/eng/pipelines/docindex.yml index b354be079a46..aba74f4c0ffc 100644 --- a/eng/pipelines/docindex.yml +++ b/eng/pipelines/docindex.yml @@ -1,4 +1,18 @@ trigger: none + +parameters: +- name: UpdateMain + displayName: | + Update main branch. + type: boolean + default: true + +- name: ForceDailyUpdate + displayName: | + Force the daily branch update (includes starting daily branch run). + type: boolean + default: false + variables: - template: /eng/pipelines/templates/variables/globals.yml jobs: @@ -65,7 +79,7 @@ jobs: # made. These updates include new libraries and CSV updates in azure-sdk which, depending # on the column(s) updated can cause anything from package deprecation to service level # readme or ToC updates. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), ne(variables['Skip.MainUpdate'], 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.UpdateMain) }}: - task: Powershell@2 inputs: pwsh: true @@ -80,7 +94,6 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackageMonikers.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Move deprecated packages to legacy moniker - condition: succeeded() - task: Powershell@2 inputs: @@ -88,7 +101,6 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackages.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for main branch - condition: succeeded() - task: Powershell@2 inputs: @@ -96,7 +108,6 @@ jobs: filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Generate Service Level Readme for main branch - condition: succeeded() - task: Powershell@2 inputs: @@ -106,7 +117,6 @@ jobs: -DocRepoLocation $(DocRepoLocation) -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml displayName: Generate ToC for main branch - condition: succeeded() # Push changes to docs repo - template: /eng/common/pipelines/templates/steps/git-push-changes.yml @@ -120,7 +130,7 @@ jobs: # The scenario for running a Manual build is normally only for the main updates. The daily build # should really only get kicked off for scheduled runs. - - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), eq(variables['Force.DailyUpdate'], 'true')) }}: + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), parameters.ForceDailyUpdate) }}: # Prepare daily docs CI - template: /eng/common/pipelines/templates/steps/set-daily-docs-branch-name.yml parameters: @@ -140,7 +150,6 @@ jobs: arguments: >- -DocRepoLocation $(DocRepoLocation) displayName: Verify Required Docs Json Members - condition: succeeded() - task: Powershell@2 inputs: @@ -148,7 +157,6 @@ jobs: filePath: eng/common/scripts/Update-DocsMsPackages.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Update Docs Onboarding for Daily docs - condition: succeeded() - task: Powershell@2 inputs: @@ -156,7 +164,6 @@ jobs: filePath: eng/common/scripts/Service-Level-Readme-Automation.ps1 arguments: -DocRepoLocation $(DocRepoLocation) displayName: Generate Service Level Readme for Daily docs - condition: succeeded() - task: Powershell@2 inputs: @@ -166,7 +173,6 @@ jobs: -DocRepoLocation $(DocRepoLocation) -OutputLocation $(DocRepoLocation)/docs-ref-mapping/reference-unified.yml displayName: Generate ToC for Daily docs - condition: succeeded() - template: /eng/common/pipelines/templates/steps/git-push-changes.yml parameters: @@ -194,4 +200,3 @@ jobs: -DefinitionId 3452 ` -BuildParametersJson $buildParamJson ` -BearerToken $accessToken - condition: succeeded()