diff --git a/eng/pipelines/templates/stages/archetype-python-release.yml b/eng/pipelines/templates/stages/archetype-python-release.yml index 8bdcfd24b8fa9..3ef4cf4faf500 100644 --- a/eng/pipelines/templates/stages/archetype-python-release.yml +++ b/eng/pipelines/templates/stages/archetype-python-release.yml @@ -18,82 +18,95 @@ stages: dependsOn: ${{parameters.DependsOn}} condition: and(succeeded(), ne(variables['SetDevVersion'], 'true'), ne(variables['Skip.Release'], 'true'), ne(variables['Build.Repository.Name'], 'Azure/azure-sdk-for-python-pr')) jobs: - - deployment: TagRepository + - job: TagRepository displayName: "Create release tag" condition: ne(variables['Skip.TagRepository'], 'true') - environment: github pool: image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general os: linux - strategy: - runOnce: - deploy: - steps: - - checkout: self - - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.12' - - - script: | - set -e - python -m pip install -r eng/release_requirements.txt - displayName: Install Release Dependencies - - - template: /eng/common/pipelines/templates/steps/retain-run.yml - - - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml - parameters: - PackageName: "azure-template" - ServiceDirectory: "template" - TestPipeline: ${{ parameters.TestPipeline }} - - - template: /eng/common/pipelines/templates/steps/verify-changelog.yml - parameters: - PackageName: ${{artifact.name}} - ServiceName: ${{parameters.ServiceDirectory}} - ForRelease: true - - - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml - parameters: - PackageName: ${{artifact.name}} - ServiceDirectory: ${{parameters.ServiceDirectory}} - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - - - script: | - python -m pip install "./tools/azure-sdk-tools" - displayName: Install tool dependencies - - - task: PythonScript@0 - displayName: Verify CI enabled - condition: succeeded() - inputs: - scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - - pwsh: | - Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml - parameters: - ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} - PackageRepository: PyPI - ReleaseSha: $(Build.SourceVersion) - RepoId: Azure/azure-sdk-for-python - WorkingDirectory: $(System.DefaultWorkingDirectory) + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.12' + + - template: /eng/common/pipelines/templates/steps/retain-run.yml + + - template: /eng/common/pipelines/templates/steps/set-test-pipeline-version.yml + parameters: + PackageName: "azure-template" + ServiceDirectory: "template" + TestPipeline: ${{ parameters.TestPipeline }} + + - template: /eng/common/pipelines/templates/steps/verify-changelog.yml + parameters: + PackageName: ${{artifact.name}} + ServiceName: ${{parameters.ServiceDirectory}} + ForRelease: true + + - template: /eng/common/pipelines/templates/steps/verify-restapi-spec-location.yml + parameters: + PackageName: ${{artifact.name}} + ServiceDirectory: ${{parameters.ServiceDirectory}} + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + + - script: | + python -m pip install "./tools/azure-sdk-tools" + displayName: Install tool dependencies + + - task: PythonScript@0 + displayName: Verify Dependency Presence + condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) + inputs: + scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' + arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - task: PythonScript@0 + displayName: Verify CI enabled + condition: succeeded() + inputs: + scriptPath: 'scripts/devops_tasks/verify_ci_enabled.py' + arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' + + - pwsh: | + Get-ChildItem -Recurse $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/create-tags-and-git-release.yml + parameters: + ArtifactLocation: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}} + PackageRepository: PyPI + ReleaseSha: $(Build.SourceVersion) + RepoId: Azure/azure-sdk-for-python + WorkingDirectory: $(System.DefaultWorkingDirectory) - ${{if ne(artifact.skipPublishPackage, 'true')}}: - deployment: PublishPackage displayName: "Publish to PyPI" condition: and(succeeded(), ne(variables['Skip.PublishPackage'], 'true')) - environment: pypi + environment: package-publish dependsOn: TagRepository + templateContext: + type: releaseJob + isProduction: true + inputs: + - input: pipelineArtifact + artifactName: release_artifact + targetPath: $(Pipeline.Workspace)/release_artifact + - input: pipelineArtifact + artifactName: packages_extended + targetPath: $(Pipeline.Workspace)/packages_extended + pool: image: azsdk-pool-mms-ubuntu-2004-1espt name: azsdk-pool-mms-ubuntu-2004-general @@ -103,28 +116,15 @@ stages: runOnce: deploy: steps: - - checkout: self - - - download: current - artifact: ${{parameters.ArtifactName}} - timeoutInMinutes: 5 - task: UsePythonVersion@0 inputs: versionSpec: '3.9' - script: | - set -e - python -m pip install -r eng/release_requirements.txt + python -m pip install -r $(Pipeline.Workspace)/release_artifact/release_requirements.txt displayName: Install Release Dependencies - - task: PythonScript@0 - displayName: Verify Dependency Presence - condition: and(succeeded(), ne(variables['Skip.VerifyDependencies'], 'true')) - inputs: - scriptPath: 'scripts/devops_tasks/verify_dependencies_present.py' - arguments: '--package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }}' - - task: TwineAuthenticate@0 displayName: 'Authenticate to feed: ${{parameters.DevFeedName}}' inputs: @@ -166,20 +166,39 @@ stages: echo "Uploaded sdist to devops feed" displayName: 'Publish package to feed: ${{parameters.DevFeedName}}' - - template: /eng/common/pipelines/templates/steps/create-apireview.yml - parameters: - ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}} - Artifacts: ${{parameters.Artifacts}} - ConfigFileDir: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo - MarkPackageAsShipped: true - ArtifactName: ${{parameters.ArtifactName}} - PackageName: ${{artifact.name}} + - job: CreateApiView + displayName: "Create APIView" + dependsOn: PublishPackage + + pool: + image: azsdk-pool-mms-ubuntu-2004-1espt + name: azsdk-pool-mms-ubuntu-2004-general + os: linux + + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.9' + + - template: /eng/common/pipelines/templates/steps/create-apireview.yml + parameters: + ArtifactPath: $(Pipeline.Workspace)/${{parameters.ArtifactName}} + Artifacts: ${{parameters.Artifacts}} + ConfigFileDir: $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo + MarkPackageAsShipped: true + ArtifactName: ${{parameters.ArtifactName}} + PackageName: ${{artifact.name}} - ${{if ne(artifact.skipPublishDocGithubIo, 'true')}}: - - deployment: PublishGitHubIODocs + - job: PublishGitHubIODocs displayName: Publish Docs to GitHubIO Blob Storage condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio dependsOn: PublishPackage pool: @@ -187,76 +206,77 @@ stages: image: azsdk-pool-mms-win-2022-1espt os: windows - strategy: - runOnce: - deploy: - steps: - - checkout: self - - - pwsh: | - if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}") { - Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - else { - New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" - } - workingDirectory: $(Pipeline.Workspace) - displayName: Output Visible Artifacts - - - template: /eng/common/pipelines/templates/steps/publish-blobs.yml - parameters: - FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}' - TargetLanguage: 'python' - ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' + steps: + - checkout: self + + - download: current + artifact: ${{parameters.ArtifactName}} + timeoutInMinutes: 5 + + - download: current + artifact: ${{parameters.DocArtifact}} + timeoutInMinutes: 5 + + - pwsh: | + if (Test-Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}") { + Get-ChildItem -Recurse "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" + } + else { + New-Item -ItemType Directory -Force -Path "$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}" + } + workingDirectory: $(Pipeline.Workspace) + displayName: Output Visible Artifacts + + - template: /eng/common/pipelines/templates/steps/publish-blobs.yml + parameters: + FolderForUpload: '$(Pipeline.Workspace)/${{parameters.DocArtifact}}/${{artifact.name}}' + TargetLanguage: 'python' + ArtifactLocation: '$(Pipeline.Workspace)/${{parameters.ArtifactName}}/${{artifact.name}}' - ${{if ne(artifact.skipPublishDocMs, 'true')}}: - - deployment: PublishDocs - displayName: Docs.MS Release - condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) - environment: githubio - dependsOn: PublishPackage - - pool: - image: azsdk-pool-mms-ubuntu-2004-1espt - name: azsdk-pool-mms-ubuntu-2004-general - os: linux - - strategy: - runOnce: - deploy: - steps: - - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml - parameters: - Paths: - - sdk/**/*.md - - .github/CODEOWNERS - - download: current - - # py2docfx requires Python >= 3.11 - - task: UsePythonVersion@0 - displayName: 'Use Python 3.11' - inputs: - versionSpec: '3.11' - - - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml - - - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml - parameters: - PackageInfoLocations: - - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json - WorkingDirectory: $(System.DefaultWorkingDirectory) - TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} - TargetDocRepoName: ${{parameters.TargetDocRepoName}} - Language: 'python' - SparseCheckoutPaths: - - docs-ref-services/ - - metadata/ - PackageSourceOverride: ${{parameters.PackageSourceOverride}} - - - deployment: UpdatePackageVersion + - job: PublishDocs + displayName: Docs.MS Release + condition: and(succeeded(), ne(variables['Skip.PublishDocs'], 'true')) + dependsOn: PublishPackage + + pool: + image: azsdk-pool-mms-ubuntu-2004-1espt + name: azsdk-pool-mms-ubuntu-2004-general + os: linux + + steps: + - template: /eng/common/pipelines/templates/steps/sparse-checkout.yml + parameters: + Paths: + - sdk/**/*.md + - .github/CODEOWNERS + + - download: current + + # py2docfx requires Python >= 3.11 + - task: UsePythonVersion@0 + displayName: 'Use Python 3.11' + inputs: + versionSpec: '3.11' + + - template: /eng/pipelines/templates/steps/install-rex-validation-tool.yml + + - template: /eng/common/pipelines/templates/steps/update-docsms-metadata.yml + parameters: + PackageInfoLocations: + - $(Pipeline.Workspace)/${{parameters.ArtifactName}}/PackageInfo/${{artifact.name}}.json + WorkingDirectory: $(System.DefaultWorkingDirectory) + TargetDocRepoOwner: ${{parameters.TargetDocRepoOwner}} + TargetDocRepoName: ${{parameters.TargetDocRepoName}} + Language: 'python' + SparseCheckoutPaths: + - docs-ref-services/ + - metadata/ + PackageSourceOverride: ${{parameters.PackageSourceOverride}} + + - job: UpdatePackageVersion displayName: "Update Package Version" condition: and(succeeded(), ne(variables['Skip.UpdatePackageVersion'], 'true')) - environment: github dependsOn: PublishPackage pool: @@ -264,31 +284,28 @@ stages: name: azsdk-pool-mms-ubuntu-2004-general os: linux - strategy: - runOnce: - deploy: - steps: - - checkout: self - - task: UsePythonVersion@0 - - script: | - python -m pip install "./tools/azure-sdk-tools[build]" - displayName: Install versioning tool dependencies - - - pwsh: | - sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }} - if (Test-Path component-detection-pip-report.json) { - Write-Host "Deleting component-detection-pip-report.json" - rm component-detection-pip-report.json - } - displayName: Increment package version - - - template: /eng/common/pipelines/templates/steps/create-pull-request.yml - parameters: - RepoName: azure-sdk-for-python - PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) - CommitMsg: "Increment package version after release of ${{ artifact.name }}" - PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" - CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' + steps: + - checkout: self + - task: UsePythonVersion@0 + - script: | + python -m pip install "./tools/azure-sdk-tools[build]" + displayName: Install versioning tool dependencies + + - pwsh: | + sdk_increment_version --package-name ${{ artifact.name }} --service ${{ parameters.ServiceDirectory }} + if (Test-Path component-detection-pip-report.json) { + Write-Host "Deleting component-detection-pip-report.json" + rm component-detection-pip-report.json + } + displayName: Increment package version + + - template: /eng/common/pipelines/templates/steps/create-pull-request.yml + parameters: + RepoName: azure-sdk-for-python + PRBranchName: increment-package-version-${{ parameters.ServiceDirectory }}-$(Build.BuildId) + CommitMsg: "Increment package version after release of ${{ artifact.name }}" + PRTitle: "Increment version for ${{ parameters.ServiceDirectory }} releases" + CloseAfterOpenForTesting: '${{ parameters.TestPipeline }}' - ${{if and(eq(variables['Build.Reason'], 'Manual'), eq(variables['System.TeamProject'], 'internal'))}}: - template: /eng/pipelines/templates/jobs/smoke.tests.yml diff --git a/eng/pipelines/templates/steps/build-package-artifacts.yml b/eng/pipelines/templates/steps/build-package-artifacts.yml index 1a5f0e7558cb2..924a69978f00d 100644 --- a/eng/pipelines/templates/steps/build-package-artifacts.yml +++ b/eng/pipelines/templates/steps/build-package-artifacts.yml @@ -88,6 +88,10 @@ steps: $(VENV_ACTIVATION_SCRIPT) which python python -m pip install --force -r eng/ci_tools.txt + if ($env:AGENT_OS -eq "Linux") { + Write-Host "Installing release reqs" + python -m pip install -r eng/release_requirements.txt + } python -m pip freeze --all displayName: 'Prep Environment' condition: and(succeeded(), or(eq(variables['ENABLE_EXTENSION_BUILD'], 'true'), eq('${{ parameters.ArtifactSuffix }}', 'linux'))) @@ -109,8 +113,8 @@ steps: env: CIBW_BUILD_VERBOSITY: 3 - - script: | - python -m pip install -r eng/release_requirements.txt + - pwsh: | + $(VENV_ACTIVATION_SCRIPT) twine check $(Build.ArtifactStagingDirectory)/**/*.whl twine check $(Build.ArtifactStagingDirectory)/**/*.tar.gz displayName: 'Verify Readme' @@ -127,6 +131,19 @@ steps: displayName: Clean up repo condition: and(succeeded(), ne(variables['ENABLE_EXTENSION_BUILD'], 'true'), ne('${{ parameters.ArtifactSuffix }}', 'linux')) + # we need to only publish this if running on linux. we only need one for all stages + - pwsh: | + New-Item -ItemType Directory -Path "$(Build.ArtifactStagingDirectory)/release_artifact" + Copy-Item -Path "$(Build.SourcesDirectory)/eng/release_requirements.txt" -Destination "$(Build.ArtifactStagingDirectory)/release_artifact/release_requirements.txt" + displayName: 'Assemble Static Files for Release Stage' + condition: and(succeeded(), eq(variables['Agent.OS'], 'Linux')) + + - ${{ if eq(parameters.ArtifactSuffix, 'linux') }}: + - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml + parameters: + ArtifactPath: '$(Build.ArtifactStagingDirectory)/release_artifact' + ArtifactName: 'release_artifact' + - ${{ if eq(parameters.ArtifactSuffix, '') }}: - template: /eng/common/pipelines/templates/steps/publish-1es-artifact.yml parameters: diff --git a/eng/release_requirements.txt b/eng/release_requirements.txt index 0ad3437ca45b8..e957043ae5644 100644 --- a/eng/release_requirements.txt +++ b/eng/release_requirements.txt @@ -1,2 +1 @@ twine==5.1.1 -tools/azure-sdk-tools \ No newline at end of file