From 9d956d9c009d15eb63416137dfb2a84d03f26638 Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Mon, 5 Feb 2024 14:46:25 -0800 Subject: [PATCH 1/9] Convert Bases Image pipelines to 1ES Template --- .../templates/_buildimageBasesJobTemplate.yml | 146 ++++++++++-------- 1 file changed, 83 insertions(+), 63 deletions(-) diff --git a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml index 701537907a..f27d7c4986 100644 --- a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml +++ b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml @@ -1,71 +1,91 @@ -parameters: - displayName: '' - imageDir: '' - imageDebianFlavor: '' - scriptPath: '' - artifactsFileName: '' - jobName: '' +# trigger: none -jobs: -- job: ${{ parameters.jobName }} - displayName: ${{ parameters.displayName }} - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - timeoutInMinutes: 250 - steps: - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' +# The `resources` specify the location and version of the 1ES PT. +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release - - task: ShellScript@2 - displayName: Build images - inputs: - scriptPath: ${{ parameters.scriptPath }} - args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} - env: - ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) - DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + # Update the pool with your team's 1ES hosted pool. + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant # Name of the image in your pool. If not specified, first image of the pool is used + os: linux # OS of the image. Allowed values: windows, linux, macOS - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: Generate Software Bill of Materials (SBOM) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)' - AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' + stages: + - stage: Stage + jobs: + - job: HostJob + timeoutInMinutes: 250 + # If the pipeline publishes artifacts, use `templateContext` to define the artifacts. + # This will enable 1ES PT to run SDL analysis tools on the artifacts and then upload them. + templateContext: + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory) + artifactName: buildImageBasesJobArtifact + # Define the steps that the pipeline will run. + # In most cases, copy and paste the steps from the original pipeline. + steps: + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - task: CopyFiles@2 - displayName: Copy artifacts to staging directory - inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts' - contents: '**/*.*' - targetFolder: $(Build.ArtifactStagingDirectory) - overWrite: true - condition: true + - task: ShellScript@2 + displayName: Build images + inputs: + scriptPath: ${{ parameters.scriptPath }} + args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) - - task: Docker@1 - displayName: Push built base images to dev ACR - inputs: - command: push - azureSubscriptionEndpoint: $(ascName) - azureContainerRegistry: $(acrName) - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' - enforceDockerNamingConvention: false + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: Generate Software Bill of Materials (SBOM) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)' + AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' - - task: ShellScript@2 - displayName: 'Clean up Docker containers and images' - inputs: - scriptPath: ./vsts/scripts/cleanDocker.sh + - task: CopyFiles@2 + displayName: Copy artifacts to staging directory + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true - - task: PublishBuildArtifacts@1 - displayName: Publish build artifacts - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) + - task: Docker@1 + displayName: Push built base images to dev ACR + inputs: + command: push + azureSubscriptionEndpoint: $(ascName) + azureContainerRegistry: $(acrName) + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' + enforceDockerNamingConvention: false - - task: ShellScript@2 - displayName: 'Clean up Docker containers and images' - inputs: - scriptPath: ./vsts/scripts/cleanDocker.sh - condition: true + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + + - task: PublishBuildArtifacts@1 + displayName: Publish build artifacts + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file From e3641833b16727c8fd1bf86661591bc4bfa233cd Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 19:52:54 -0700 Subject: [PATCH 2/9] Update node --- vsts/pipelines/PlatformBinaries/node.yml | 182 +++++++++++------------ 1 file changed, 89 insertions(+), 93 deletions(-) diff --git a/vsts/pipelines/PlatformBinaries/node.yml b/vsts/pipelines/PlatformBinaries/node.yml index 906b086d40..172cb39b14 100644 --- a/vsts/pipelines/PlatformBinaries/node.yml +++ b/vsts/pipelines/PlatformBinaries/node.yml @@ -1,95 +1,3 @@ -variables: - - group: Oryx - -parameters: -- name: destinationStorageAccountName - displayName: Destination Storage Account Name - type: string - default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Node_Bookworm - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -100,4 +8,92 @@ trigger: - /* include: - platforms/nodejs - - vsts/PlatformBinaries/node.yml \ No newline at end of file + - vsts/PlatformBinaries/node.yml + +# The `resources` specify the location and version of the 1ES PT. +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release + +variables: + - group: Oryx + +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + # Update the pool with your team's 1ES hosted pool. + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant # Name of the image in your pool. If not specified, first image of the pool is used + os: linux # OS of the image. Allowed values: windows, linux, macOS + + # - name: destinationStorageAccountName + # displayName: Destination Storage Account Name + # type: string + # default: oryxsdksstaging + + stages: + - stage: Build + jobs: + - job: Node_Bookworm + timeoutInMinutes: 250 + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Bullseye + timeoutInMinutes: 250 + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Buster + timeoutInMinutes: 250 + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Stretch + timeoutInMinutes: 250 + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Ubuntu + timeoutInMinutes: 250 + pool: + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file From 44b12adadc6436c31e33e9a145234d87ab569cb4 Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 20:09:46 -0700 Subject: [PATCH 3/9] Update node --- vsts/pipelines/baseImages/dotnetcore.yml | 99 ++-- vsts/pipelines/baseImages/node.yml | 95 +-- vsts/pipelines/baseImages/php-fpm.yml | 80 +-- vsts/pipelines/baseImages/php.yml | 80 +-- vsts/pipelines/ci.yml | 556 +++++++++--------- vsts/pipelines/templates/_buildTemplate.yml | 83 +-- vsts/pipelines/templates/_builderTemplate.yml | 29 +- .../templates/_buildimageBasesJobTemplate.yml | 131 ++--- .../templates/_integrationJobTemplate.yml | 26 +- .../_releaseBaseImagesStepTemplate.yml | 19 +- .../templates/_releaseJobTemplate.yml | 30 +- .../templates/_releaseStepTemplate.yml | 58 +- vsts/pipelines/templates/_securityChecks.yml | 43 -- vsts/pipelines/templates/_setReleaseTag.yml | 1 - vsts/pipelines/templates/_signBinary.yml | 39 +- .../_dotnetcoreIntegrationJobTemplate.yml | 94 +-- .../_golangIntegrationJobTemplate.yml | 74 +-- .../_nodeIntegrationJobTemplate.yml | 90 +-- .../_phpIntegrationJobTemplate.yml | 82 +-- .../_pythonIntegrationJobTemplate.yml | 94 +-- 20 files changed, 800 insertions(+), 1003 deletions(-) diff --git a/vsts/pipelines/baseImages/dotnetcore.yml b/vsts/pipelines/baseImages/dotnetcore.yml index 8ed0e6415e..fa23b571d3 100644 --- a/vsts/pipelines/baseImages/dotnetcore.yml +++ b/vsts/pipelines/baseImages/dotnetcore.yml @@ -8,47 +8,60 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true - -jobs: -- template: ../templates/_buildimageBasesJobTemplate.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - displayName: Build DotNetCore runtime buster base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: buster - artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt - jobName: Build_DotNetCore_BaseImage_Buster - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build DotNetCore runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: bullseye - artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt - jobName: Build_DotNetCore_BaseImage_Bullseye - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build DotNetCore runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: bookworm - artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt - jobName: Build_DotNetCore_BaseImage_Bookworm - -- job: Release_DotNetCoreRuntimeBaseImage - dependsOn: - - Build_DotNetCore_BaseImage_Buster - - Build_DotNetCore_BaseImage_Bullseye - - Build_DotNetCore_BaseImage_Bookworm - displayName: Push images to MCR - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_releaseBaseImagesStepTemplate.yml - parameters: - baseImageName: 'dotnetcore' \ No newline at end of file + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: buster + artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt + jobName: Build_DotNetCore_BaseImage_Buster + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bullseye + artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt + jobName: Build_DotNetCore_BaseImage_Bullseye + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bookworm + artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt + jobName: Build_DotNetCore_BaseImage_Bookworm + - job: Release_DotNetCoreRuntimeBaseImage + dependsOn: + - Build_DotNetCore_BaseImage_Buster + - Build_DotNetCore_BaseImage_Bullseye + - Build_DotNetCore_BaseImage_Bookworm + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'dotnetcore' \ No newline at end of file diff --git a/vsts/pipelines/baseImages/node.yml b/vsts/pipelines/baseImages/node.yml index 4a75cb0211..14a7dcd722 100644 --- a/vsts/pipelines/baseImages/node.yml +++ b/vsts/pipelines/baseImages/node.yml @@ -8,45 +8,58 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true - -jobs: -- template: ../templates/_buildimageBasesJobTemplate.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - displayName: Build Node runtime buster based images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: buster - artifactsFileName: node-runtimeimage-bases-buster.txt - jobName: Build_Buster_BaseImages - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build Node runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: bullseye - artifactsFileName: node-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build Node runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: bookworm - artifactsFileName: node-runtimeimage-bases-bookworm.txt - jobName: Build_Bookworm_BaseImages - -- job: Release_NodeRuntimeBaseImage - dependsOn: - - Build_Buster_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_releaseBaseImagesStepTemplate.yml - parameters: - baseImageName: 'node' + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime buster based images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: buster + artifactsFileName: node-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bullseye + artifactsFileName: node-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bookworm + artifactsFileName: node-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_NodeRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'node' \ No newline at end of file diff --git a/vsts/pipelines/baseImages/php-fpm.yml b/vsts/pipelines/baseImages/php-fpm.yml index 631b1bc090..b57bd7f722 100644 --- a/vsts/pipelines/baseImages/php-fpm.yml +++ b/vsts/pipelines/baseImages/php-fpm.yml @@ -8,37 +8,51 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true - -jobs: -- template: ../templates/_buildimageBasesJobTemplate.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - displayName: Build php buster runtime base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php-fpm - imageDebianFlavor: buster - artifactsFileName: php-fpm-runtimeimage-bases-buster.txt - jobName: Build_PHP_FPM_Buster_Base - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build php bullseye runtime base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php-fpm - imageDebianFlavor: bullseye - artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt - jobName: Build_PHP_Fpm_Bullseye_Base - -- job: Release_PhpRuntimeBaseImage - dependsOn: - - Build_Php_Fpm_Buster_Base - - Build_PHP_Fpm_Bullseye_Base - displayName: Push images to MCR - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_releaseBaseImagesStepTemplate.yml - parameters: - baseImageName: 'php-fpm' + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php buster runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: buster + artifactsFileName: php-fpm-runtimeimage-bases-buster.txt + jobName: Build_PHP_FPM_Buster_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bullseye runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bullseye + artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt + jobName: Build_PHP_Fpm_Bullseye_Base + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Php_Fpm_Buster_Base + - Build_PHP_Fpm_Bullseye_Base + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php-fpm' \ No newline at end of file diff --git a/vsts/pipelines/baseImages/php.yml b/vsts/pipelines/baseImages/php.yml index 6f290860cd..0964c12f4a 100644 --- a/vsts/pipelines/baseImages/php.yml +++ b/vsts/pipelines/baseImages/php.yml @@ -8,37 +8,51 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true - -jobs: -- template: ../templates/_buildimageBasesJobTemplate.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - displayName: Build php runtime buster base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php - imageDebianFlavor: buster - artifactsFileName: php-runtimeimage-bases-buster.txt - jobName: Build_Buster_BaseImages - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build php runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php - imageDebianFlavor: bullseye - artifactsFileName: php-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - -- job: Release_PhpRuntimeBaseImage - dependsOn: - - Build_Buster_BaseImages - - Build_Bullseye_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_releaseBaseImagesStepTemplate.yml - parameters: - baseImageName: 'php' + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: buster + artifactsFileName: php-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bullseye + artifactsFileName: php-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + - Build_Bullseye_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php' \ No newline at end of file diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml index 424a168125..8971135162 100644 --- a/vsts/pipelines/ci.yml +++ b/vsts/pipelines/ci.yml @@ -1,305 +1,305 @@ parameters: - - name: storageAccountUrl - displayName: SDK storage account URL for production images and testing - type: string - default: https://oryx-cdn.microsoft.io - values: - - https://oryx-cdn.microsoft.io - - name: buildImages - type: object - default: - - - key: Latest - value: latest - - - key: LtsVersions - value: ltsversions - - - key: Jamstack - value: jamstack - - - key: GithubActions - value: githubactions - - - key: VsoFocal - value: vso-focal - - - key: VsoBullseye - value: vso-bullseye - - - key: Full - value: full - - - key: Cli - value: cli-stretch - - - key: CliBuster - value: cli-buster - - - key: CliBullseye - value: cli-bullseye - - - key: CliBuilderBullseye - value: cli-builder-bullseye - - - key: Buildpack - value: buildpack - +- name: storageAccountUrl + displayName: SDK storage account URL for production images and testing + type: string + default: https://oryx-cdn.microsoft.io + values: + - https://oryx-cdn.microsoft.io +- name: buildImages + type: object + default: + - key: Latest + value: latest + - key: LtsVersions + value: ltsversions + - key: Jamstack + value: jamstack + - key: GithubActions + value: githubactions + - key: VsoFocal + value: vso-focal + - key: VsoBullseye + value: vso-bullseye + - key: Full + value: full + - key: Cli + value: cli-stretch + - key: CliBuster + value: cli-buster + - key: CliBullseye + value: cli-bullseye + - key: CliBuilderBullseye + value: cli-builder-bullseye + - key: Buildpack + value: buildpack resources: -- repo: self - + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release variables: group: Oryx Packaging.EnableSBOMSigning: true - -stages: - - stage: CreateReleaseTag - jobs: - - job: CreateReleaseTag - pool: +trigger: none +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: name: AzurePipelines-EO - demands: + image: AzurePipelinesWindows2022compliantGPT + os: windows + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish artifact file having the release tag name' + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + steps: + - checkout: self + clean: true + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: /vsts/pipelines/templates/_securityChecks.yml@self + - job: Job_SignBinaries + displayName: Sign Oryx Binaries + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + templateContext: + mb: + signing: + enabled: true + signType: '$(setSignTypeVariable.SignType)' + outputs: + - output: pipelineArtifact + displayName: 'Publish Pipeline Artifact' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + artifactName: 'signed_binaries' + targetPath: '$(Build.ArtifactStagingDirectory)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' + - powershell: | + $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" + if (Test-Path $file) { + $tagName = [IO.File]::ReadAllText($file) + Write-Host "Content is $tagName" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + } + displayName: 'Set release tag name as environment variable' + - template: /vsts/pipelines/templates/_signBinary.yml@self + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: ${{ buildImage.value }} + - job: Job_BuilderImages + displayName: Build Builder Images + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - skipComponentGovernanceDetection: true - steps: - - checkout: self - clean: true - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - - task: ShellScript@2 - inputs: - scriptPath: ./vsts/scripts/createReleaseTag.sh - displayName: 'Create release tag' - - script: | - set -ex - sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" - mkdir -p "$sourcesArtifactsDir" - echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" - displayName: 'Write release tag name to an artifact file' - - task: PublishPipelineArtifact@1 - displayName: 'Publish artifact file having the release tag name' - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt - artifactName: releaseTag - - - stage: Build - displayName: Build Stage - jobs: - - job: Job_Security - displayName: Security - condition: succeeded() - pool: - name: Azure Pipelines - vmImage: windows-2022 - steps: - - template: templates/_securityChecks.yml - - - job: Job_SignBinaries - displayName: Sign Oryx Binaries - pool: - name: VSEngSS-MicroBuild2022-1ES - demands: - - msbuild - - visualstudio - variables: - SignType: 'test' - skipComponentGovernanceDetection: true - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download artifact file having release tag name' - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag - artifactName: 'releaseTag' - - powershell: | - $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" - if (Test-Path $file) { - $tagName = [IO.File]::ReadAllText($file) - Write-Host "Content is $tagName" - echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" - } - displayName: 'Set release tag name as environment variable' - - template: templates/_signBinary.yml - - # Job loop for BuildImages - - ${{ each buildImage in parameters.buildImages }}: - - job: Job_BuildImage_${{ buildImage.key }} - displayName: 'Build & Test ${{ buildImage.key }} Build image' - #dependsOn: Job_SignBinaries + timeoutInMinutes: 480 + dependsOn: Job_BuildImage_CliBuilderBullseye + steps: + - template: /vsts/pipelines/templates/_builderTemplate.yml@self + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + dependsOn: Job_SignBinaries condition: succeeded() timeoutInMinutes: 480 pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] skipComponentGovernanceDetection: true - + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]true" - echo "##vso[task.setvariable variable=TestBuildImages;]true" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=PushBuildImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then - echo "##vso[task.setvariable variable=PushBuilderImages;]true" - fi displayName: 'Set variables' - - - template: templates/_setReleaseTag.yml - - - template: templates/_buildTemplate.yml + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self parameters: - imageType: ${{ buildImage.value }} - - - job: Job_BuilderImages - displayName: Build Builder Images - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - timeoutInMinutes: 480 - steps: - - template: templates/_builderTemplate.yml - dependsOn: Job_BuildImage_CliBuilderBullseye - - - job: Job_Buster_RuntimeImages - displayName: Build and Test Buster Runtime Images - dependsOn: Job_SignBinaries - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable - skipComponentGovernanceDetection: true - - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - - template: templates/_setReleaseTag.yml - - - template: templates/_buildTemplate.yml - parameters: imageType: buster - - - job: Job_Bullseye_RuntimeImages - displayName: Build and Test Bullseye Runtime Images - dependsOn: Job_SignBinaries - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable - skipComponentGovernanceDetection: true - - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - - template: templates/_setReleaseTag.yml - - - template: templates/_buildTemplate.yml - parameters: + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: imageType: bullseye - - - job: Job_Bookworm_RuntimeImages - displayName: Build and Test Bookworm Runtime Images - dependsOn: Job_SignBinaries - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable - skipComponentGovernanceDetection: true - - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - - template: templates/_setReleaseTag.yml - - - template: templates/_buildTemplate.yml - parameters: + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: imageType: bookworm - - - template: templates/_integrationJobTemplate.yml - parameters: + - template: /vsts/pipelines/templates/_integrationJobTemplate.yml@self + parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - - - job: Job_ProductionStorageAccountTest - displayName: Test Production Storage Account - pool: - name: AzurePipelines-EO - demands: + - job: Job_ProductionStorageAccountTest + displayName: Test Production Storage Account + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - skipComponentGovernanceDetection: true - steps: - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk 7.x' - inputs: - version: 7.0.306 - - - task: ShellScript@2 - displayName: 'Test production storage account' - env: - ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} - inputs: - scriptPath: ./build/testIntegration.sh - args: StorageAccountTests=Prod - - - stage: Release - displayName: Release Stage - dependsOn: Build - condition: > - and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), - startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), - startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) - - jobs: - - template: templates/_releaseJobTemplate.yml - -trigger: none \ No newline at end of file + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + - task: ShellScript@2 + displayName: 'Test production storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: > + and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + jobs: + - template: /vsts/pipelines/templates/_releaseJobTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/templates/_buildTemplate.yml b/vsts/pipelines/templates/_buildTemplate.yml index 043bda343b..f98cc5a60f 100644 --- a/vsts/pipelines/templates/_buildTemplate.yml +++ b/vsts/pipelines/templates/_buildTemplate.yml @@ -3,7 +3,6 @@ parameters: acrName: oryxdevmcr.azurecr.io imageName: oryxdevmcr.azurecr.io/public/oryx imageType: null - steps: - script: | if [ "$(BuildBuildImages)" != "true" ] && [ "$(BuildRuntimeImages)" != "true" ] && [ "$(TestIntegration)" != "true" ] @@ -13,7 +12,6 @@ steps: exit 1 fi displayName: 'Validate pipeline run' - - script: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc @@ -22,10 +20,8 @@ steps: docker images && docker system prune -fa && docker images && echo displayName: 'clean docker images' condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true')) - - checkout: self clean: true - - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact' inputs: @@ -33,55 +29,38 @@ steps: artifactName: 'signed_binaries' targetPath: '$(Build.SourcesDirectory)' condition: > - and(succeeded(), - or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), - or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), - startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), - startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) - + and(succeeded(), or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 - - script: | dotnet --version && dotnet msbuild -version && echo displayName: 'Print dotnet and msbuild version' - - task: ShellScript@2 displayName: 'Build Oryx.sln' inputs: scriptPath: ./build/buildSln.sh args: Oryx.sln - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrName }} - - script: | docker images && docker system prune -fa && df -h && echo displayName: 'Free up space for oryx layers' - - task: ShellScript@2 displayName: 'Test build script generator' inputs: scriptPath: ./build/testBuildScriptGenerator.sh condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) - - task: ShellScript@2 displayName: 'Test startup script generators' inputs: scriptPath: ./build/testStartupScriptGenerators.sh condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) - - task: ShellScript@2 displayName: 'Build build images' env: @@ -90,7 +69,6 @@ steps: scriptPath: ./build/buildBuildImages.sh args: -t ${{ parameters.imageType }} -s $(storageAccountUrl) condition: and(succeeded(), eq(variables['BuildBuildImages'], 'true')) - - task: ShellScript@2 displayName: 'Build runtime images' env: @@ -99,24 +77,20 @@ steps: scriptPath: ./build/buildRunTimeImages.sh args: -s $(storageAccountUrl) ${{ parameters.imageType }} condition: and(succeeded(), eq(variables['BuildRuntimeImages'], 'true')) - - - script: | docker images && docker system prune -f && df -h && echo displayName: 'clean docker unknown layers' - - script: | docker -v - echo "Installing Docker CE version 24.0.5................" + echo "Installing Docker CE version 24.0.5." curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt update sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - echo "After installing ................" + echo "After installing ." docker -v displayName: 'Install Docker 24.0.5' condition: true - - task: ShellScript@2 displayName: 'Test build images' env: @@ -126,7 +100,6 @@ steps: scriptPath: ./build/testBuildImages.sh args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) ${{ parameters.imageType }} condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) - - task: ShellScript@2 displayName: 'Test runtime images' env: @@ -136,13 +109,11 @@ steps: scriptPath: ./build/testRunTimeImages.sh args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) category=runtime-${{ parameters.imageType }} condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) - - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: artifactName: drop condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - - task: Shellpp@0 displayName: 'Pull and Retag recently built oryx build and runtime images' inputs: @@ -150,14 +121,12 @@ steps: scriptPath: ./vsts/scripts/pullAndTag.sh args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - - task: Shellpp@0 displayName: 'Build a build image for tests' inputs: type: FilePath scriptPath: ./build/buildTestBuildImages.sh condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - - task: ShellScript@2 displayName: 'Test integration' inputs: @@ -171,7 +140,6 @@ steps: ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) - - task: CopyFiles@2 displayName: 'Copy source projects output to artifacts folder' inputs: @@ -182,7 +150,6 @@ steps: overWrite: true flattenFolders: true condition: true - - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -191,77 +158,47 @@ steps: targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true condition: true - -- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: Generate Software Bill of Materials (SBOM) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)' - AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' - - task: PublishTestResults@2 inputs: testRunner: 'xUnit' testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' mergeTestResults: true condition: true - -- task: Docker@1 +- task: 1ES.PushContainerImage@1 displayName: 'Push build images to ACR' inputs: - azureSubscriptionEndpoint: ${{ parameters.ascName }} - azureContainerRegistry: ${{ parameters.acrName }} - command: 'Push an image' - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/build-images-acr.txt' - includeLatestTag: false - enforceDockerNamingConvention: false - condition: and(succeeded(), or(eq(variables['PushBuildImages'], 'true'), eq(variables['PushBuilderImages'], 'true')), eq(variables['BuildBuildImages'], 'true')) - -- task: Docker@1 + image: ${{ parameters.acrName }}/${{ parameters.ascName }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:${{ parameters.acrName }} +- task: 1ES.PushContainerImage@1 displayName: 'Push runtime images to ACR' inputs: - azureSubscriptionEndpoint: ${{ parameters.ascName }} - azureContainerRegistry: ${{ parameters.acrName }} - command: 'Push an image' - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/runtime-images-acr.${{ parameters.imageType }}.txt' - includeLatestTag: false - enforceDockerNamingConvention: false + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} condition: and(succeeded(), eq(variables['PushRuntimeImages'], 'true'), eq(variables['BuildRuntimeImages'], 'true')) - - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: scriptPath: ./vsts/scripts/cleanDocker.sh condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) - - task: ShellScript@2 displayName: 'Clean up Docker mounted directories' inputs: scriptPath: ./vsts/scripts/removeDockerArtifacts.sh condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) - - script: | docker images && docker system prune -fa && df -h && docker images && echo displayName: 'clean docker images and stopped containers' - - task: ShellScript@2 displayName: 'Generate release notes' inputs: scriptPath: ./vsts/scripts/generateReleaseNotes.sh condition: and(succeeded(), eq(variables['PushBuildImages'], 'true'), eq(variables['BuildBuildImages'], 'true')) - - task: ArchiveFiles@2 displayName: 'Archive docker files and scripts for Oryx build and runtime images' inputs: rootFolderOrFile: images archiveFile: '$(Build.ArtifactStagingDirectory)/images/dockerFiles.zip' condition: true - -- task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: drop' - condition: true - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' - condition: true + condition: true \ No newline at end of file diff --git a/vsts/pipelines/templates/_builderTemplate.yml b/vsts/pipelines/templates/_builderTemplate.yml index ced672617e..b5841ce2e9 100644 --- a/vsts/pipelines/templates/_builderTemplate.yml +++ b/vsts/pipelines/templates/_builderTemplate.yml @@ -1,32 +1,27 @@ parameters: ascName: oryx-automation-service-principal acrName: oryxdevmcr.azurecr.io - steps: - script: | curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.28.0/pack-v0.28.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack displayName: 'Install Pack CLI' condition: true - - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrName }} - - task: ShellScript@2 displayName: 'Build base builder images' inputs: scriptPath: ./builders/base/buildBaseBuilder.sh args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -v $(Build.DefinitionName).$(Build.BuildNumber) - - task: ShellScript@2 displayName: 'Build container apps builder images' inputs: scriptPath: ./builders/container-apps/buildCappsBuilder.sh args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -t "capps-$(Build.DefinitionName).$(Build.BuildNumber)" -b "${{ parameters.acrName }}/public/oryx/builder:$(Build.DefinitionName).$(Build.BuildNumber)" - - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -34,25 +29,13 @@ steps: contents: '**/*.*' targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true - -- task: Docker@1 +- task: 1ES.PushContainerImage@1 displayName: 'Push base build images to ACR' inputs: - azureSubscriptionEndpoint: ${{ parameters.ascName }} - azureContainerRegistry: ${{ parameters.acrName }} - command: 'Push an image' - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/builder-images-acr.txt' - includeLatestTag: false - enforceDockerNamingConvention: false - -- task: Docker@1 + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) +- task: 1ES.PushContainerImage@1 displayName: 'Push container apps builder images to ACR' inputs: - azureSubscriptionEndpoint: ${{ parameters.ascName }} - azureContainerRegistry: ${{ parameters.acrName }} - command: 'Push an image' - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt' - includeLatestTag: false - enforceDockerNamingConvention: false \ No newline at end of file + image: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt \ No newline at end of file diff --git a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml index f27d7c4986..812bc2ced5 100644 --- a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml +++ b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml @@ -1,91 +1,40 @@ -# trigger: none - -# The `resources` specify the location and version of the 1ES PT. -resources: - repositories: - - repository: 1esPipelines - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release - -extends: - # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. - # For non-production pipelines, use "Unofficial" as defined below. - # For productions pipelines, use "Official". - template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines - parameters: - # Update the pool with your team's 1ES hosted pool. - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant # Name of the image in your pool. If not specified, first image of the pool is used - os: linux # OS of the image. Allowed values: windows, linux, macOS - - stages: - - stage: Stage - jobs: - - job: HostJob - timeoutInMinutes: 250 - # If the pipeline publishes artifacts, use `templateContext` to define the artifacts. - # This will enable 1ES PT to run SDL analysis tools on the artifacts and then upload them. - templateContext: - outputs: - - output: pipelineArtifact - targetPath: $(Build.ArtifactStagingDirectory) - artifactName: buildImageBasesJobArtifact - # Define the steps that the pipeline will run. - # In most cases, copy and paste the steps from the original pipeline. - steps: - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - - task: ShellScript@2 - displayName: Build images - inputs: - scriptPath: ${{ parameters.scriptPath }} - args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} - env: - ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) - DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) - - - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 - displayName: Generate Software Bill of Materials (SBOM) - inputs: - BuildDropPath: '$(Build.ArtifactStagingDirectory)' - AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' - - - task: CopyFiles@2 - displayName: Copy artifacts to staging directory - inputs: - sourceFolder: '$(Build.SourcesDirectory)/artifacts' - contents: '**/*.*' - targetFolder: $(Build.ArtifactStagingDirectory) - overWrite: true - condition: true - - - task: Docker@1 - displayName: Push built base images to dev ACR - inputs: - command: push - azureSubscriptionEndpoint: $(ascName) - azureContainerRegistry: $(acrName) - pushMultipleImages: true - imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' - enforceDockerNamingConvention: false - - - task: ShellScript@2 - displayName: 'Clean up Docker containers and images' - inputs: - scriptPath: ./vsts/scripts/cleanDocker.sh - - - task: PublishBuildArtifacts@1 - displayName: Publish build artifacts - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - - - task: ShellScript@2 - displayName: 'Clean up Docker containers and images' - inputs: - scriptPath: ./vsts/scripts/cleanDocker.sh - condition: true \ No newline at end of file +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - task: ShellScript@2 + displayName: Build images + inputs: + scriptPath: ${{ parameters.scriptPath }} + args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + - task: CopyFiles@2 + displayName: Copy artifacts to staging directory + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true + - task: 1ES.PushContainerImage@1 + displayName: Push built base images to dev ACR + inputs: + image: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' + remoteImage: $(acrname)/$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }} + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/templates/_integrationJobTemplate.yml b/vsts/pipelines/templates/_integrationJobTemplate.yml index f27e8d9dad..07ff95bfe5 100644 --- a/vsts/pipelines/templates/_integrationJobTemplate.yml +++ b/vsts/pipelines/templates/_integrationJobTemplate.yml @@ -1,30 +1,20 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net jobs: -# Python integration tests -- template: integrationTests/_pythonIntegrationJobTemplate.yml +- template: /vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - -# DotNetCore integration tests -- template: integrationTests/_dotnetcoreIntegrationJobTemplate.yml +- template: /vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - -# NodeJS integration tests -- template: integrationTests/_nodeIntegrationJobTemplate.yml +- template: /vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - -# Golang integration tests -- template: integrationTests/_golangIntegrationJobTemplate.yml +- template: /vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - -# PHP integration tests -- template: integrationTests/_phpIntegrationJobTemplate.yml +- template: /vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} \ No newline at end of file diff --git a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml index 0e5110ca06..af1163ed03 100644 --- a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml @@ -4,64 +4,49 @@ parameters: acrProdName: oryxmcr acrPmeProdName: oryxprodmcr acrPmeProdSrvConnection: Oryx-PME-ACR-Production - baseImageName: '' # defaults + baseImageName: '' baseImageRepository: public/oryx/base - steps: - checkout: self clean: true - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }}.azurecr.io - - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts for release' inputs: artifactName: drop - - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(System.ArtifactsDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' - - task: Shellpp@0 displayName: 'Pull and create release tags for PME staging ACR' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBaseImagesForRelease.sh args: '${{ parameters.baseImageName }} ${{ parameters.acrPmeProdName }}' - - task: Docker@2 displayName: Login to production PME ACR inputs: command: login containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} - -- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} - +- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} - task: Shellpp@0 displayName: 'Push images to PME staging ACR' inputs: type: FilePath scriptPath: ./vsts/scripts/pushBaseImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.baseImageName }}/${{ parameters.acrPmeProdName }}' - - task: Docker@2 displayName: Logout from PME ACR inputs: command: logout containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} - - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: diff --git a/vsts/pipelines/templates/_releaseJobTemplate.yml b/vsts/pipelines/templates/_releaseJobTemplate.yml index 4064b0cd6e..39b0528908 100644 --- a/vsts/pipelines/templates/_releaseJobTemplate.yml +++ b/vsts/pipelines/templates/_releaseJobTemplate.yml @@ -4,7 +4,7 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 @@ -13,15 +13,13 @@ jobs: echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" displayName: 'Set variables' - - - template: _releaseStepTemplate.yml - + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self - job: Release_RuntimeImages displayName: Push Runtime Images to MCR pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 400 @@ -30,35 +28,25 @@ jobs: echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" displayName: 'Set variables' - - - template: _releaseStepTemplate.yml - + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self - job: Release_GitHub displayName: Create GitHub release - dependsOn: - - Release_BuildImages - - Release_RuntimeImages + dependsOn: + - Release_BuildImages + - Release_RuntimeImages pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true - steps: - task: DownloadPipelineArtifact@2 displayName: 'Download Oryx binaries to publish to release' inputs: artifactName: 'signed_binaries' path: $(Build.SourcesDirectory)/artifacts - - - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - - template: _setReleaseTag.yml - + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - task: GitHubRelease@0 displayName: 'GitHub release (create)' inputs: diff --git a/vsts/pipelines/templates/_releaseStepTemplate.yml b/vsts/pipelines/templates/_releaseStepTemplate.yml index 8cd0545dde..cc7f489f2e 100644 --- a/vsts/pipelines/templates/_releaseStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseStepTemplate.yml @@ -5,14 +5,7 @@ parameters: acrPmeProdName: oryxprodmcr acrPmeProdSrvConnection: Oryx-PME-ACR-Production prodImageName: oryxmcr.azurecr.io/public/oryx - steps: - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - script: | if [ "$(ReleaseBuildImages)" != "true" ] && [ "$(ReleaseRuntimeImages)" != "true" ] then @@ -21,19 +14,17 @@ steps: exit 1 fi displayName: 'Validate release pipeline run' - - script: | docker -v - echo "Installing Docker CE version 24.0.5................" + echo "Installing Docker CE version 24.0.5." curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt update sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - echo "After installing ................" + echo "After installing ." docker -v displayName: 'Install Docker 24.0.5' condition: true - - script: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc @@ -41,104 +32,83 @@ steps: sudo rm -rf "$AGENT_TOOLSDIRECTORY" docker images && docker system prune -fa && docker images && echo displayName: 'clean docker images' - - script: | - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt update - sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - docker -v + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + docker -v displayName: 'Install Docker 24.0.5' - - checkout: self clean: true - - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }} - - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts for release' inputs: artifactName: drop - - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(System.ArtifactsDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - -- template: _setReleaseTag.yml - +- template: /vsts/pipelines/templates/_setReleaseTag.yml@self - task: Shellpp@0 displayName: 'Pull and create release tags for build images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBuildImagesForRelease.sh - args: + args: '' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Pull and create release tags for runtime images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagRunTimeImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) - - task: Shellpp@0 displayName: 'Pull and create release tags for buildpack images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBuildpacksImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Pull and create release tags for CLI images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagCliImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Pull and create release tags for Builder images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBuilderImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 - - task: ShellScript@2 displayName: 'Test runtime images for pme staging registry' inputs: scriptPath: ./build/testRunTimeImages.sh args: skipBuildingImages ${{ parameters.acrPmeProdName }}.azurecr.io/public/oryx $(RELEASE_TAG_NAME) Category=Release condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) - - task: Docker@1 displayName: Dev Container registry logout inputs: command: logout azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }} - - task: Docker@2 displayName: Login to PME ACR - inputs: + inputs: command: login containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} - - task: Shellpp@0 displayName: 'Push build images to PME staging ACR' inputs: @@ -146,7 +116,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-build-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Push build-pack images to PME staging ACR' inputs: @@ -154,7 +123,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-buildpack-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Push CLI image to PME staging ACR' inputs: @@ -162,7 +130,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-cli-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Push builder images to PME staging ACR' inputs: @@ -170,7 +137,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-builder-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - - task: Shellpp@0 displayName: 'Push runtime images to PME staging ACR' inputs: @@ -178,13 +144,11 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-runtime-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) - - task: Docker@2 displayName: Logout from PME ACR - inputs: + inputs: command: logout containerRegistry: '${{ parameters.acrPmeProdSrvConnection }}' - - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: diff --git a/vsts/pipelines/templates/_securityChecks.yml b/vsts/pipelines/templates/_securityChecks.yml index e5ab02583a..1f9d7e63d9 100644 --- a/vsts/pipelines/templates/_securityChecks.yml +++ b/vsts/pipelines/templates/_securityChecks.yml @@ -1,38 +1,21 @@ steps: - checkout: self clean: true - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 - displayName: 'Run PoliCheck' - inputs: - targetType: F - condition: always() - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 - displayName: 'Run CredScan' - inputs: - debugMode: false - condition: always() - - task: NuGetToolInstaller@1 inputs: versionSpec: 5.8.x - - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 - - script: | dotnet --version && dotnet msbuild -version && echo displayName: 'Print dotnet and msbuild version' - - task: NuGetCommand@2 displayName: 'Run "nuget restore" on Oryx solution' inputs: command: 'restore' restoreSolution: '$(Build.SourcesDirectory)\Oryx.sln' - - task: VSBuild@1 displayName: 'Build the Oryx solution' inputs: @@ -40,37 +23,11 @@ steps: configuration: 'debug' createLogFile: true logFileVerbosity: diagnostic - - task: PowerShell@2 displayName: 'Wait 30 seconds after building solution' inputs: targetType: 'inline' script: 'Start-Sleep -Seconds 30' - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@2 - displayName: 'Run Roslyn Analyzers' - condition: always() - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 - displayName: 'Run BinSkim ' - inputs: - arguments: 'analyze Microsoft.Oryx*.dll --recurse --output $(Build.BinariesDirectory)\binskim.sarif --verbose' - condition: always() - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - -- task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@1 - displayName: 'Post Analysis' - inputs: - BinSkim: true - CredScan: true - PoliCheck: true - RoslynAnalyzers: true - continueOnError: false - - task: AntiMalware@3 displayName: 'Run Antimalware Scan' inputs: diff --git a/vsts/pipelines/templates/_setReleaseTag.yml b/vsts/pipelines/templates/_setReleaseTag.yml index e08e805b14..91b71548c2 100644 --- a/vsts/pipelines/templates/_setReleaseTag.yml +++ b/vsts/pipelines/templates/_setReleaseTag.yml @@ -4,7 +4,6 @@ steps: inputs: targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag artifactName: 'releaseTag' - - script: | set -e file="$(Build.SourcesDirectory)/artifacts/releaseTag/releaseTag.txt" diff --git a/vsts/pipelines/templates/_signBinary.yml b/vsts/pipelines/templates/_signBinary.yml index bd27bef72e..f9d054900d 100644 --- a/vsts/pipelines/templates/_signBinary.yml +++ b/vsts/pipelines/templates/_signBinary.yml @@ -1,41 +1,25 @@ steps: -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - powershell: | - Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" + Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" name: setSignTypeVariable displayName: 'Set sign binary variable' condition: > - and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), - or( - startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), - startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) - + and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), or( + startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) - script: | echo $(setSignTypeVariable.SignType) name: SignType - - task: NuGetToolInstaller@0 displayName: 'Use NuGet 4.6.2' inputs: versionSpec: 4.6.2 condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - -- task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 - displayName: 'Install Signing Plugin' - inputs: - signType: '$(setSignTypeVariable.SignType)' - condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 - - powershell: | Write-Host "Setting up git_commit and build_number as env variable" $env:GIT_COMMIT=$(git rev-parse HEAD) @@ -44,7 +28,6 @@ steps: dotnet publish -r linux-x64 -c Release src\BuildServer\BuildServer.csproj displayName: 'dotnet publish and after setting git_commit and build_number as env variable' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: VSBuild@1 displayName: 'Sign Oryx Binaries' inputs: @@ -52,7 +35,6 @@ steps: msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' configuration: '$(BuildConfiguration)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: VSBuild@1 displayName: 'Sign Oryx Build Server Binaries' inputs: @@ -60,7 +42,6 @@ steps: msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' configuration: '$(BuildConfiguration)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: ArchiveFiles@2 displayName: 'Create compressed file of Oryx binaries' inputs: @@ -72,7 +53,6 @@ steps: replaceExistingArchive: true verbose: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: ArchiveFiles@2 displayName: 'Create compressed file of Oryx Build Server binaries' inputs: @@ -84,7 +64,6 @@ steps: replaceExistingArchive: true verbose: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/binaries' inputs: @@ -93,7 +72,6 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' CleanTargetFolder: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - - task: CopyFiles@2 displayName: 'Copy BuildServer Files to: $(Build.ArtifactStagingDirectory)/binaries' inputs: @@ -101,11 +79,4 @@ steps: Contents: '**\*' TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' CleanTargetFolder: false - condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - -- task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline Artifact' - inputs: - artifactName: 'signed_binaries' - targetPath: '$(Build.ArtifactStagingDirectory)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml index 8ebf62a5f1..d0e64f1128 100644 --- a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -1,36 +1,35 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - - name: testMappings - type: object - default: - - category: '3.0' - buildTag: 'debian-stretch' - - category: '3.1' - buildTag: 'debian-stretch' - - category: '3.1' - buildTag: 'github-actions-debian-stretch' - - category: '5.0' - buildTag: 'debian-stretch' - - category: '5.0' - buildTag: 'github-actions-debian-stretch' - - category: '6.0' - buildTag: 'debian-stretch' - - category: '6.0' - buildTag: 'github-actions-debian-stretch' - - category: '7.0' - buildTag: 'github-actions-debian-buster' - - category: '8.0' - buildTag: 'github-actions-debian-bookworm' - - category: 'dynamic' - buildTag: 'debian-stretch' - - category: 'dynamic' - buildTag: 'github-actions-debian-stretch' - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '3.0' + buildTag: 'debian-stretch' + - category: '3.1' + buildTag: 'debian-stretch' + - category: '3.1' + buildTag: 'github-actions-debian-stretch' + - category: '5.0' + buildTag: 'debian-stretch' + - category: '5.0' + buildTag: 'github-actions-debian-stretch' + - category: '6.0' + buildTag: 'debian-stretch' + - category: '6.0' + buildTag: 'github-actions-debian-stretch' + - category: '7.0' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-bookworm' + - category: 'dynamic' + buildTag: 'debian-stretch' + - category: 'dynamic' + buildTag: 'github-actions-debian-stretch' jobs: - ${{ each mapping in parameters.testMappings }}: - - job: + - job: '' displayName: 'Run .NET Core ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -41,22 +40,27 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: ../_setReleaseTag.yml - - template: ../_buildTemplate.yml \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml index 2c620f7126..ad0487c996 100644 --- a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml @@ -1,26 +1,25 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - - name: testMappings - type: object - default: - - category: '1.17' - buildTag: 'full-debian-buster' - - category: '1.17' - buildTag: 'full-debian-bullseye' - - category: '1.18' - buildTag: 'full-debian-buster' - - category: '1.18' - buildTag: 'full-debian-bullseye' - - category: '1.19' - buildTag: 'full-debian-buster' - - category: '1.19' - buildTag: 'full-debian-bullseye' - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '1.17' + buildTag: 'full-debian-buster' + - category: '1.17' + buildTag: 'full-debian-bullseye' + - category: '1.18' + buildTag: 'full-debian-buster' + - category: '1.18' + buildTag: 'full-debian-bullseye' + - category: '1.19' + buildTag: 'full-debian-buster' + - category: '1.19' + buildTag: 'full-debian-bullseye' jobs: - ${{ each mapping in parameters.testMappings }}: - - job: + - job: '' displayName: 'Run Golang ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Full @@ -30,22 +29,27 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: ../_setReleaseTag.yml - - template: ../_buildTemplate.yml \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml index b69b6af5e2..85fe0d9e69 100644 --- a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -1,34 +1,33 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - - name: testMappings - type: object - default: - - category: '14-stretch-1' - buildTag: 'debian-stretch' - - category: '14-stretch-2' - buildTag: 'debian-stretch' - - category: '14-stretch-3' - buildTag: 'debian-stretch' - - category: '14-stretch-4' - buildTag: 'debian-stretch' - - category: '14-gh-buster' - buildTag: 'github-actions-debian-buster' - - category: '16' - buildTag: 'debian-stretch' - - category: '16-nuxt' - buildTag: 'debian-stretch' - - category: '18' - buildTag: 'github-actions-debian-bullseye' - - category: '20' - buildTag: 'github-actions-debian-bookworm' - - category: '20-bullseye' - buildTag: 'github-actions-debian-bullseye' - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '14-stretch-1' + buildTag: 'debian-stretch' + - category: '14-stretch-2' + buildTag: 'debian-stretch' + - category: '14-stretch-3' + buildTag: 'debian-stretch' + - category: '14-stretch-4' + buildTag: 'debian-stretch' + - category: '14-gh-buster' + buildTag: 'github-actions-debian-buster' + - category: '16' + buildTag: 'debian-stretch' + - category: '16-nuxt' + buildTag: 'debian-stretch' + - category: '18' + buildTag: 'github-actions-debian-bullseye' + - category: '20' + buildTag: 'github-actions-debian-bookworm' + - category: '20-bullseye' + buildTag: 'github-actions-debian-bullseye' jobs: - ${{ each mapping in parameters.testMappings }}: - - job: + - job: '' displayName: 'Run Node ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -39,22 +38,27 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: ../_setReleaseTag.yml - - template: ../_buildTemplate.yml \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml index 8289a9eda8..da02f7f986 100644 --- a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -1,30 +1,29 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - - name: testMappings - type: object - default: - - category: '7.4' - buildTag: 'debian-stretch' - - category: '7.4' - buildTag: 'github-actions-debian-buster' - - category: '8.0' - buildTag: 'debian-stretch' - - category: '8.0' - buildTag: 'github-actions-debian-buster' - - category: '8.1' - buildTag: 'debian-stretch' - - category: '8.1' - buildTag: 'github-actions-debian-buster' - - category: '8.2' - buildTag: 'debian-stretch' - - category: '8.2' - buildTag: 'github-actions-debian-buster' - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '7.4' + buildTag: 'debian-stretch' + - category: '7.4' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'debian-stretch' + - category: '8.0' + buildTag: 'github-actions-debian-buster' + - category: '8.1' + buildTag: 'debian-stretch' + - category: '8.1' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'debian-stretch' + - category: '8.2' + buildTag: 'github-actions-debian-buster' jobs: - ${{ each mapping in parameters.testMappings }}: - - job: + - job: '' displayName: 'Run PHP ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -35,22 +34,27 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: ../_setReleaseTag.yml - - template: ../_buildTemplate.yml \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml index 19a76fc930..31854b7680 100644 --- a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -1,36 +1,35 @@ parameters: - - name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net - - name: testMappings - type: object - default: - - category: '3.7' - buildTag: 'debian-stretch' - - category: '3.7' - buildTag: 'github-actions-debian-bullseye' - - category: '3.8' - buildTag: 'debian-stretch' - - category: '3.8' - buildTag: 'github-actions-debian-bullseye' - - category: '3.9' - buildTag: 'debian-stretch' - - category: '3.9' - buildTag: 'github-actions-debian-buster' - - category: '3.10' - buildTag: 'github-actions-debian-bullseye' - - category: '3.11' - buildTag: 'github-actions-debian-bullseye' - - category: '3.11' - buildTag: 'github-actions-debian-bookworm' - - category: '3.12' - buildTag: 'github-actions-debian-bullseye' - - category: '3.12' - buildTag: 'github-actions-debian-bookworm' - +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '3.7' + buildTag: 'debian-stretch' + - category: '3.7' + buildTag: 'github-actions-debian-bullseye' + - category: '3.8' + buildTag: 'debian-stretch' + - category: '3.8' + buildTag: 'github-actions-debian-bullseye' + - category: '3.9' + buildTag: 'debian-stretch' + - category: '3.9' + buildTag: 'github-actions-debian-buster' + - category: '3.10' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bookworm' + - category: '3.12' + buildTag: 'github-actions-debian-bullseye' + - category: '3.12' + buildTag: 'github-actions-debian-bookworm' jobs: - ${{ each mapping in parameters.testMappings }}: - - job: + - job: '' displayName: 'Run Python ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -41,22 +40,27 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish Artifact: drop' + condition: true steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: ../_setReleaseTag.yml - - template: ../_buildTemplate.yml \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file From dd5aee03e162344f0988371d88b5f9e73fd9d131 Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 22:40:47 -0700 Subject: [PATCH 4/9] Remove empty pipelineArtifact --- vsts/pipelines/ci.yml | 20 ------------------- .../_dotnetcoreIntegrationJobTemplate.yml | 5 ----- .../_golangIntegrationJobTemplate.yml | 5 ----- .../_nodeIntegrationJobTemplate.yml | 5 ----- .../_phpIntegrationJobTemplate.yml | 5 ----- .../_pythonIntegrationJobTemplate.yml | 5 ----- 6 files changed, 45 deletions(-) diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml index 8971135162..e38196b52c 100644 --- a/vsts/pipelines/ci.yml +++ b/vsts/pipelines/ci.yml @@ -144,11 +144,6 @@ extends: variables: SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] skipComponentGovernanceDetection: true - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]true" @@ -189,11 +184,6 @@ extends: variables: SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] skipComponentGovernanceDetection: true - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" @@ -221,11 +211,6 @@ extends: variables: SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] skipComponentGovernanceDetection: true - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" @@ -253,11 +238,6 @@ extends: variables: SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] skipComponentGovernanceDetection: true - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" diff --git a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml index d0e64f1128..bac8bae086 100644 --- a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -44,11 +44,6 @@ jobs: variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" diff --git a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml index ad0487c996..36443745c1 100644 --- a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml @@ -33,11 +33,6 @@ jobs: variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" diff --git a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml index 85fe0d9e69..b509dce703 100644 --- a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -42,11 +42,6 @@ jobs: variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" diff --git a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml index da02f7f986..6595764357 100644 --- a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -38,11 +38,6 @@ jobs: variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" diff --git a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml index 31854b7680..0c19a3ccec 100644 --- a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -44,11 +44,6 @@ jobs: variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish Artifact: drop' - condition: true steps: - script: | echo "##vso[task.setvariable variable=BuildBuildImages;]false" From 0f12b535789d98af4b95b42323ddc218c4dfd41a Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 22:51:44 -0700 Subject: [PATCH 5/9] Convert Oryx-PlatformBinary-Php --- vsts/pipelines/PlatformBinaries/php.yml | 191 ++++++++++-------- .../_platformBinariesReleaseTemplate.yml | 11 - .../templates/_platformBinariesTemplate.yml | 23 +-- 3 files changed, 108 insertions(+), 117 deletions(-) diff --git a/vsts/pipelines/PlatformBinaries/php.yml b/vsts/pipelines/PlatformBinaries/php.yml index 290d45beec..b6d58465b9 100644 --- a/vsts/pipelines/PlatformBinaries/php.yml +++ b/vsts/pipelines/PlatformBinaries/php.yml @@ -1,95 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Php_Bookworm_SDK - timeoutInMinutes: 1440 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'php' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Php_Bullseye_SDK - timeoutInMinutes: 1440 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'php' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Php_Stretch_SDK - timeoutInMinutes: 1440 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'php' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Php_Buster_SDK - timeoutInMinutes: 1440 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'php' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Php_Ubuntu_SDK - timeoutInMinutes: 1440 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'php' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -100,4 +15,106 @@ trigger: - /* include: - platforms/php - - vsts/PlatformBinaries/php.yml \ No newline at end of file + - vsts/PlatformBinaries/php.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Php_Bookworm_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Bullseye_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Stretch_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Buster_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Ubuntu_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml b/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml index df85c79267..461a771033 100644 --- a/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml +++ b/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml @@ -1,21 +1,12 @@ parameters: destinationSdkStorageAccountName: 'oryxsdksstaging' - steps: - - checkout: self clean: true - - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts' inputs: artifactName: drop - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - task: ShellScript@2 displayName: Upload files to Azure Storage env: @@ -27,12 +18,10 @@ steps: inputs: scriptPath: ./vsts/scripts/publishFilesToAzureStorage.sh args: ${{ parameters.destinationSdkStorageAccountName }} - - task: UseDotNet@2 displayName: 'Use .NET Core SDK 7.x' inputs: version: 7.0.306 - - task: ShellScript@2 displayName: 'Test Dev storage account' env: diff --git a/vsts/pipelines/templates/_platformBinariesTemplate.yml b/vsts/pipelines/templates/_platformBinariesTemplate.yml index 4ae719caff..294e83bf79 100644 --- a/vsts/pipelines/templates/_platformBinariesTemplate.yml +++ b/vsts/pipelines/templates/_platformBinariesTemplate.yml @@ -1,22 +1,14 @@ parameters: - platformName: '' - debianFlavor: '' - destinationSdkStorageAccountName: '' - + platformName: '' + debianFlavor: '' + destinationSdkStorageAccountName: '' steps: - checkout: self clean: true - -- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 - displayName: 'Component Detection - OSS Compliance' - inputs: - ignoreDirectories: '$(Build.SourcesDirectory)/tests' - - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 - - task: ShellScript@2 displayName: 'Building platform binaries' env: @@ -25,7 +17,6 @@ steps: inputs: scriptPath: ./build/buildPlatformBinaries.sh args: ${{ parameters.platformName }} ${{ parameters.debianFlavor }} https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net - - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -33,10 +24,4 @@ steps: contents: '**/*.*' targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true - condition: true - -- task: PublishBuildArtifacts@1 - displayName: Publish build artifacts - inputs: - pathtoPublish: $(Build.ArtifactStagingDirectory) - condition: true + condition: true \ No newline at end of file From 59e7808dc80a0b1fb32893d70b2996c62a3954f4 Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 23:05:51 -0700 Subject: [PATCH 6/9] Convert more PlatformBinaries --- .../pipelines/PlatformBinaries/dotnetcore.yml | 191 ++++++------ vsts/pipelines/PlatformBinaries/golang.yml | 164 ++++++----- vsts/pipelines/PlatformBinaries/java.yml | 272 ++++++++++-------- vsts/pipelines/PlatformBinaries/node.yml | 179 +++++++----- vsts/pipelines/PlatformBinaries/python.yml | 191 ++++++------ vsts/pipelines/PlatformBinaries/ruby.yml | 164 ++++++----- 6 files changed, 634 insertions(+), 527 deletions(-) diff --git a/vsts/pipelines/PlatformBinaries/dotnetcore.yml b/vsts/pipelines/PlatformBinaries/dotnetcore.yml index 87e83d0238..c61bf3d570 100644 --- a/vsts/pipelines/PlatformBinaries/dotnetcore.yml +++ b/vsts/pipelines/PlatformBinaries/dotnetcore.yml @@ -1,95 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: DotNetCore_Bookworm - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'dotnet' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: DotNetCore_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'dotnet' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: DotNetCore_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'dotnet' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: DotNetCore_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'dotnet' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: DotNetCore_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'dotnet' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -100,4 +15,106 @@ trigger: - /* include: - platforms/dotnet - - vsts/PlatformBinaries/dotnetcore.yml \ No newline at end of file + - vsts/PlatformBinaries/dotnetcore.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: DotNetCore_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/golang.yml b/vsts/pipelines/PlatformBinaries/golang.yml index 4fc974d8a4..4fc2046c69 100644 --- a/vsts/pipelines/PlatformBinaries/golang.yml +++ b/vsts/pipelines/PlatformBinaries/golang.yml @@ -1,82 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Golang_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'golang' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Golang_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'golang' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Golang_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'golang' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Golang_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'golang' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -87,4 +15,92 @@ trigger: - /* include: - platforms/golang - - vsts/PlatformBinaries/golang.yml \ No newline at end of file + - vsts/PlatformBinaries/golang.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Golang_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/java.yml b/vsts/pipelines/PlatformBinaries/java.yml index 01c262e715..cd6513896b 100644 --- a/vsts/pipelines/PlatformBinaries/java.yml +++ b/vsts/pipelines/PlatformBinaries/java.yml @@ -1,134 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Java_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'java' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Java_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'java' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Java_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'java' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Java_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'java' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Maven_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'maven' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Maven_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'maven' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Maven_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'maven' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Maven_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'maven' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -139,4 +15,148 @@ trigger: - /* include: - platforms/java - - vsts/PlatformBinaries/java.yml \ No newline at end of file + - vsts/PlatformBinaries/java.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Java_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/node.yml b/vsts/pipelines/PlatformBinaries/node.yml index 172cb39b14..cc9ec7f03a 100644 --- a/vsts/pipelines/PlatformBinaries/node.yml +++ b/vsts/pipelines/PlatformBinaries/node.yml @@ -1,3 +1,10 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging trigger: batch: true branches: @@ -9,91 +16,105 @@ trigger: include: - platforms/nodejs - vsts/PlatformBinaries/node.yml - -# The `resources` specify the location and version of the 1ES PT. resources: repositories: - - repository: 1esPipelines + - repository: 1ESPipelineTemplates type: git name: 1ESPipelineTemplates/1ESPipelineTemplates ref: refs/tags/release - -variables: - - group: Oryx - extends: - # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. - # For non-production pipelines, use "Unofficial" as defined below. - # For productions pipelines, use "Official". - template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - # Update the pool with your team's 1ES hosted pool. pool: name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant # Name of the image in your pool. If not specified, first image of the pool is used - os: linux # OS of the image. Allowed values: windows, linux, macOS - - # - name: destinationStorageAccountName - # displayName: Destination Storage Account Name - # type: string - # default: oryxsdksstaging - - stages: - - stage: Build - jobs: - - job: Node_Bookworm - timeoutInMinutes: 250 - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Bullseye - timeoutInMinutes: 250 - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Buster - timeoutInMinutes: 250 - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Stretch - timeoutInMinutes: 250 - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Node_Ubuntu - timeoutInMinutes: 250 - pool: - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'nodejs' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Node_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/python.yml b/vsts/pipelines/PlatformBinaries/python.yml index 48378035cf..38cfa613b1 100644 --- a/vsts/pipelines/PlatformBinaries/python.yml +++ b/vsts/pipelines/PlatformBinaries/python.yml @@ -1,95 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Python_Bookworm_SDK - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'python' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Python_Bullseye_SDK - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'python' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Python_Buster_SDK - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'python' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Python_Stretch_SDK - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'python' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Python_Ubuntu_SDK - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'python' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -100,4 +15,106 @@ trigger: - /* include: - platforms/python - - vsts/PlatformBinaries/python.yml \ No newline at end of file + - vsts/PlatformBinaries/python.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Python_Bookworm_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Bullseye_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Buster_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Stretch_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Ubuntu_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/ruby.yml b/vsts/pipelines/PlatformBinaries/ruby.yml index 8405f9d8bb..3c16e028cd 100644 --- a/vsts/pipelines/PlatformBinaries/ruby.yml +++ b/vsts/pipelines/PlatformBinaries/ruby.yml @@ -1,82 +1,10 @@ variables: - - group: Oryx - +- group: Oryx parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging - -stages: -- stage: Build - jobs: - - job: Ruby_Bullseye - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'ruby' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Ruby_Buster - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'ruby' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Ruby_Stretch - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'ruby' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - - job: Ruby_Ubuntu - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesTemplate.yml - parameters: - platformName: 'ruby' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - -- stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_platformBinariesReleaseTemplate.yml - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - trigger: batch: true branches: @@ -87,4 +15,92 @@ trigger: - /* include: - platforms/ruby - - vsts/PlatformBinaries/ruby.yml \ No newline at end of file + - vsts/PlatformBinaries/ruby.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Ruby_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file From 450be56ea2caf56bfbdbdf5537b295792a767062 Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 23:13:11 -0700 Subject: [PATCH 7/9] Add python base image --- vsts/pipelines/baseImages/python.yml | 80 ++++++++++++++++------------ 1 file changed, 47 insertions(+), 33 deletions(-) diff --git a/vsts/pipelines/baseImages/python.yml b/vsts/pipelines/baseImages/python.yml index c6b377d507..1970265023 100644 --- a/vsts/pipelines/baseImages/python.yml +++ b/vsts/pipelines/baseImages/python.yml @@ -8,37 +8,51 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true - -jobs: -- template: ../templates/_buildimageBasesJobTemplate.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates parameters: - displayName: Build python runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: python - imageDebianFlavor: bullseye - artifactsFileName: python-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - -- template: ../templates/_buildimageBasesJobTemplate.yml - parameters: - displayName: Build python runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: python - imageDebianFlavor: bookworm - artifactsFileName: python-runtimeimage-bases-bookworm.txt - jobName: Build_Bookworm_BaseImages - -- job: Release_PythonRuntimeBaseImage - dependsOn: - - Build_Bullseye_BaseImages - - Build_Bookworm_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - steps: - - template: ../templates/_releaseBaseImagesStepTemplate.yml - parameters: - baseImageName: 'python' + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bullseye + artifactsFileName: python-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bookworm + artifactsFileName: python-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_PythonRuntimeBaseImage + dependsOn: + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'python' \ No newline at end of file From fcca1138153785c360f0b52ea0a6dfa4e6aacfeb Mon Sep 17 00:00:00 2001 From: Dan Vouaux Date: Thu, 21 Mar 2024 23:49:19 -0700 Subject: [PATCH 8/9] Fix merge conflicts --- vsts/pipelines/baseImages/dotnetcore.yml | 2 +- vsts/pipelines/baseImages/node.yml | 2 +- vsts/pipelines/baseImages/php-fpm.yml | 11 ++++- vsts/pipelines/baseImages/php.yml | 11 ++++- vsts/pipelines/baseImages/python.yml | 2 +- vsts/pipelines/ci.yml | 16 ------- vsts/pipelines/templates/_buildTemplate.yml | 2 +- vsts/pipelines/templates/_builderTemplate.yml | 2 +- .../templates/_integrationJobTemplate.yml | 3 -- .../_releaseBaseImagesStepTemplate.yml | 2 +- .../templates/_releaseStepTemplate.yml | 42 +++---------------- .../_dotnetcoreIntegrationJobTemplate.yml | 18 ++------ .../_nodeIntegrationJobTemplate.yml | 12 +----- .../_phpIntegrationJobTemplate.yml | 10 +---- .../_pythonIntegrationJobTemplate.yml | 6 +-- 15 files changed, 39 insertions(+), 102 deletions(-) diff --git a/vsts/pipelines/baseImages/dotnetcore.yml b/vsts/pipelines/baseImages/dotnetcore.yml index fa23b571d3..0695591eed 100644 --- a/vsts/pipelines/baseImages/dotnetcore.yml +++ b/vsts/pipelines/baseImages/dotnetcore.yml @@ -1,7 +1,7 @@ variables: - group: Oryx - name: ascName - value: oryx-automation-service-principal + value: oryx-new-service-connection - name: acrName value: oryxdevmcr.azurecr.io - name: skipComponentGovernanceDetection diff --git a/vsts/pipelines/baseImages/node.yml b/vsts/pipelines/baseImages/node.yml index 14a7dcd722..7e89a2cd0c 100644 --- a/vsts/pipelines/baseImages/node.yml +++ b/vsts/pipelines/baseImages/node.yml @@ -1,7 +1,7 @@ variables: - group: Oryx - name: ascName - value: oryx-automation-service-principal + value: oryx-new-service-connection - name: acrName value: oryxdevmcr.azurecr.io - name: skipComponentGovernanceDetection diff --git a/vsts/pipelines/baseImages/php-fpm.yml b/vsts/pipelines/baseImages/php-fpm.yml index b57bd7f722..b83c80ca9c 100644 --- a/vsts/pipelines/baseImages/php-fpm.yml +++ b/vsts/pipelines/baseImages/php-fpm.yml @@ -1,7 +1,7 @@ variables: - group: Oryx - name: ascName - value: oryx-automation-service-principal + value: oryx-new-service-connection - name: acrName value: oryxdevmcr.azurecr.io - name: skipComponentGovernanceDetection @@ -46,10 +46,19 @@ extends: imageDebianFlavor: bullseye artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt jobName: Build_PHP_Fpm_Bullseye_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bookworm runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bookworm + artifactsFileName: php-fpm-runtimeimage-bases-bookworm.txt + jobName: Build_PHP_Fpm_Bookworm_Base - job: Release_PhpRuntimeBaseImage dependsOn: - Build_Php_Fpm_Buster_Base - Build_PHP_Fpm_Bullseye_Base + - Build_PHP_Fpm_Bookworm_Base displayName: Push images to MCR timeoutInMinutes: 250 steps: diff --git a/vsts/pipelines/baseImages/php.yml b/vsts/pipelines/baseImages/php.yml index 0964c12f4a..72f979edee 100644 --- a/vsts/pipelines/baseImages/php.yml +++ b/vsts/pipelines/baseImages/php.yml @@ -1,7 +1,7 @@ variables: - group: Oryx - name: ascName - value: oryx-automation-service-principal + value: oryx-new-service-connection - name: acrName value: oryxdevmcr.azurecr.io - name: skipComponentGovernanceDetection @@ -46,10 +46,19 @@ extends: imageDebianFlavor: bullseye artifactsFileName: php-runtimeimage-bases-bullseye.txt jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bookworm + artifactsFileName: php-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages - job: Release_PhpRuntimeBaseImage dependsOn: - Build_Buster_BaseImages - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages displayName: Push images to MCR timeoutInMinutes: 250 steps: diff --git a/vsts/pipelines/baseImages/python.yml b/vsts/pipelines/baseImages/python.yml index 1970265023..86a0b8f73e 100644 --- a/vsts/pipelines/baseImages/python.yml +++ b/vsts/pipelines/baseImages/python.yml @@ -1,7 +1,7 @@ variables: - group: Oryx - name: ascName - value: oryx-automation-service-principal + value: oryx-new-service-connection - name: acrName value: oryxdevmcr.azurecr.io - name: skipComponentGovernanceDetection diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml index e38196b52c..94195443a0 100644 --- a/vsts/pipelines/ci.yml +++ b/vsts/pipelines/ci.yml @@ -10,16 +10,10 @@ parameters: default: - key: Latest value: latest - - key: LtsVersions - value: ltsversions - key: Jamstack value: jamstack - key: GithubActions value: githubactions - - key: VsoFocal - value: vso-focal - - key: VsoBullseye - value: vso-bullseye - key: Full value: full - key: Cli @@ -162,16 +156,6 @@ extends: - template: /vsts/pipelines/templates/_buildTemplate.yml@self parameters: imageType: ${{ buildImage.value }} - - job: Job_BuilderImages - displayName: Build Builder Images - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - timeoutInMinutes: 480 - dependsOn: Job_BuildImage_CliBuilderBullseye - steps: - - template: /vsts/pipelines/templates/_builderTemplate.yml@self - job: Job_Buster_RuntimeImages displayName: Build and Test Buster Runtime Images dependsOn: Job_SignBinaries diff --git a/vsts/pipelines/templates/_buildTemplate.yml b/vsts/pipelines/templates/_buildTemplate.yml index f98cc5a60f..b6d3ba250a 100644 --- a/vsts/pipelines/templates/_buildTemplate.yml +++ b/vsts/pipelines/templates/_buildTemplate.yml @@ -1,5 +1,5 @@ parameters: - ascName: oryx-automation-service-principal + ascName: oryx-new-service-connection acrName: oryxdevmcr.azurecr.io imageName: oryxdevmcr.azurecr.io/public/oryx imageType: null diff --git a/vsts/pipelines/templates/_builderTemplate.yml b/vsts/pipelines/templates/_builderTemplate.yml index b5841ce2e9..0890946854 100644 --- a/vsts/pipelines/templates/_builderTemplate.yml +++ b/vsts/pipelines/templates/_builderTemplate.yml @@ -1,5 +1,5 @@ parameters: - ascName: oryx-automation-service-principal + ascName: oryx-new-service-connection acrName: oryxdevmcr.azurecr.io steps: - script: | diff --git a/vsts/pipelines/templates/_integrationJobTemplate.yml b/vsts/pipelines/templates/_integrationJobTemplate.yml index 07ff95bfe5..7f14fafcb1 100644 --- a/vsts/pipelines/templates/_integrationJobTemplate.yml +++ b/vsts/pipelines/templates/_integrationJobTemplate.yml @@ -12,9 +12,6 @@ jobs: - template: /vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} -- template: /vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml@self - parameters: - storageAccountUrl: ${{ parameters.storageAccountUrl }} - template: /vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml@self parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} \ No newline at end of file diff --git a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml index af1163ed03..6f050f5dd6 100644 --- a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml @@ -1,5 +1,5 @@ parameters: - ascName: oryx-automation-service-principal + ascName: oryx-new-service-connection acrDevName: oryxdevmcr acrProdName: oryxmcr acrPmeProdName: oryxprodmcr diff --git a/vsts/pipelines/templates/_releaseStepTemplate.yml b/vsts/pipelines/templates/_releaseStepTemplate.yml index cc7f489f2e..671c653163 100644 --- a/vsts/pipelines/templates/_releaseStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseStepTemplate.yml @@ -1,5 +1,5 @@ parameters: - ascName: oryx-automation-service-principal + ascName: oryx-new-service-connection acrDevName: oryxdevmcr.azurecr.io acrProdName: oryxmcr acrPmeProdName: oryxprodmcr @@ -32,6 +32,10 @@ steps: sudo rm -rf "$AGENT_TOOLSDIRECTORY" docker images && docker system prune -fa && docker images && echo displayName: 'clean docker images' +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 - script: | curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null @@ -70,34 +74,12 @@ steps: type: FilePath scriptPath: ./vsts/scripts/tagRunTimeImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) -- task: Shellpp@0 - displayName: 'Pull and create release tags for buildpack images' - inputs: - type: FilePath - scriptPath: ./vsts/scripts/tagBuildpacksImagesForRelease.sh - condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - task: Shellpp@0 displayName: 'Pull and create release tags for CLI images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagCliImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) -- task: Shellpp@0 - displayName: 'Pull and create release tags for Builder images' - inputs: - type: FilePath - scriptPath: ./vsts/scripts/tagBuilderImagesForRelease.sh - condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) -- task: UseDotNet@2 - displayName: 'Use .NET Core sdk 7.x' - inputs: - version: 7.0.306 -- task: ShellScript@2 - displayName: 'Test runtime images for pme staging registry' - inputs: - scriptPath: ./build/testRunTimeImages.sh - args: skipBuildingImages ${{ parameters.acrPmeProdName }}.azurecr.io/public/oryx $(RELEASE_TAG_NAME) Category=Release - condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) - task: Docker@1 displayName: Dev Container registry logout inputs: @@ -116,13 +98,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-build-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) -- task: Shellpp@0 - displayName: 'Push build-pack images to PME staging ACR' - inputs: - type: FilePath - scriptPath: ./vsts/scripts/pushImagesToRegistry.sh - args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-buildpack-images-mcr.txt' - condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - task: Shellpp@0 displayName: 'Push CLI image to PME staging ACR' inputs: @@ -130,13 +105,6 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-cli-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) -- task: Shellpp@0 - displayName: 'Push builder images to PME staging ACR' - inputs: - type: FilePath - scriptPath: ./vsts/scripts/pushImagesToRegistry.sh - args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-builder-images-mcr.txt' - condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) - task: Shellpp@0 displayName: 'Push runtime images to PME staging ACR' inputs: diff --git a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml index bac8bae086..78b2aed23b 100644 --- a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -5,28 +5,18 @@ parameters: - name: testMappings type: object default: - - category: '3.0' - buildTag: 'debian-stretch' - - category: '3.1' - buildTag: 'debian-stretch' - - category: '3.1' - buildTag: 'github-actions-debian-stretch' - - category: '5.0' - buildTag: 'debian-stretch' - - category: '5.0' - buildTag: 'github-actions-debian-stretch' - category: '6.0' - buildTag: 'debian-stretch' + buildTag: 'debian-buster' - category: '6.0' - buildTag: 'github-actions-debian-stretch' + buildTag: 'github-actions-debian-buster' - category: '7.0' buildTag: 'github-actions-debian-buster' - category: '8.0' buildTag: 'github-actions-debian-bookworm' - category: 'dynamic' - buildTag: 'debian-stretch' + buildTag: 'debian-buster' - category: 'dynamic' - buildTag: 'github-actions-debian-stretch' + buildTag: 'github-actions-debian-buster' jobs: - ${{ each mapping in parameters.testMappings }}: - job: '' diff --git a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml index b509dce703..d79da61eae 100644 --- a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -5,20 +5,10 @@ parameters: - name: testMappings type: object default: - - category: '14-stretch-1' - buildTag: 'debian-stretch' - - category: '14-stretch-2' - buildTag: 'debian-stretch' - - category: '14-stretch-3' - buildTag: 'debian-stretch' - - category: '14-stretch-4' - buildTag: 'debian-stretch' - category: '14-gh-buster' buildTag: 'github-actions-debian-buster' - category: '16' - buildTag: 'debian-stretch' - - category: '16-nuxt' - buildTag: 'debian-stretch' + buildTag: 'debian-buster' - category: '18' buildTag: 'github-actions-debian-bullseye' - category: '20' diff --git a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml index 6595764357..871fd35362 100644 --- a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -5,22 +5,16 @@ parameters: - name: testMappings type: object default: - - category: '7.4' - buildTag: 'debian-stretch' - category: '7.4' buildTag: 'github-actions-debian-buster' - - category: '8.0' - buildTag: 'debian-stretch' - category: '8.0' buildTag: 'github-actions-debian-buster' - - category: '8.1' - buildTag: 'debian-stretch' - category: '8.1' buildTag: 'github-actions-debian-buster' - - category: '8.2' - buildTag: 'debian-stretch' - category: '8.2' buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-bullseye' jobs: - ${{ each mapping in parameters.testMappings }}: - job: '' diff --git a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml index 0c19a3ccec..4342f98679 100644 --- a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -5,16 +5,12 @@ parameters: - name: testMappings type: object default: - - category: '3.7' - buildTag: 'debian-stretch' - category: '3.7' buildTag: 'github-actions-debian-bullseye' - - category: '3.8' - buildTag: 'debian-stretch' - category: '3.8' buildTag: 'github-actions-debian-bullseye' - category: '3.9' - buildTag: 'debian-stretch' + buildTag: 'debian-buster' - category: '3.9' buildTag: 'github-actions-debian-buster' - category: '3.10' From 9c3c45b4c3a27e312fc5b518528c1b28fc6fbc8a Mon Sep 17 00:00:00 2001 From: harrli Date: Tue, 26 Mar 2024 13:47:36 -0700 Subject: [PATCH 9/9] Temporarily moved 1ES pipeline templates to separate folder and kept old templates --- .../PlatformBinaries/dotnetcore.yml | 120 +++++ .../1ESPipelines/PlatformBinaries/golang.yml | 106 ++++ .../1ESPipelines/PlatformBinaries/java.yml | 162 ++++++ .../1ESPipelines/PlatformBinaries/node.yml | 120 +++++ .../1ESPipelines/PlatformBinaries/php.yml | 120 +++++ .../1ESPipelines/PlatformBinaries/python.yml | 120 +++++ .../1ESPipelines/PlatformBinaries/ruby.yml | 106 ++++ vsts/pipelines/1ESPipelines/agentCleanUp.yml | 25 + .../1ESPipelines/baseImages/dotnetcore.yml | 67 +++ .../1ESPipelines/baseImages/node.yml | 65 +++ .../1ESPipelines/baseImages/php-fpm.yml | 67 +++ .../pipelines/1ESPipelines/baseImages/php.yml | 67 +++ .../1ESPipelines/baseImages/python.yml | 58 ++ vsts/pipelines/1ESPipelines/ci-detector.yml | 115 ++++ vsts/pipelines/1ESPipelines/ci.yml | 269 ++++++++++ vsts/pipelines/1ESPipelines/ciDebug.yml | 196 +++++++ .../1ESPipelines/copyAllBlobsToProd.yml | 54 ++ .../copySdksFromProdToStorageAccount.yml | 62 +++ vsts/pipelines/1ESPipelines/nightly.yml | 235 ++++++++ .../1ESPipelines/publishSdkToProd.yml | 75 +++ .../templates/_agentCleanUpJobTemplate.yml | 13 + .../1ESPipelines/templates/_buildTemplate.yml | 211 ++++++++ .../templates/_buildTemplateDetector.yml | 52 ++ .../templates/_builderTemplate.yml | 41 ++ .../templates/_buildimageBasesJobTemplate.yml | 40 ++ .../templates/_cleanImageCacheTemplate.yml | 18 + .../templates/_copyImgFromDevToTest.yml | 246 +++++++++ .../templates/_integrationJobTemplate.yml | 17 + .../_platformBinariesReleaseTemplate.yml | 32 ++ .../templates/_platformBinariesTemplate.yml | 27 + .../_releaseBaseImagesJobTemplate.yml | 61 +++ .../_releaseBaseImagesStepTemplate.yml | 54 ++ .../templates/_releaseJobTemplate.yml | 57 ++ .../templates/_releaseStepTemplate.yml | 124 +++++ .../templates/_securityChecks.yml | 41 ++ .../1ESPipelines/templates/_setReleaseTag.yml | 18 + .../1ESPipelines/templates/_signBinary.yml | 82 +++ .../templates/_signBinaryDetector.yml | 71 +++ .../_dotnetcoreIntegrationJobTemplate.yml | 51 ++ .../_golangIntegrationJobTemplate.yml | 50 ++ .../_nodeIntegrationJobTemplate.yml | 49 ++ .../_phpIntegrationJobTemplate.yml | 49 ++ .../_pythonIntegrationJobTemplate.yml | 57 ++ .../1ESPipelines/testACRImgUpload.yml | 28 + .../1ESPipelines/validation-detector.yml | 22 + vsts/pipelines/1ESPipelines/validation.yml | 171 ++++++ .../pipelines/PlatformBinaries/dotnetcore.yml | 191 +++---- vsts/pipelines/PlatformBinaries/golang.yml | 164 +++--- vsts/pipelines/PlatformBinaries/java.yml | 272 +++++----- vsts/pipelines/PlatformBinaries/node.yml | 191 +++---- vsts/pipelines/PlatformBinaries/php.yml | 191 +++---- vsts/pipelines/PlatformBinaries/python.yml | 191 +++---- vsts/pipelines/PlatformBinaries/ruby.yml | 164 +++--- vsts/pipelines/baseImages/dotnetcore.yml | 99 ++-- vsts/pipelines/baseImages/node.yml | 95 ++-- vsts/pipelines/baseImages/php-fpm.yml | 99 ++-- vsts/pipelines/baseImages/php.yml | 99 ++-- vsts/pipelines/baseImages/python.yml | 80 ++- vsts/pipelines/ci.yml | 500 +++++++++--------- vsts/pipelines/templates/_buildTemplate.yml | 83 ++- vsts/pipelines/templates/_builderTemplate.yml | 29 +- .../templates/_buildimageBasesJobTemplate.yml | 51 +- .../templates/_integrationJobTemplate.yml | 22 +- .../_platformBinariesReleaseTemplate.yml | 11 + .../templates/_platformBinariesTemplate.yml | 23 +- .../_releaseBaseImagesStepTemplate.yml | 19 +- .../templates/_releaseJobTemplate.yml | 30 +- .../templates/_releaseStepTemplate.yml | 56 +- vsts/pipelines/templates/_securityChecks.yml | 43 ++ vsts/pipelines/templates/_setReleaseTag.yml | 1 + vsts/pipelines/templates/_signBinary.yml | 39 +- .../_dotnetcoreIntegrationJobTemplate.yml | 69 +-- .../_golangIntegrationJobTemplate.yml | 69 +-- .../_nodeIntegrationJobTemplate.yml | 65 +-- .../_phpIntegrationJobTemplate.yml | 75 +-- .../_pythonIntegrationJobTemplate.yml | 81 +-- 76 files changed, 5500 insertions(+), 1493 deletions(-) create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml create mode 100644 vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml create mode 100644 vsts/pipelines/1ESPipelines/agentCleanUp.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/node.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/php.yml create mode 100644 vsts/pipelines/1ESPipelines/baseImages/python.yml create mode 100644 vsts/pipelines/1ESPipelines/ci-detector.yml create mode 100644 vsts/pipelines/1ESPipelines/ci.yml create mode 100644 vsts/pipelines/1ESPipelines/ciDebug.yml create mode 100644 vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml create mode 100644 vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml create mode 100644 vsts/pipelines/1ESPipelines/nightly.yml create mode 100644 vsts/pipelines/1ESPipelines/publishSdkToProd.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_securityChecks.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_signBinary.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml create mode 100644 vsts/pipelines/1ESPipelines/testACRImgUpload.yml create mode 100644 vsts/pipelines/1ESPipelines/validation-detector.yml create mode 100644 vsts/pipelines/1ESPipelines/validation.yml diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml new file mode 100644 index 0000000000..c61bf3d570 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/dotnetcore.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/dotnet + - vsts/PlatformBinaries/dotnetcore.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: DotNetCore_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: DotNetCore_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'dotnet' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml new file mode 100644 index 0000000000..4fc2046c69 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/golang.yml @@ -0,0 +1,106 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/golang + - vsts/PlatformBinaries/golang.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Golang_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Golang_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'golang' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml new file mode 100644 index 0000000000..cd6513896b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/java.yml @@ -0,0 +1,162 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/java + - vsts/PlatformBinaries/java.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Java_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Java_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'java' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Maven_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'maven' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml new file mode 100644 index 0000000000..cc9ec7f03a --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/node.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/nodejs + - vsts/PlatformBinaries/node.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Node_Bookworm + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Node_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'nodejs' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml new file mode 100644 index 0000000000..b6d58465b9 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/php.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/php + - vsts/PlatformBinaries/php.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Php_Bookworm_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Bullseye_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Stretch_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Buster_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Php_Ubuntu_SDK + timeoutInMinutes: 1440 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'php' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml new file mode 100644 index 0000000000..38cfa613b1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/python.yml @@ -0,0 +1,120 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/python + - vsts/PlatformBinaries/python.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Python_Bookworm_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Bullseye_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Buster_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Stretch_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Python_Ubuntu_SDK + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'python' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml b/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml new file mode 100644 index 0000000000..3c16e028cd --- /dev/null +++ b/vsts/pipelines/1ESPipelines/PlatformBinaries/ruby.yml @@ -0,0 +1,106 @@ +variables: +- group: Oryx +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksstaging +trigger: + batch: true + branches: + include: + - main + paths: + exclude: + - /* + include: + - platforms/ruby + - vsts/PlatformBinaries/ruby.yml +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling + stages: + - stage: Build + jobs: + - job: Ruby_Bullseye + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Buster + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Stretch + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - job: Ruby_Ubuntu + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + condition: true + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self + parameters: + platformName: 'ruby' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + - stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + steps: + - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/agentCleanUp.yml b/vsts/pipelines/1ESPipelines/agentCleanUp.yml new file mode 100644 index 0000000000..379f067a0e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/agentCleanUp.yml @@ -0,0 +1,25 @@ +parameters: +- name: poolName + displayName: Pool Name + type: string + default: Azure Pipelines + values: + - Azure Pipelines +- name: agentName + displayName: Agent Name + type: string + +variables: + skipComponentGovernanceDetection: true + +stages: +- stage: CleanAgents + displayName: Clean Oryx Agents + jobs: + - template: templates/_agentCleanUpJobTemplate.yml + parameters: + agentName: ${{ parameters.agentName }} + poolName: ${{ parameters.poolName }} + +# trigger is set using vsts UI, cleanup build will run every sundays 10 PM PST/PDT +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml b/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml new file mode 100644 index 0000000000..0695591eed --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/dotnetcore.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: buster + artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt + jobName: Build_DotNetCore_BaseImage_Buster + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bullseye + artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt + jobName: Build_DotNetCore_BaseImage_Bullseye + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build DotNetCore runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bookworm + artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt + jobName: Build_DotNetCore_BaseImage_Bookworm + - job: Release_DotNetCoreRuntimeBaseImage + dependsOn: + - Build_DotNetCore_BaseImage_Buster + - Build_DotNetCore_BaseImage_Bullseye + - Build_DotNetCore_BaseImage_Bookworm + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'dotnetcore' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/node.yml b/vsts/pipelines/1ESPipelines/baseImages/node.yml new file mode 100644 index 0000000000..7e89a2cd0c --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/node.yml @@ -0,0 +1,65 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime buster based images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: buster + artifactsFileName: node-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bullseye + artifactsFileName: node-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build Node runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bookworm + artifactsFileName: node-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_NodeRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'node' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml b/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml new file mode 100644 index 0000000000..95b62a21ca --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/php-fpm.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php buster runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: buster + artifactsFileName: php-fpm-runtimeimage-bases-buster.txt + jobName: Build_PHP_FPM_Buster_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bullseye runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bullseye + artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt + jobName: Build_PHP_Fpm_Bullseye_Base + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php bookworm runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bookworm + artifactsFileName: php-fpm-runtimeimage-bases-bookworm.txt + jobName: Build_PHP_Fpm_Bookworm_Base + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Php_Fpm_Buster_Base + - Build_PHP_Fpm_Bullseye_Base + - Build_PHP_Fpm_Bookworm_Base + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php-fpm' diff --git a/vsts/pipelines/1ESPipelines/baseImages/php.yml b/vsts/pipelines/1ESPipelines/baseImages/php.yml new file mode 100644 index 0000000000..1ca3afd92b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/php.yml @@ -0,0 +1,67 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: buster + artifactsFileName: php-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bullseye + artifactsFileName: php-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build php runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bookworm + artifactsFileName: php-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'php' diff --git a/vsts/pipelines/1ESPipelines/baseImages/python.yml b/vsts/pipelines/1ESPipelines/baseImages/python.yml new file mode 100644 index 0000000000..86a0b8f73e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/baseImages/python.yml @@ -0,0 +1,58 @@ +variables: +- group: Oryx +- name: ascName + value: oryx-new-service-connection +- name: acrName + value: oryxdevmcr.azurecr.io +- name: skipComponentGovernanceDetection + value: true +- name: Packaging.EnableSBOMSigning + value: true +resources: + repositories: + - repository: 1ESPipelineTemplates + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release +extends: + template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + parameters: + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + os: windows + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: stage + jobs: + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bullseye + artifactsFileName: python-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self + parameters: + displayName: Build python runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bookworm + artifactsFileName: python-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + - job: Release_PythonRuntimeBaseImage + dependsOn: + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + steps: + - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self + parameters: + baseImageName: 'python' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ci-detector.yml b/vsts/pipelines/1ESPipelines/ci-detector.yml new file mode 100644 index 0000000000..39220d3c54 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ci-detector.yml @@ -0,0 +1,115 @@ +resources: +- repo: self + +variables: +- group: Oryx + +stages: + - stage: ValidatePipeline + displayName: Validate Pipeline + jobs: + - job: ValidatePipeline + displayName: Validate pipeline + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - script: | + set -x + if [ "$(Build.Reason)" != "Manual" ]; then + echo + echo "Invalid configuration." + echo "Build should be Manual" + exit 1 + fi + if [ "$(Build.DefinitionName)" != "Oryx-Detector-CI" ]; then + echo + echo "Invalid configuration." + echo "Build definition name should be Oryx-Detector-CI." + exit 1 + fi + if [[ "$(Build.SourceBranch)" != "refs/heads/main" ]] \ + && [[ "$(Build.SourceBranch)" != refs/heads/patch* ]] \ + && [[ "$(Build.SourceBranch)" != refs/heads/exp* ]]; then + echo + echo "Invalid configuration." + echo 'Only 'main', 'patch/*' and 'exp/*' branches are allowed.' + exit 1 + fi + displayName: 'Validate pipeline run' + + - stage: Build + displayName: Build Stage + dependsOn: ValidatePipeline + jobs: + - job: Job_SignBinariesAndPackages + displayName: Sign Detector Binaries and NuGet packages + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + skipComponentGovernanceDetection: true + steps: + - template: templates/_signBinaryDetector.yml + - job: Job_BuildAndTestDetector + displayName: Build and test detector + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: templates/_buildTemplateDetector.yml + + - stage: Release + displayName: Release Stage + dependsOn: Build + jobs: + - job: Release_GitHub + displayName: Create GitHub release + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - checkout: self + clean: true + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifacts to publish to release' + inputs: + artifactName: 'detector_signed' + path: $(Build.ArtifactStagingDirectory)/detector_signed + - bash: | + . build/detector/__detectorNugetPackagesVersions.sh + echo "Setting release name to '$releaseName'..." + echo "##vso[task.setvariable variable=RELEASE_NAME;]$VERSION" + displayName: 'Set relase name environment variable' + - bash: | + dotnet nuget push \ + $(Build.ArtifactStagingDirectory)/detector_signed/packages/*.nupkg \ + -s https://api.nuget.org/v3/index.json \ + --api-key $NUGETORG_API_KEY + env: + # Map the value that we got from keyvault/variablegroup to environment variable + NUGETORG_API_KEY: $(NUGETORG-API-KEY) + displayName: 'Push nuget packages to nuget.org' + - task: GitHubRelease@0 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: 'Oryx-GitHub' + repositoryName: microsoft/oryx + tagSource: manual + tag: '$(RELEASE_NAME)' + assets: $(Build.ArtifactStagingDirectory)/detector_signed/packages/*.nupkg +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ci.yml b/vsts/pipelines/1ESPipelines/ci.yml new file mode 100644 index 0000000000..94195443a0 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ci.yml @@ -0,0 +1,269 @@ +parameters: +- name: storageAccountUrl + displayName: SDK storage account URL for production images and testing + type: string + default: https://oryx-cdn.microsoft.io + values: + - https://oryx-cdn.microsoft.io +- name: buildImages + type: object + default: + - key: Latest + value: latest + - key: Jamstack + value: jamstack + - key: GithubActions + value: githubactions + - key: Full + value: full + - key: Cli + value: cli-stretch + - key: CliBuster + value: cli-buster + - key: CliBullseye + value: cli-bullseye + - key: CliBuilderBullseye + value: cli-builder-bullseye + - key: Buildpack + value: buildpack +resources: + repositories: + - repository: MicroBuildTemplate + type: git + name: 1ESPipelineTemplates/MicroBuildTemplate + ref: refs/tags/release +variables: + group: Oryx + Packaging.EnableSBOMSigning: true +trigger: none +extends: + template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate + parameters: + sdl: + sourceAnalysisPool: + name: AzurePipelines-EO + image: AzurePipelinesWindows2022compliantGPT + os: windows + pool: + name: AzurePipelines-EO + image: AzurePipelinesUbuntu20.04compliant + os: linux + customBuildTags: + - ES365AIMigrationTooling-BulkMigrated + stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish artifact file having the release tag name' + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + steps: + - checkout: self + clean: true + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: /vsts/pipelines/templates/_securityChecks.yml@self + - job: Job_SignBinaries + displayName: Sign Oryx Binaries + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + templateContext: + mb: + signing: + enabled: true + signType: '$(setSignTypeVariable.SignType)' + outputs: + - output: pipelineArtifact + displayName: 'Publish Pipeline Artifact' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + artifactName: 'signed_binaries' + targetPath: '$(Build.ArtifactStagingDirectory)' + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' + - powershell: | + $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" + if (Test-Path $file) { + $tagName = [IO.File]::ReadAllText($file) + Write-Host "Content is $tagName" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + } + displayName: 'Set release tag name as environment variable' + - template: /vsts/pipelines/templates/_signBinary.yml@self + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: ${{ buildImage.value }} + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: buster + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: bullseye + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + skipComponentGovernanceDetection: true + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self + parameters: + imageType: bookworm + - template: /vsts/pipelines/templates/_integrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} + - job: Job_ProductionStorageAccountTest + displayName: Test Production Storage Account + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + - task: ShellScript@2 + displayName: 'Test production storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: > + and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + jobs: + - template: /vsts/pipelines/templates/_releaseJobTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/ciDebug.yml b/vsts/pipelines/1ESPipelines/ciDebug.yml new file mode 100644 index 0000000000..8ddcbcd0a2 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/ciDebug.yml @@ -0,0 +1,196 @@ +resources: +- repo: self + +variables: +- group: Oryx + +stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - task: PublishPipelineArtifact@1 + displayName: 'Publish artifact file having the release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + + - job: Job_BuildImage + displayName: Build and Test Build Image + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + timeoutInMinutes: 300 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: succeeded() + + jobs: + - job: Release_BuildImage + displayName: Push Build Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" + displayName: 'Set variables' + + - template: templates/_releaseStepTemplate.yml + + - job: Release_RuntimeImages + displayName: Push Runtime Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 400 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" + displayName: 'Set variables' + + - template: templates/_releaseStepTemplate.yml + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml b/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml new file mode 100644 index 0000000000..01151a1d48 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/copyAllBlobsToProd.yml @@ -0,0 +1,54 @@ +variables: +- group: Oryx + +parameters: +- name: sourceStorageAccount + displayName: Source Storage Account + type: string + default: https://oryxsdkssandbox.blob.core.windows.net + values: + - https://oryxsdkssandbox.blob.core.windows.net + - https://oryxsdksstaging.blob.core.windows.net +- name: dryRun + displayName: Dry Run? + type: boolean + default: true + +trigger: none + +jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Copy all blobs from a source storage account to the prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/copyAllBlobsToProdStorageAccount.sh + args: ${{ parameters.sourceStorageAccount }} ${{ parameters.dryRun }} + + - task: ShellScript@2 + displayName: 'Test Prod storage account' + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml b/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml new file mode 100644 index 0000000000..75de2dbc92 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/copySdksFromProdToStorageAccount.yml @@ -0,0 +1,62 @@ +variables: +- group: Oryx + +parameters: +- name: destinationStorageAccountName + displayName: Destination Storage Account Name + type: string + default: oryxsdksprodbackup +- name: dryRun + displayName: Dry Run? + type: boolean + default: false +# If overwrite is false, the only time SDKs will be copied is when the destination +# account does not have the existing container (e.g. dotnet, nodejs, ruby) +- name: overwrite + displayName: Copy new SDKs to existing containers? + type: boolean + default: true + +jobs: + - job: CopySDKsToStorageAccount + timeoutInMinutes: 300 # sometimes copying a fresh storage account takes longer than the default 60 minute timeout + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Copy SDKs from the prod storage account to a destination storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + PROD_BACKUP_STORAGE_SAS_TOKEN: $(PROD-BACKUP-STORAGE-SAS-TOKEN) + PERSONAL_STORAGE_SAS_TOKEN: $(${{ upper(parameters.destinationStorageAccountName) }}-PERSONAL-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/copySdksFromProdToStorageAccount.sh + args: ${{ parameters.destinationStorageAccountName }} ${{ parameters.dryRun }} ${{ parameters.overwrite }} + +trigger: none +pr: none +schedules: +- cron: "0 12 * * 0" + displayName: Weekly Sunday Copy to Backup Prod + branches: + include: + - main + always: true # whether to always run the pipeline even if there have been source code changes since the last run \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/nightly.yml b/vsts/pipelines/1ESPipelines/nightly.yml new file mode 100644 index 0000000000..cbef3404df --- /dev/null +++ b/vsts/pipelines/1ESPipelines/nightly.yml @@ -0,0 +1,235 @@ +parameters: + - name: storageAccountUrl + displayName: SDK storage account URL for testing + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: buildImages + type: object + default: + - + key: Latest + value: latest + - + key: LtsVersions + value: ltsversions + - + key: Jamstack + value: jamstack + - + key: GithubActions + value: githubactions + - + key: VsoFocal + value: vso-focal + - + key: VsoBullseye + value: vso-bullseye + - + key: Full + value: full + - + key: Cli + value: cli-stretch + - + key: CliBuster + value: cli-buster + - + key: CliBullseye + value: cli-bullseye + - + key: CliBuilderBullseye + value: cli-builder-bullseye + - + key: Buildpack + value: buildpack + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - task: PublishPipelineArtifact@1 + displayName: 'Publish artifact file having the release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + + # Job loop for BuildImages + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: ${{ buildImage.value }} + +# commented out temporarily + # - job: Job_BuilderImages + # displayName: Build Builder Images + # pool: + # name: AzurePipelines-EO + # demands: + # - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # timeoutInMinutes: 480 + # steps: + # - template: templates/_builderTemplate.yml + # dependsOn: Job_BuildImage_CliBuilderBullseye + + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + condition: succeeded() + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + # Building runtime images can take a long time due to our PHP images + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + + - template: templates/_integrationJobTemplate.yml + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} + + - job: Job_StagingStorageAccountTest + displayName: Test Staging Storage Account + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Test staging storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Staging + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/publishSdkToProd.yml b/vsts/pipelines/1ESPipelines/publishSdkToProd.yml new file mode 100644 index 0000000000..a983e52417 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/publishSdkToProd.yml @@ -0,0 +1,75 @@ +variables: +- group: Oryx + +stages: + - stage: DryRun + jobs: + - job: DryRun + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: '(Dry run) Publish SDKs from dev to prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishSdksFromStagingToProdStorageAccount.sh + args: True # dry run + + - stage: PublishSDKsToProduction + jobs: + - deployment: PublishSDKsToProduction + environment: oryx-prod # maps to the oryx-prod ADO environment, which requires manual approval to begin this stage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + strategy: + runOnce: + deploy: + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Publish SDKs from dev to prod storage account' + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + PROD_STORAGE_SAS_TOKEN: $(PROD-STORAGE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishSdksFromStagingToProdStorageAccount.sh + args: False # no dry run + + - task: ShellScript@2 + displayName: 'Test prod storage account' + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml new file mode 100644 index 0000000000..5e01904f8e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_agentCleanUpJobTemplate.yml @@ -0,0 +1,13 @@ +parameters: + agentName: '' + +jobs: +- job: Cleaning_${{ parameters.agentName }} + displayName: ${{ parameters.agentName }} + pool: + name: ${{ parameters.poolName }} + demands: + - agent.name -equals ${{ parameters.agentName }} + timeoutInMinutes: 50 + steps: + - template: _cleanImageCacheTemplate.yml diff --git a/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml new file mode 100644 index 0000000000..000593473b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildTemplate.yml @@ -0,0 +1,211 @@ +parameters: + ascName: oryx-new-service-connection + acrName: oryxdevmcr.azurecr.io + imageName: oryxdevmcr.azurecr.io/public/oryx + imageType: null +steps: +- script: | + if [ "$(BuildBuildImages)" != "true" ] && [ "$(BuildRuntimeImages)" != "true" ] && [ "$(TestIntegration)" != "true" ] + then + echo "Invalid configuration." + echo "Variable 'BuildBuildImages' or 'BuildRuntimeImages' needs to be 'true' to run this build." + exit 1 + fi + displayName: 'Validate pipeline run' +- script: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker images && docker system prune -fa && docker images && echo + displayName: 'clean docker images' + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true')) +- checkout: self + clean: true +- task: DownloadPipelineArtifact@0 + displayName: 'Download Pipeline Artifact' + inputs: + pipelineId: 'Oryx-CI' + artifactName: 'signed_binaries' + targetPath: '$(Build.SourcesDirectory)' + condition: > + and(succeeded(), or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + dotnet --version && dotnet msbuild -version && echo + displayName: 'Print dotnet and msbuild version' +- task: ShellScript@2 + displayName: 'Build Oryx.sln' + inputs: + scriptPath: ./build/buildSln.sh + args: Oryx.sln +- task: Docker@1 + displayName: Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} +- script: | + docker images && docker system prune -fa && df -h && echo + displayName: 'Free up space for oryx layers' +- task: ShellScript@2 + displayName: 'Test build script generator' + inputs: + scriptPath: ./build/testBuildScriptGenerator.sh + condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Test startup script generators' + inputs: + scriptPath: ./build/testStartupScriptGenerators.sh + condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) +- task: ShellScript@2 + displayName: 'Build build images' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/buildBuildImages.sh + args: -t ${{ parameters.imageType }} -s $(storageAccountUrl) + condition: and(succeeded(), eq(variables['BuildBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Build runtime images' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/buildRunTimeImages.sh + args: -s $(storageAccountUrl) ${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['BuildRuntimeImages'], 'true')) +- script: | + docker images && docker system prune -f && df -h && echo + displayName: 'clean docker unknown layers' +- script: | + docker -v + echo "Installing Docker CE version 24.0.5." + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + echo "After installing ." + docker -v + displayName: 'Install Docker 24.0.5' + condition: true +- task: ShellScript@2 + displayName: 'Test build images' + env: + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testBuildImages.sh + args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) ${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) +- task: ShellScript@2 + displayName: 'Test runtime images' + env: + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testRunTimeImages.sh + args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) category=runtime-${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) +- task: DownloadBuildArtifacts@0 + displayName: 'Download Build Artifacts' + inputs: + artifactName: drop + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and Retag recently built oryx build and runtime images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pullAndTag.sh + args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: Shellpp@0 + displayName: 'Build a build image for tests' + inputs: + type: FilePath + scriptPath: ./build/buildTestBuildImages.sh + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: ShellScript@2 + displayName: 'Test integration' + inputs: + scriptPath: ./build/testIntegration.sh + args: $(TestIntegrationCaseFilter) ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) + env: + SQLSERVER_DATABASE_HOST: $(SQLSERVER-DATABASE-HOST) + SQLSERVER_DATABASE_NAME: $(SQLSERVER-DATABASE-NAME) + SQLSERVER_DATABASE_USERNAME: $(SQLSERVER-DATABASE-USERNAME) + SQLSERVER_DATABASE_PASSWORD: $(SQLSERVER-DATABASE-PASSWORD) + ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) +- task: CopyFiles@2 + displayName: 'Copy source projects output to artifacts folder' + inputs: + sourceFolder: $(Build.SourcesDirectory) + contents: src/**/bin/**/*.* + targetFolder: $(Build.ArtifactStagingDirectory) + cleanTargetFolder: true + overWrite: true + flattenFolders: true + condition: true +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true +- task: PublishTestResults@2 + inputs: + testRunner: 'xUnit' + testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' + mergeTestResults: true + condition: true +- task: 1ES.PushContainerImage@1 + displayName: 'Push build images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.ascName }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:${{ parameters.acrName }} +- task: 1ES.PushContainerImage@1 + displayName: 'Push runtime images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + condition: and(succeeded(), eq(variables['PushRuntimeImages'], 'true'), eq(variables['BuildRuntimeImages'], 'true')) +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) +- task: ShellScript@2 + displayName: 'Clean up Docker mounted directories' + inputs: + scriptPath: ./vsts/scripts/removeDockerArtifacts.sh + condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) +- script: | + docker images && docker system prune -fa && df -h && docker images && echo + displayName: 'clean docker images and stopped containers' +- task: ShellScript@2 + displayName: 'Generate release notes' + inputs: + scriptPath: ./vsts/scripts/generateReleaseNotes.sh + condition: and(succeeded(), eq(variables['PushBuildImages'], 'true'), eq(variables['BuildBuildImages'], 'true')) +- task: ArchiveFiles@2 + displayName: 'Archive docker files and scripts for Oryx build and runtime images' + inputs: + rootFolderOrFile: images + archiveFile: '$(Build.ArtifactStagingDirectory)/images/dockerFiles.zip' + condition: true +- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml b/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml new file mode 100644 index 0000000000..bb554f5fbd --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildTemplateDetector.yml @@ -0,0 +1,52 @@ +steps: +- checkout: self + clean: true + +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 + +- task: ShellScript@2 + displayName: 'Build Detector.sln' + inputs: + scriptPath: ./build/buildSln.sh + args: Detector.sln + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- task: ShellScript@2 + displayName: 'Testing Detector' + inputs: + scriptPath: ./build/testDetector.sh + +- task: CopyFiles@2 + displayName: 'Copy content from artifacts folder in repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: succeededOrFailed() + +- task: PublishTestResults@2 + displayName: 'Publish test results' + inputs: + testRunner: 'xUnit' + testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' + mergeTestResults: true + condition: succeededOrFailed() + +- task: PublishBuildArtifacts@1 + displayName: 'Publish build artifacts' + inputs: + pathToPublish: $(Build.ArtifactStagingDirectory) + artifactName: 'detector' + condition: succeededOrFailed() + +- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: succeededOrFailed() \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml new file mode 100644 index 0000000000..0890946854 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_builderTemplate.yml @@ -0,0 +1,41 @@ +parameters: + ascName: oryx-new-service-connection + acrName: oryxdevmcr.azurecr.io +steps: +- script: | + curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.28.0/pack-v0.28.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack + displayName: 'Install Pack CLI' + condition: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} +- task: ShellScript@2 + displayName: 'Build base builder images' + inputs: + scriptPath: ./builders/base/buildBaseBuilder.sh + args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -v $(Build.DefinitionName).$(Build.BuildNumber) +- task: ShellScript@2 + displayName: 'Build container apps builder images' + inputs: + scriptPath: ./builders/container-apps/buildCappsBuilder.sh + args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -t "capps-$(Build.DefinitionName).$(Build.BuildNumber)" -b "${{ parameters.acrName }}/public/oryx/builder:$(Build.DefinitionName).$(Build.BuildNumber)" +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true +- task: 1ES.PushContainerImage@1 + displayName: 'Push base build images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) + remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) +- task: 1ES.PushContainerImage@1 + displayName: 'Push container apps builder images to ACR' + inputs: + image: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt + remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml new file mode 100644 index 0000000000..812bc2ced5 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_buildimageBasesJobTemplate.yml @@ -0,0 +1,40 @@ +jobs: +- job: ${{ parameters.jobName }} + displayName: ${{ parameters.displayName }} + timeoutInMinutes: 250 + templateContext: + outputs: + - output: pipelineArtifact + displayName: 'Publish build artifacts' + targetPath: $(Build.ArtifactStagingDirectory) + steps: + - task: ShellScript@2 + displayName: Build images + inputs: + scriptPath: ${{ parameters.scriptPath }} + args: ${{ parameters.imageDir }} ${{ parameters.imageDebianFlavor }} + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + - task: CopyFiles@2 + displayName: Copy artifacts to staging directory + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true + - task: 1ES.PushContainerImage@1 + displayName: Push built base images to dev ACR + inputs: + image: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' + remoteImage: $(acrname)/$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }} + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + - task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml new file mode 100644 index 0000000000..fe96f9f98e --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_cleanImageCacheTemplate.yml @@ -0,0 +1,18 @@ +steps: +- script: | + echo "List of containers dead/alive ..." + docker ps -a + echo "List of images ..." + docker images + echo "Cleaning containers and images ..." + docker system prune -af + echo "List of containers dead/alive ..." + docker ps -a + echo "List of images ..." + docker images + displayName: 'Remove all existing docker images from machine' + +- script: | + echo "Restarting in 1 minutes" + sudo shutdown -r +1 + displayName: 'Restart machine after cleaning up' diff --git a/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml b/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml new file mode 100644 index 0000000000..030e22230f --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_copyImgFromDevToTest.yml @@ -0,0 +1,246 @@ +parameters: + - name: ascName + type: string + default: oryx-new-service-connection + - name: acrDevName + type: string + default: oryxdevmcr.azurecr.io + - name: acrTestName + type: string + default: oryxtestmcr.azurecr.io + - name: testImageName + type: string + default: oryxtestmcr.azurecr.io/public/oryx + - name: acrDevContainerName + type: string + default: oryxdevmcr.azurecr.io/public/oryx + - name: acrTestContainerName + type: string + default: oryxtestmcr.azurecr.io/public/oryx + - name: nightlyTag + type: string + - name: cliImages + type: object + default: + - name: 'cli-buster' + sourceTag: 'debian-buster-Oryx-Nightly' + destinationTag: 'debian-buster' + - name: 'cli-bullseye' + sourceTag: 'debian-bullseye-Oryx-Nightly' + destinationTag: 'debian-bullseye' + - name: 'cli-stretch' + sourceTag: 'debian-stretch-Oryx-Nightly' + destinationTag: 'debian-stretch' + - name: ltsImages + type: object + default: + - name: 'lts-buster' + sourceTag: 'lts-versions-debian-buster-Oryx-Nightly' + destinationTag: 'lts-versions-debian-buster' + - name: 'lts-stretch' + sourceTag: 'lts-versions-debian-stretch-Oryx-Nightly' + destinationTag: 'lts-versions-debian-stretch' + - name: vsoImages + type: object + default: + - name: 'vso-ubuntu' + sourceTag: 'vso-ubuntu-focal-Oryx-Nightly' + destinationTag: 'vso-ubuntu-focal' + - name: 'vso-bullseye' + sourceTag: 'vso-debian-bullseye-Oryx-Nightly' + destinationTag: 'vso-debian-bullseye' + - name: fullImages + type: object + default: + - name: 'full-buster' + sourceTag: 'full-debian-buster-Oryx-Nightly' + destinationTag: 'full-debian-buster' + - name: 'full-bullseye' + sourceTag: 'full-debian-bullseye-Oryx-Nightly' + destinationTag: 'full-debian-bullseye' + - name: githubActions + type: object + default: + - name: 'github-actions-bookworm' + sourceTag: 'github-actions-debian-bookworm-Oryx-Nightly' + destinationTag: 'github-actions-debian-bookworm' + - name: 'github-actions-bullseye' + sourceTag: 'github-actions-debian-bullseye-Oryx-Nightly' + destinationTag: 'github-actions-debian-bullseye' + - name: 'github-actions-buster' + sourceTag: 'github-actions-debian-buster-Oryx-Nightly' + destinationTag: 'github-actions-debian-buster' + - name: 'github-actions-stretch' + sourceTag: 'github-actions-debian-stretch-Oryx-Nightly' + destinationTag: 'github-actions-debian-stretch' + - name: dotnet + type: object + default: + - name: '3.0-debian-buster' + sourceTag: '3.0-debian-buster-Oryx-Nightly' + destinationTag: '3.0-debian-buster' + - name: '3.1-debian-buster' + sourceTag: '3.1-debian-buster-Oryx-Nightly' + destinationTag: '3.1-debian-buster' + - name: '5.0-debian-buster' + sourceTag: '5.0-debian-buster-Oryx-Nightly' + destinationTag: '5.0-debian-buster' + - name: '6.0-debian-buster' + sourceTag: '6.0-debian-buster-Oryx-Nightly' + destinationTag: '6.0-debian-buster' + - name: '7.0-debian-buster' + sourceTag: '7.0-debian-buster-Oryx-Nightly' + destinationTag: '7.0-debian-buster' + - name: node + type: object + default: + - name: '18-debian-bullseye' + sourceTag: '18-debian-bullseye-Oryx-Nightly' + destinationTag: '18-debian-bullseye' + - name: '16-debian-buster' + sourceTag: '16-debian-buster-Oryx-Nightly' + destinationTag: '16-debian-buster' + - name: '14-debian-buster' + sourceTag: '14-debian-buster-Oryx-Nightly' + destinationTag: '14-debian-buster' + - name: python + type: object + default: + - name: '3.7-debian-bullseye' + sourceTag: '3.7-debian-bullseye-Oryx-Nightly' + destinationTag: '3.7-debian-bullseye' + - name: '3.8-debian-bullseye' + sourceTag: '3.8-debian-bullseye-Oryx-Nightly' + destinationTag: '3.8-debian-bullseye' + - name: '3.9-debian-buster' + sourceTag: '3.9-debian-buster-Oryx-Nightly' + destinationTag: '3.9-debian-buster' + - name: '3.10-debian-bullseye' + sourceTag: '3.10-debian-bullseye-Oryx-Nightly' + destinationTag: '3.10-debian-bullseye' + - name: '3.11-debian-bullseye' + sourceTag: '3.11-debian-bullseye-Oryx-Nightly' + destinationTag: '3.11-debian-bullseye' + + + +steps: + +- checkout: self + clean: true + +- task: Docker@1 + displayName: Dev container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} + +- ${{ each image in parameters.cliImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/cli:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/cli:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/cli:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull CLI Images and Retag' + +- ${{ each image in parameters.ltsImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag lts Images' + +- ${{ each image in parameters.vsoImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Vso Images' + +- ${{ each image in parameters.fullImages }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Full Images' + +- ${{ each image in parameters.githubActions }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/build:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Github Action Images' + +- ${{ each image in parameters.dotnet }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/dotnetcore:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/dotnetcore:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/dotnetcore:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag dotnet Images' + +- ${{ each image in parameters.node }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/node:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/node:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/node:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Node Images' + +- ${{ each image in parameters.python }}: + - script: | + docker pull "${{ parameters.acrDevContainerName }}/python:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" + docker tag "${{ parameters.acrDevContainerName }}/python:${{ image.sourceTag }}.${{ parameters.nightlyTag }}" "${{ parameters.acrTestContainerName }}/python:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Pull and Retag Python Images' + +- task: Docker@1 + displayName: Dev Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} + +- task: Docker@1 + displayName: Test container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrTestName }} + +- ${{ each image in parameters.cliImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/cli:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push CLI Images to Test ACR' + +- ${{ each image in parameters.ltsImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push lts Images to Test ACR' + +- ${{ each image in parameters.vsoImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Vso Images to Test ACR' + +- ${{ each image in parameters.fullImages }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Full Images to Test ACR' + +- ${{ each image in parameters.githubActions }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/build:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Github Action Images to Test ACR' + +- ${{ each image in parameters.dotnet }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/dotnetcore:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push dotnet Images to Test ACR' + +- ${{ each image in parameters.node }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/node:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Node Images to Test ACR' + +- ${{ each image in parameters.python }}: + - script: | + docker push "${{ parameters.acrTestContainerName }}/python:${{ image.destinationTag }}.${{ parameters.nightlyTag }}" + displayName: 'Push Python Images to Test ACR' + +- task: Docker@1 + displayName: Test Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrTestName }} diff --git a/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml new file mode 100644 index 0000000000..7f14fafcb1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_integrationJobTemplate.yml @@ -0,0 +1,17 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +jobs: +- template: /vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} +- template: /vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml@self + parameters: + storageAccountUrl: ${{ parameters.storageAccountUrl }} \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml new file mode 100644 index 0000000000..461a771033 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_platformBinariesReleaseTemplate.yml @@ -0,0 +1,32 @@ +parameters: + destinationSdkStorageAccountName: 'oryxsdksstaging' +steps: +- checkout: self + clean: true +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts' + inputs: + artifactName: drop +- task: ShellScript@2 + displayName: Upload files to Azure Storage + env: + DEV_STORAGE_SAS_TOKEN: $(DEV-STORAGE-SAS-TOKEN) + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + SANDBOX_STORAGE_SAS_TOKEN: $(SANDBOX-STORAGE-SAS-TOKEN) + PERSONAL_STORAGE_SAS_TOKEN: $(${{ upper(parameters.destinationSdkStorageAccountName) }}-PERSONAL-STORAGE-SAS-TOKEN) + PRIVATE_STORAGE_SAS_TOKEN: $(ORYX-SDK-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./vsts/scripts/publishFilesToAzureStorage.sh + args: ${{ parameters.destinationSdkStorageAccountName }} +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 +- task: ShellScript@2 + displayName: 'Test Dev storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Dev \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml new file mode 100644 index 0000000000..294e83bf79 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_platformBinariesTemplate.yml @@ -0,0 +1,27 @@ +parameters: + platformName: '' + debianFlavor: '' + destinationSdkStorageAccountName: '' +steps: +- checkout: self + clean: true +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- task: ShellScript@2 + displayName: 'Building platform binaries' + env: + ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) + DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + inputs: + scriptPath: ./build/buildPlatformBinaries.sh + args: ${{ parameters.platformName }} ${{ parameters.debianFlavor }} https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net +- task: CopyFiles@2 + displayName: 'Copy artifacts from source repo to agent artifacts folder' + inputs: + sourceFolder: '$(Build.SourcesDirectory)/artifacts' + contents: '**/*.*' + targetFolder: $(Build.ArtifactStagingDirectory) + overWrite: true + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml new file mode 100644 index 0000000000..0c231482a6 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesJobTemplate.yml @@ -0,0 +1,61 @@ +jobs: +- job: Release_PhpRuntimeBaseImage + displayName: Push PHP Runtime Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php' + +- job: Release_PhpFpmNginxRuntimeBaseImage + displayName: Push PHP-FPM Runtime Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php-fpm' + +- job: Release_DotNetCoreRuntimeBaseImage + displayName: Push DotNetCore Runtime Base Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 250 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'dotnetcore' + +- job: Release_NodeBaseImage + displayName: Push Node Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 300 + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'node' + +- job: Release_RubyBaseImage + displayName: Push Ruby Base Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 300 + + steps: + - template: _releaseBaseImagesStepTemplate.yml + parameters: + aseImageName: 'ruby' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml new file mode 100644 index 0000000000..6f050f5dd6 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseBaseImagesStepTemplate.yml @@ -0,0 +1,54 @@ +parameters: + ascName: oryx-new-service-connection + acrDevName: oryxdevmcr + acrProdName: oryxmcr + acrPmeProdName: oryxprodmcr + acrPmeProdSrvConnection: Oryx-PME-ACR-Production + baseImageName: '' + baseImageRepository: public/oryx/base +steps: +- checkout: self + clean: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }}.azurecr.io +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts for release' + inputs: + artifactName: drop +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(System.ArtifactsDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' +- task: Shellpp@0 + displayName: 'Pull and create release tags for PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagBaseImagesForRelease.sh + args: '${{ parameters.baseImageName }} ${{ parameters.acrPmeProdName }}' +- task: Docker@2 + displayName: Login to production PME ACR + inputs: + command: login + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} +- task: Shellpp@0 + displayName: 'Push images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushBaseImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.baseImageName }}/${{ parameters.acrPmeProdName }}' +- task: Docker@2 + displayName: Logout from PME ACR + inputs: + command: logout + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml new file mode 100644 index 0000000000..39b0528908 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseJobTemplate.yml @@ -0,0 +1,57 @@ +jobs: +- job: Release_BuildImages + displayName: Push Build Image to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self +- job: Release_RuntimeImages + displayName: Push Runtime Images to MCR + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 400 + steps: + - script: | + echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" + echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self +- job: Release_GitHub + displayName: Create GitHub release + dependsOn: + - Release_BuildImages + - Release_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download Oryx binaries to publish to release' + inputs: + artifactName: 'signed_binaries' + path: $(Build.SourcesDirectory)/artifacts + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - task: GitHubRelease@0 + displayName: 'GitHub release (create)' + inputs: + gitHubConnection: 'Oryx-GitHub' + repositoryName: Microsoft/Oryx + tagSource: manual + tag: '$(RELEASE_TAG_NAME)' + assets: $(Build.SourcesDirectory)/artifacts/oryx_linux_x64.tar.gz \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml b/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml new file mode 100644 index 0000000000..671c653163 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_releaseStepTemplate.yml @@ -0,0 +1,124 @@ +parameters: + ascName: oryx-new-service-connection + acrDevName: oryxdevmcr.azurecr.io + acrProdName: oryxmcr + acrPmeProdName: oryxprodmcr + acrPmeProdSrvConnection: Oryx-PME-ACR-Production + prodImageName: oryxmcr.azurecr.io/public/oryx +steps: +- script: | + if [ "$(ReleaseBuildImages)" != "true" ] && [ "$(ReleaseRuntimeImages)" != "true" ] + then + echo "Invalid configuration." + echo "Variable 'ReleaseBuildImages' or 'ReleaseRuntimeImages' needs to be 'true' to run this" + exit 1 + fi + displayName: 'Validate release pipeline run' +- script: | + docker -v + echo "Installing Docker CE version 24.0.5." + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + echo "After installing ." + docker -v + displayName: 'Install Docker 24.0.5' + condition: true +- script: | + sudo rm -rf /usr/share/dotnet + sudo rm -rf /opt/ghc + sudo rm -rf "/usr/local/share/boost" + sudo rm -rf "$AGENT_TOOLSDIRECTORY" + docker images && docker system prune -fa && docker images && echo + displayName: 'clean docker images' +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + docker -v + displayName: 'Install Docker 24.0.5' +- checkout: self + clean: true +- task: Docker@1 + displayName: Container registry login + inputs: + command: login + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} +- task: DownloadBuildArtifacts@0 + displayName: 'Download Artifacts for release' + inputs: + artifactName: drop +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(System.ArtifactsDirectory)' + TargetFolder: '$(Build.ArtifactStagingDirectory)' +- template: /vsts/pipelines/templates/_setReleaseTag.yml@self +- task: Shellpp@0 + displayName: 'Pull and create release tags for build images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagBuildImagesForRelease.sh + args: '' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and create release tags for runtime images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagRunTimeImagesForRelease.sh + condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) +- task: Shellpp@0 + displayName: 'Pull and create release tags for CLI images' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/tagCliImagesForRelease.sh + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Docker@1 + displayName: Dev Container registry logout + inputs: + command: logout + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrDevName }} +- task: Docker@2 + displayName: Login to PME ACR + inputs: + command: login + containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} +- task: Shellpp@0 + displayName: 'Push build images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-build-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Push CLI image to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-cli-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) +- task: Shellpp@0 + displayName: 'Push runtime images to PME staging ACR' + inputs: + type: FilePath + scriptPath: ./vsts/scripts/pushImagesToRegistry.sh + args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-runtime-images-mcr.txt' + condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) +- task: Docker@2 + displayName: Logout from PME ACR + inputs: + command: logout + containerRegistry: '${{ parameters.acrPmeProdSrvConnection }}' +- task: ShellScript@2 + displayName: 'Clean up Docker containers and images' + inputs: + scriptPath: ./vsts/scripts/cleanDocker.sh + condition: true \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml b/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml new file mode 100644 index 0000000000..1f9d7e63d9 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_securityChecks.yml @@ -0,0 +1,41 @@ +steps: +- checkout: self + clean: true +- task: NuGetToolInstaller@1 + inputs: + versionSpec: 5.8.x +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- script: | + dotnet --version && dotnet msbuild -version && echo + displayName: 'Print dotnet and msbuild version' +- task: NuGetCommand@2 + displayName: 'Run "nuget restore" on Oryx solution' + inputs: + command: 'restore' + restoreSolution: '$(Build.SourcesDirectory)\Oryx.sln' +- task: VSBuild@1 + displayName: 'Build the Oryx solution' + inputs: + solution: '$(Build.SourcesDirectory)\Oryx.sln' + configuration: 'debug' + createLogFile: true + logFileVerbosity: diagnostic +- task: PowerShell@2 + displayName: 'Wait 30 seconds after building solution' + inputs: + targetType: 'inline' + script: 'Start-Sleep -Seconds 30' +- task: AntiMalware@3 + displayName: 'Run Antimalware Scan' + inputs: + InputType: 'Basic' + ScanType: 'CustomScan' + EnableServices: true + TreatSignatureUpdateFailureAs: 'Warning' + SupportLogOnError: false + FileDirPath: '$(Build.StagingDirectory)' + SignatureFreshness: 'UpToDate' + TreatStaleSignatureAs: 'Warning' \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml b/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml new file mode 100644 index 0000000000..91b71548c2 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_setReleaseTag.yml @@ -0,0 +1,18 @@ +steps: +- task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' +- script: | + set -e + file="$(Build.SourcesDirectory)/artifacts/releaseTag/releaseTag.txt" + echo "Release tag name is:" + cat "$file" + tagName="$(Build.BuildNumber)" + if [ -f "$file" ]; then + tagName=`cat $file` + fi + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + displayName: 'Read release tag name from artifact file and set it as environment variable' + name: setReleaseTag \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_signBinary.yml b/vsts/pipelines/1ESPipelines/templates/_signBinary.yml new file mode 100644 index 0000000000..f9d054900d --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_signBinary.yml @@ -0,0 +1,82 @@ +steps: +- powershell: | + Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" + name: setSignTypeVariable + displayName: 'Set sign binary variable' + condition: > + and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), or( + startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) +- script: | + echo $(setSignTypeVariable.SignType) + name: SignType +- task: NuGetToolInstaller@0 + displayName: 'Use NuGet 4.6.2' + inputs: + versionSpec: 4.6.2 + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 +- powershell: | + Write-Host "Setting up git_commit and build_number as env variable" + $env:GIT_COMMIT=$(git rev-parse HEAD) + $env:BUILD_NUMBER=$env:BUILD_BUILDNUMBER + dotnet publish -r linux-x64 -c Release src\BuildScriptGeneratorCLI\BuildScriptGeneratorCli.csproj + dotnet publish -r linux-x64 -c Release src\BuildServer\BuildServer.csproj + displayName: 'dotnet publish and after setting git_commit and build_number as env variable' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: VSBuild@1 + displayName: 'Sign Oryx Binaries' + inputs: + solution: 'src/BuildScriptGeneratorCli/Oryx_sign.signproj' + msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: VSBuild@1 + displayName: 'Sign Oryx Build Server Binaries' + inputs: + solution: 'src/BuildServer/BuildServer.signproj' + msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: ArchiveFiles@2 + displayName: 'Create compressed file of Oryx binaries' + inputs: + rootFolderOrFile: 'src\BuildScriptGeneratorCli\bin\$(BuildConfiguration)\linux-x64\publish' + includeRootFolder: false + archiveType: tar + tarCompression: 'gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/oryx_linux_x64.tar.gz' + replaceExistingArchive: true + verbose: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: ArchiveFiles@2 + displayName: 'Create compressed file of Oryx Build Server binaries' + inputs: + rootFolderOrFile: 'src\BuildServer\bin\$(BuildConfiguration)\linux-x64\publish' + includeRootFolder: false + archiveType: tar + tarCompression: 'gz' + archiveFile: '$(Build.ArtifactStagingDirectory)/oryx_buildServer_linux_x64.tar.gz' + replaceExistingArchive: true + verbose: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: CopyFiles@2 + displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/binaries' + inputs: + SourceFolder: 'src/BuildScriptGeneratorCli/bin/$(BuildConfiguration)/linux-x64/publish' + Contents: '**\*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' + CleanTargetFolder: true + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) +- task: CopyFiles@2 + displayName: 'Copy BuildServer Files to: $(Build.ArtifactStagingDirectory)/binaries' + inputs: + SourceFolder: 'src/BuildServer/bin/$(BuildConfiguration)/linux-x64/publish' + Contents: '**\*' + TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' + CleanTargetFolder: false + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml b/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml new file mode 100644 index 0000000000..368d021982 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/_signBinaryDetector.yml @@ -0,0 +1,71 @@ +steps: +- checkout: self + clean: true + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 + displayName: 'Install Signing Plugin' + inputs: + signType: 'Real' + +- task: UseDotNet@2 + displayName: 'Use .NET Core SDK 7.x' + inputs: + version: 7.0.306 + +- powershell: | + Write-Host "Setting up git_commit and build_number as env variable" + $env:GIT_COMMIT=$(git rev-parse HEAD) + $env:BUILD_NUMBER=$env:BUILD_BUILDNUMBER + $projectFile="src\Detector\Detector.csproj" + + $env:SIGN_PACKAGE="true" + dotnet build Detector.sln -c Release + dotnet restore src\Detector\Detector.signproj + displayName: 'Build Detector.sln' + +- task: VSBuild@1 + displayName: 'Sign Oryx Binaries' + inputs: + solution: 'src/Detector/Detector.signproj' + msbuildArgs: '/t:SignFiles /p:MicroBuild_SigningEnabled=true' + configuration: '$(BuildConfiguration)' + +- powershell: | + Write-Host "Running 'dotnet pack'..." + dotnet pack --no-build --include-symbols -c Release src\Detector\Detector.csproj + displayName: 'Re-package already generated NuGet packages with signed binaries' + +- task: VSBuild@1 + displayName: 'Sign NuGet packages' + inputs: + solution: 'vsts/nugetpackages.signproj' + msbuildArgs: '/t:SignFiles /p:MicroBuild_SigningEnabled=true' + +- task: NuGetToolInstaller@1 + inputs: + versionSpec: '5.6' + +- powershell: | + . $(Build.SourcesDirectory)\build\detector\__detectorNugetPackagesVersions.ps1 + cd "artifacts\packages" + nuget verify -signature ` + -CertificateFingerprint 3F9001EA83C560D712C24CF213C3D312CB3BFF51EE89435D3430BD06B5D0EECE ` + Microsoft.Oryx.Detector.$VERSION.nupkg ` + -verbosity detailed + displayName: 'Verify nuget packages' + +- powershell: | + Copy-Item -Path artifacts\packages -Destination $(Build.ArtifactStagingDirectory)\packages -Recurse + displayName: 'Copying nuget packages to artifact staging directory' + +- task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'detector_signed' + targetPath: '$(Build.ArtifactStagingDirectory)' + \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml new file mode 100644 index 0000000000..78b2aed23b --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -0,0 +1,51 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '6.0' + buildTag: 'debian-buster' + - category: '6.0' + buildTag: 'github-actions-debian-buster' + - category: '7.0' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-bookworm' + - category: 'dynamic' + buildTag: 'debian-buster' + - category: 'dynamic' + buildTag: 'github-actions-debian-buster' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run .NET Core ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml new file mode 100644 index 0000000000..36443745c1 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml @@ -0,0 +1,50 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '1.17' + buildTag: 'full-debian-buster' + - category: '1.17' + buildTag: 'full-debian-bullseye' + - category: '1.18' + buildTag: 'full-debian-buster' + - category: '1.18' + buildTag: 'full-debian-bullseye' + - category: '1.19' + buildTag: 'full-debian-buster' + - category: '1.19' + buildTag: 'full-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Golang ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Full + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml new file mode 100644 index 0000000000..d79da61eae --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -0,0 +1,49 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '14-gh-buster' + buildTag: 'github-actions-debian-buster' + - category: '16' + buildTag: 'debian-buster' + - category: '18' + buildTag: 'github-actions-debian-bullseye' + - category: '20' + buildTag: 'github-actions-debian-bookworm' + - category: '20-bullseye' + buildTag: 'github-actions-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Node ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml new file mode 100644 index 0000000000..871fd35362 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -0,0 +1,49 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '7.4' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-buster' + - category: '8.1' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-bullseye' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run PHP ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml new file mode 100644 index 0000000000..4342f98679 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -0,0 +1,57 @@ +parameters: +- name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net +- name: testMappings + type: object + default: + - category: '3.7' + buildTag: 'github-actions-debian-bullseye' + - category: '3.8' + buildTag: 'github-actions-debian-bullseye' + - category: '3.9' + buildTag: 'debian-buster' + - category: '3.9' + buildTag: 'github-actions-debian-buster' + - category: '3.10' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bookworm' + - category: '3.12' + buildTag: 'github-actions-debian-bullseye' + - category: '3.12' + buildTag: 'github-actions-debian-bookworm' +jobs: +- ${{ each mapping in parameters.testMappings }}: + - job: '' + displayName: 'Run Python ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' + dependsOn: + - Job_BuildImage_Latest + - Job_BuildImage_GithubActions + - Job_Buster_RuntimeImages + - Job_Bullseye_RuntimeImages + - Job_Bookworm_RuntimeImages + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + skipComponentGovernanceDetection: true + timeoutInMinutes: 300 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/testACRImgUpload.yml b/vsts/pipelines/1ESPipelines/testACRImgUpload.yml new file mode 100644 index 0000000000..ff79d03797 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/testACRImgUpload.yml @@ -0,0 +1,28 @@ +parameters: + - name: nightlyTagName + displayName: The tag of image built by nightly pipeline + - name: containerRegistryName + displayName: Container registry name for pushing built image + type: string + default: oryxtestmcr.azurecr.io + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +stages: + - stage: Copy + displayName: Copy Stage + + jobs: + - job: Start_Copy + steps: + - template: templates/_copyImgFromDevToTest.yml + parameters: + nightlyTag: ${{ parameters.nightlyTagName }} + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/validation-detector.yml b/vsts/pipelines/1ESPipelines/validation-detector.yml new file mode 100644 index 0000000000..3cd46a4153 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/validation-detector.yml @@ -0,0 +1,22 @@ +# specific branch +pr: + paths: + exclude: + - /* + include: + - Detector.sln + - src/Detector/* + - tests/Detector.Tests/* + - vsts/nugetpackages.signproj + - vsts/pipelines/validation-detector.yml + +jobs: +- job: Job_Detector + displayName: Detector Validation + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + timeoutInMinutes: 480 + steps: + - template: templates/_buildTemplateDetector.yml \ No newline at end of file diff --git a/vsts/pipelines/1ESPipelines/validation.yml b/vsts/pipelines/1ESPipelines/validation.yml new file mode 100644 index 0000000000..a7afdcede7 --- /dev/null +++ b/vsts/pipelines/1ESPipelines/validation.yml @@ -0,0 +1,171 @@ +parameters: + - name: storageAccountUrl + displayName: SDK storage account URL for testing + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: buildImages + type: object + default: + - + key: Latest + value: latest + - + key: LtsVersions + value: ltsversions + - + key: Jamstack + value: jamstack + - + key: GithubActions + value: githubactions + - + key: VsoFocal + value: vso-focal + - + key: VsoBullseye + value: vso-bullseye + - + key: Full + value: full + - + key: Cli + value: cli-stretch + - + key: CliBuster + value: cli-buster + - + key: CliBullseye + value: cli-bullseye + - + key: CliBuilderBullseye + value: cli-builder-bullseye + - + key: Buildpack + value: buildpack + +resources: +- repo: self + fetchDepth: 15 + +variables: + group: Oryx + Packaging.EnableSBOMSigning: true + +jobs: +- job: Job_Security + displayName: Security + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + +- ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: Build and Test Build ${{ buildImage.key }} Image + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: ${{ buildImage.value }} + +# commented out temporarily +# - job: Job_BuilderImages +# displayName: Build Builder Images +# pool: +# name: AzurePipelines-EO +# demands: +# - ImageOverride -equals AzurePipelinesUbuntu20.04compliant +# timeoutInMinutes: 480 +# steps: +# - template: templates/_builderTemplate.yml +# dependsOn: Job_BuildImage_CliBuilderBullseye + +- job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: buster + +- job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bullseye + +- job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + - group: Oryx + timeoutInMinutes: 480 + steps: + - script: | + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$(Build.BuildNumber)" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: templates/_buildTemplate.yml + parameters: + imageType: bookworm + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/dotnetcore.yml b/vsts/pipelines/PlatformBinaries/dotnetcore.yml index c61bf3d570..87e83d0238 100644 --- a/vsts/pipelines/PlatformBinaries/dotnetcore.yml +++ b/vsts/pipelines/PlatformBinaries/dotnetcore.yml @@ -1,10 +1,95 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: DotNetCore_Bookworm + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'dotnet' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: DotNetCore_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'dotnet' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: DotNetCore_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'dotnet' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: DotNetCore_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'dotnet' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: DotNetCore_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'dotnet' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,106 +100,4 @@ trigger: - /* include: - platforms/dotnet - - vsts/PlatformBinaries/dotnetcore.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: DotNetCore_Bookworm - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'dotnet' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: DotNetCore_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'dotnet' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: DotNetCore_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'dotnet' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: DotNetCore_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'dotnet' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: DotNetCore_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'dotnet' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/dotnetcore.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/golang.yml b/vsts/pipelines/PlatformBinaries/golang.yml index 4fc2046c69..4fc974d8a4 100644 --- a/vsts/pipelines/PlatformBinaries/golang.yml +++ b/vsts/pipelines/PlatformBinaries/golang.yml @@ -1,10 +1,82 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Golang_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'golang' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Golang_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'golang' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Golang_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'golang' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Golang_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'golang' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,92 +87,4 @@ trigger: - /* include: - platforms/golang - - vsts/PlatformBinaries/golang.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Golang_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'golang' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Golang_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'golang' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Golang_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'golang' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Golang_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'golang' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/golang.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/java.yml b/vsts/pipelines/PlatformBinaries/java.yml index cd6513896b..01c262e715 100644 --- a/vsts/pipelines/PlatformBinaries/java.yml +++ b/vsts/pipelines/PlatformBinaries/java.yml @@ -1,10 +1,134 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Java_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'java' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Java_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'java' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Java_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'java' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Java_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'java' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Maven_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'maven' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Maven_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'maven' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Maven_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'maven' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Maven_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'maven' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,148 +139,4 @@ trigger: - /* include: - platforms/java - - vsts/PlatformBinaries/java.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Java_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'java' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Java_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'java' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Java_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'java' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Java_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'java' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Maven_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'maven' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Maven_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'maven' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Maven_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'maven' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Maven_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'maven' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/java.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/node.yml b/vsts/pipelines/PlatformBinaries/node.yml index cc9ec7f03a..906b086d40 100644 --- a/vsts/pipelines/PlatformBinaries/node.yml +++ b/vsts/pipelines/PlatformBinaries/node.yml @@ -1,10 +1,95 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Node_Bookworm + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Node_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'nodejs' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,106 +100,4 @@ trigger: - /* include: - platforms/nodejs - - vsts/PlatformBinaries/node.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Node_Bookworm - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'nodejs' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Node_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'nodejs' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Node_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'nodejs' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Node_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'nodejs' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Node_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'nodejs' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/node.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/php.yml b/vsts/pipelines/PlatformBinaries/php.yml index b6d58465b9..290d45beec 100644 --- a/vsts/pipelines/PlatformBinaries/php.yml +++ b/vsts/pipelines/PlatformBinaries/php.yml @@ -1,10 +1,95 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Php_Bookworm_SDK + timeoutInMinutes: 1440 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'php' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Php_Bullseye_SDK + timeoutInMinutes: 1440 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'php' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Php_Stretch_SDK + timeoutInMinutes: 1440 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'php' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Php_Buster_SDK + timeoutInMinutes: 1440 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'php' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Php_Ubuntu_SDK + timeoutInMinutes: 1440 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'php' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,106 +100,4 @@ trigger: - /* include: - platforms/php - - vsts/PlatformBinaries/php.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Php_Bookworm_SDK - timeoutInMinutes: 1440 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'php' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Php_Bullseye_SDK - timeoutInMinutes: 1440 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'php' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Php_Stretch_SDK - timeoutInMinutes: 1440 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'php' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Php_Buster_SDK - timeoutInMinutes: 1440 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'php' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Php_Ubuntu_SDK - timeoutInMinutes: 1440 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'php' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/php.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/python.yml b/vsts/pipelines/PlatformBinaries/python.yml index 38cfa613b1..48378035cf 100644 --- a/vsts/pipelines/PlatformBinaries/python.yml +++ b/vsts/pipelines/PlatformBinaries/python.yml @@ -1,10 +1,95 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Python_Bookworm_SDK + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'python' + debianFlavor: 'bookworm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Python_Bullseye_SDK + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'python' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Python_Buster_SDK + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'python' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Python_Stretch_SDK + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'python' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Python_Ubuntu_SDK + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'python' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,106 +100,4 @@ trigger: - /* include: - platforms/python - - vsts/PlatformBinaries/python.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Python_Bookworm_SDK - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'python' - debianFlavor: 'bookworm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Python_Bullseye_SDK - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'python' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Python_Buster_SDK - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'python' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Python_Stretch_SDK - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'python' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Python_Ubuntu_SDK - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'python' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/python.yml \ No newline at end of file diff --git a/vsts/pipelines/PlatformBinaries/ruby.yml b/vsts/pipelines/PlatformBinaries/ruby.yml index 3c16e028cd..8405f9d8bb 100644 --- a/vsts/pipelines/PlatformBinaries/ruby.yml +++ b/vsts/pipelines/PlatformBinaries/ruby.yml @@ -1,10 +1,82 @@ variables: -- group: Oryx + - group: Oryx + parameters: - name: destinationStorageAccountName displayName: Destination Storage Account Name type: string default: oryxsdksstaging + +stages: +- stage: Build + jobs: + - job: Ruby_Bullseye + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'ruby' + debianFlavor: 'bullseye' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Ruby_Buster + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'ruby' + debianFlavor: 'buster' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Ruby_Stretch + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'ruby' + debianFlavor: 'stretch' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + + - job: Ruby_Ubuntu + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesTemplate.yml + parameters: + platformName: 'ruby' + debianFlavor: 'focal-scm' + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + +- stage: Release + dependsOn: Build + jobs: + - job: Publish_Platform_Binaries + timeoutInMinutes: 250 + displayName: Publish to Azure Blob Storage + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_platformBinariesReleaseTemplate.yml + parameters: + destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' + trigger: batch: true branches: @@ -15,92 +87,4 @@ trigger: - /* include: - platforms/ruby - - vsts/PlatformBinaries/ruby.yml -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates - parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling - stages: - - stage: Build - jobs: - - job: Ruby_Bullseye - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'ruby' - debianFlavor: 'bullseye' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Ruby_Buster - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'ruby' - debianFlavor: 'buster' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Ruby_Stretch - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'ruby' - debianFlavor: 'stretch' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - job: Ruby_Ubuntu - timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - condition: true - targetPath: $(Build.ArtifactStagingDirectory) - steps: - - template: /vsts/pipelines/templates/_platformBinariesTemplate.yml@self - parameters: - platformName: 'ruby' - debianFlavor: 'focal-scm' - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' - - stage: Release - dependsOn: Build - jobs: - - job: Publish_Platform_Binaries - timeoutInMinutes: 250 - displayName: Publish to Azure Blob Storage - steps: - - template: /vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml@self - parameters: - destinationSdkStorageAccountName: '${{ parameters.destinationStorageAccountName }}' \ No newline at end of file + - vsts/PlatformBinaries/ruby.yml \ No newline at end of file diff --git a/vsts/pipelines/baseImages/dotnetcore.yml b/vsts/pipelines/baseImages/dotnetcore.yml index 0695591eed..e2022cfc13 100644 --- a/vsts/pipelines/baseImages/dotnetcore.yml +++ b/vsts/pipelines/baseImages/dotnetcore.yml @@ -8,60 +8,47 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + +jobs: +- template: ../templates/_buildimageBasesJobTemplate.yml parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: stage - jobs: - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build DotNetCore runtime buster base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: buster - artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt - jobName: Build_DotNetCore_BaseImage_Buster - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build DotNetCore runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: bullseye - artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt - jobName: Build_DotNetCore_BaseImage_Bullseye - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build DotNetCore runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: dotnetcore - imageDebianFlavor: bookworm - artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt - jobName: Build_DotNetCore_BaseImage_Bookworm - - job: Release_DotNetCoreRuntimeBaseImage - dependsOn: - - Build_DotNetCore_BaseImage_Buster - - Build_DotNetCore_BaseImage_Bullseye - - Build_DotNetCore_BaseImage_Bookworm - displayName: Push images to MCR - timeoutInMinutes: 250 - steps: - - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self - parameters: - baseImageName: 'dotnetcore' \ No newline at end of file + displayName: Build DotNetCore runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: buster + artifactsFileName: dotnetcore-runtimeimage-bases-buster.txt + jobName: Build_DotNetCore_BaseImage_Buster + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build DotNetCore runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bullseye + artifactsFileName: dotnetcore-runtimeimage-bases-bullseye.txt + jobName: Build_DotNetCore_BaseImage_Bullseye + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build DotNetCore runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: dotnetcore + imageDebianFlavor: bookworm + artifactsFileName: dotnetcore-runtimeimage-bases-bookworm.txt + jobName: Build_DotNetCore_BaseImage_Bookworm + +- job: Release_DotNetCoreRuntimeBaseImage + dependsOn: + - Build_DotNetCore_BaseImage_Buster + - Build_DotNetCore_BaseImage_Bullseye + - Build_DotNetCore_BaseImage_Bookworm + displayName: Push images to MCR + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'dotnetcore' \ No newline at end of file diff --git a/vsts/pipelines/baseImages/node.yml b/vsts/pipelines/baseImages/node.yml index 7e89a2cd0c..d89ce545ba 100644 --- a/vsts/pipelines/baseImages/node.yml +++ b/vsts/pipelines/baseImages/node.yml @@ -8,58 +8,45 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + +jobs: +- template: ../templates/_buildimageBasesJobTemplate.yml parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: stage - jobs: - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build Node runtime buster based images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: buster - artifactsFileName: node-runtimeimage-bases-buster.txt - jobName: Build_Buster_BaseImages - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build Node runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: bullseye - artifactsFileName: node-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build Node runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: node - imageDebianFlavor: bookworm - artifactsFileName: node-runtimeimage-bases-bookworm.txt - jobName: Build_Bookworm_BaseImages - - job: Release_NodeRuntimeBaseImage - dependsOn: - - Build_Buster_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - steps: - - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self - parameters: - baseImageName: 'node' \ No newline at end of file + displayName: Build Node runtime buster based images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: buster + artifactsFileName: node-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build Node runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bullseye + artifactsFileName: node-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build Node runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: node + imageDebianFlavor: bookworm + artifactsFileName: node-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + +- job: Release_NodeRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'node' diff --git a/vsts/pipelines/baseImages/php-fpm.yml b/vsts/pipelines/baseImages/php-fpm.yml index 95b62a21ca..f2bf16197a 100644 --- a/vsts/pipelines/baseImages/php-fpm.yml +++ b/vsts/pipelines/baseImages/php-fpm.yml @@ -8,60 +8,47 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + +jobs: +- template: ../templates/_buildimageBasesJobTemplate.yml parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: stage - jobs: - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php buster runtime base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php-fpm - imageDebianFlavor: buster - artifactsFileName: php-fpm-runtimeimage-bases-buster.txt - jobName: Build_PHP_FPM_Buster_Base - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php bullseye runtime base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php-fpm - imageDebianFlavor: bullseye - artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt - jobName: Build_PHP_Fpm_Bullseye_Base - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php bookworm runtime base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php-fpm - imageDebianFlavor: bookworm - artifactsFileName: php-fpm-runtimeimage-bases-bookworm.txt - jobName: Build_PHP_Fpm_Bookworm_Base - - job: Release_PhpRuntimeBaseImage - dependsOn: - - Build_Php_Fpm_Buster_Base - - Build_PHP_Fpm_Bullseye_Base - - Build_PHP_Fpm_Bookworm_Base - displayName: Push images to MCR - timeoutInMinutes: 250 - steps: - - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self - parameters: - baseImageName: 'php-fpm' + displayName: Build php buster runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: buster + artifactsFileName: php-fpm-runtimeimage-bases-buster.txt + jobName: Build_PHP_FPM_Buster_Base + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build php bullseye runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bullseye + artifactsFileName: php-fpm-runtimeimage-bases-bullseye.txt + jobName: Build_PHP_Fpm_Bullseye_Base + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build php bookworm runtime base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php-fpm + imageDebianFlavor: bookworm + artifactsFileName: php-fpm-runtimeimage-bases-bookworm.txt + jobName: Build_PHP_Fpm_Bookworm_Base + +- job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Php_Fpm_Buster_Base + - Build_PHP_Fpm_Bullseye_Base + - Build_PHP_Fpm_Bookworm_Base + displayName: Push images to MCR + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php-fpm' diff --git a/vsts/pipelines/baseImages/php.yml b/vsts/pipelines/baseImages/php.yml index 1ca3afd92b..78944bd174 100644 --- a/vsts/pipelines/baseImages/php.yml +++ b/vsts/pipelines/baseImages/php.yml @@ -8,60 +8,47 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + +jobs: +- template: ../templates/_buildimageBasesJobTemplate.yml parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: stage - jobs: - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php runtime buster base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php - imageDebianFlavor: buster - artifactsFileName: php-runtimeimage-bases-buster.txt - jobName: Build_Buster_BaseImages - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php - imageDebianFlavor: bullseye - artifactsFileName: php-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build php runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: php - imageDebianFlavor: bookworm - artifactsFileName: php-runtimeimage-bases-bookworm.txt - jobName: Build_Bookworm_BaseImages - - job: Release_PhpRuntimeBaseImage - dependsOn: - - Build_Buster_BaseImages - - Build_Bullseye_BaseImages - - Build_Bookworm_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - steps: - - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self - parameters: - baseImageName: 'php' + displayName: Build php runtime buster base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: buster + artifactsFileName: php-runtimeimage-bases-buster.txt + jobName: Build_Buster_BaseImages + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build php runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bullseye + artifactsFileName: php-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build php runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: php + imageDebianFlavor: bookworm + artifactsFileName: php-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + +- job: Release_PhpRuntimeBaseImage + dependsOn: + - Build_Buster_BaseImages + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'php' diff --git a/vsts/pipelines/baseImages/python.yml b/vsts/pipelines/baseImages/python.yml index 86a0b8f73e..1d29aba81f 100644 --- a/vsts/pipelines/baseImages/python.yml +++ b/vsts/pipelines/baseImages/python.yml @@ -8,51 +8,37 @@ variables: value: true - name: Packaging.EnableSBOMSigning value: true -resources: - repositories: - - repository: 1ESPipelineTemplates - type: git - name: 1ESPipelineTemplates/1ESPipelineTemplates - ref: refs/tags/release -extends: - template: v1/1ES.Official.PipelineTemplate.yml@1ESPipelineTemplates + +jobs: +- template: ../templates/_buildimageBasesJobTemplate.yml parameters: - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - sdl: - sourceAnalysisPool: - name: AzurePipelines-EO - os: windows - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: stage - jobs: - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build python runtime bullseye base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: python - imageDebianFlavor: bullseye - artifactsFileName: python-runtimeimage-bases-bullseye.txt - jobName: Build_Bullseye_BaseImages - - template: /vsts/pipelines/templates/_buildimageBasesJobTemplate.yml@self - parameters: - displayName: Build python runtime bookworm base images - scriptPath: ./build/buildRunTimeImageBases.sh - imageDir: python - imageDebianFlavor: bookworm - artifactsFileName: python-runtimeimage-bases-bookworm.txt - jobName: Build_Bookworm_BaseImages - - job: Release_PythonRuntimeBaseImage - dependsOn: - - Build_Bullseye_BaseImages - - Build_Bookworm_BaseImages - displayName: Push images to MCR - timeoutInMinutes: 250 - steps: - - template: /vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml@self - parameters: - baseImageName: 'python' \ No newline at end of file + displayName: Build python runtime bullseye base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bullseye + artifactsFileName: python-runtimeimage-bases-bullseye.txt + jobName: Build_Bullseye_BaseImages + +- template: ../templates/_buildimageBasesJobTemplate.yml + parameters: + displayName: Build python runtime bookworm base images + scriptPath: ./build/buildRunTimeImageBases.sh + imageDir: python + imageDebianFlavor: bookworm + artifactsFileName: python-runtimeimage-bases-bookworm.txt + jobName: Build_Bookworm_BaseImages + +- job: Release_PythonRuntimeBaseImage + dependsOn: + - Build_Bullseye_BaseImages + - Build_Bookworm_BaseImages + displayName: Push images to MCR + timeoutInMinutes: 250 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + steps: + - template: ../templates/_releaseBaseImagesStepTemplate.yml + parameters: + baseImageName: 'python' diff --git a/vsts/pipelines/ci.yml b/vsts/pipelines/ci.yml index 94195443a0..fdab6281c9 100644 --- a/vsts/pipelines/ci.yml +++ b/vsts/pipelines/ci.yml @@ -1,269 +1,285 @@ parameters: -- name: storageAccountUrl - displayName: SDK storage account URL for production images and testing - type: string - default: https://oryx-cdn.microsoft.io - values: - - https://oryx-cdn.microsoft.io -- name: buildImages - type: object - default: - - key: Latest - value: latest - - key: Jamstack - value: jamstack - - key: GithubActions - value: githubactions - - key: Full - value: full - - key: Cli - value: cli-stretch - - key: CliBuster - value: cli-buster - - key: CliBullseye - value: cli-bullseye - - key: CliBuilderBullseye - value: cli-builder-bullseye - - key: Buildpack - value: buildpack + - name: storageAccountUrl + displayName: SDK storage account URL for production images and testing + type: string + default: https://oryx-cdn.microsoft.io + values: + - https://oryx-cdn.microsoft.io + - name: buildImages + type: object + default: + - + key: Latest + value: latest + - + key: Jamstack + value: jamstack + - + key: GithubActions + value: githubactions + - + key: Full + value: full + - + key: Cli + value: cli-stretch + - + key: CliBuster + value: cli-buster + - + key: CliBullseye + value: cli-bullseye + - + key: CliBuilderBullseye + value: cli-builder-bullseye + - + key: Buildpack + value: buildpack + resources: - repositories: - - repository: MicroBuildTemplate - type: git - name: 1ESPipelineTemplates/MicroBuildTemplate - ref: refs/tags/release +- repo: self + variables: group: Oryx Packaging.EnableSBOMSigning: true -trigger: none -extends: - template: azure-pipelines/MicroBuild.1ES.Official.yml@MicroBuildTemplate - parameters: - sdl: - sourceAnalysisPool: + +stages: + - stage: CreateReleaseTag + jobs: + - job: CreateReleaseTag + pool: name: AzurePipelines-EO - image: AzurePipelinesWindows2022compliantGPT - os: windows - pool: - name: AzurePipelines-EO - image: AzurePipelinesUbuntu20.04compliant - os: linux - customBuildTags: - - ES365AIMigrationTooling-BulkMigrated - stages: - - stage: CreateReleaseTag - jobs: - - job: CreateReleaseTag - pool: - name: AzurePipelines-EO - demands: + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - skipComponentGovernanceDetection: true - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish artifact file having the release tag name' - targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt - artifactName: releaseTag - steps: - - checkout: self - clean: true - - task: ShellScript@2 - inputs: - scriptPath: ./vsts/scripts/createReleaseTag.sh - displayName: 'Create release tag' - - script: | - set -ex - sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" - mkdir -p "$sourcesArtifactsDir" - echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" - displayName: 'Write release tag name to an artifact file' - - stage: Build - displayName: Build Stage - jobs: - - job: Job_Security - displayName: Security - condition: succeeded() - pool: - name: Azure Pipelines - vmImage: windows-2022 - steps: - - template: /vsts/pipelines/templates/_securityChecks.yml@self - - job: Job_SignBinaries - displayName: Sign Oryx Binaries - pool: - name: VSEngSS-MicroBuild2022-1ES - demands: - - msbuild - - visualstudio - variables: - SignType: 'test' - skipComponentGovernanceDetection: true - templateContext: - mb: - signing: - enabled: true - signType: '$(setSignTypeVariable.SignType)' - outputs: - - output: pipelineArtifact - displayName: 'Publish Pipeline Artifact' - condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) - artifactName: 'signed_binaries' - targetPath: '$(Build.ArtifactStagingDirectory)' - steps: - - task: DownloadPipelineArtifact@2 - displayName: 'Download artifact file having release tag name' - inputs: - targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag - artifactName: 'releaseTag' - - powershell: | - $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" - if (Test-Path $file) { - $tagName = [IO.File]::ReadAllText($file) - Write-Host "Content is $tagName" - echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" - } - displayName: 'Set release tag name as environment variable' - - template: /vsts/pipelines/templates/_signBinary.yml@self - - ${{ each buildImage in parameters.buildImages }}: - - job: Job_BuildImage_${{ buildImage.key }} - displayName: 'Build & Test ${{ buildImage.key }} Build image' - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] - skipComponentGovernanceDetection: true - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]true" - echo "##vso[task.setvariable variable=TestBuildImages;]true" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=PushBuildImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then - echo "##vso[task.setvariable variable=PushBuilderImages;]true" - fi - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self - parameters: - imageType: ${{ buildImage.value }} - - job: Job_Buster_RuntimeImages - displayName: Build and Test Buster Runtime Images - dependsOn: Job_SignBinaries + variables: + skipComponentGovernanceDetection: true + steps: + - checkout: self + clean: true + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - task: ShellScript@2 + inputs: + scriptPath: ./vsts/scripts/createReleaseTag.sh + displayName: 'Create release tag' + - script: | + set -ex + sourcesArtifactsDir="$(Build.SourcesDirectory)/artifacts" + mkdir -p "$sourcesArtifactsDir" + echo "$(RELEASE_TAG_NAME)" > "$sourcesArtifactsDir/releaseTag.txt" + displayName: 'Write release tag name to an artifact file' + - task: PublishPipelineArtifact@1 + displayName: 'Publish artifact file having the release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag.txt + artifactName: releaseTag + + - stage: Build + displayName: Build Stage + jobs: + - job: Job_Security + displayName: Security + condition: succeeded() + pool: + name: Azure Pipelines + vmImage: windows-2022 + steps: + - template: templates/_securityChecks.yml + + - job: Job_SignBinaries + displayName: Sign Oryx Binaries + pool: + name: VSEngSS-MicroBuild2022-1ES + demands: + - msbuild + - visualstudio + variables: + SignType: 'test' + skipComponentGovernanceDetection: true + steps: + - task: DownloadPipelineArtifact@2 + displayName: 'Download artifact file having release tag name' + inputs: + targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag + artifactName: 'releaseTag' + - powershell: | + $file = "$env:BUILD_SOURCESDIRECTORY\artifacts\releaseTag\releaseTag.txt" + if (Test-Path $file) { + $tagName = [IO.File]::ReadAllText($file) + Write-Host "Content is $tagName" + echo "##vso[task.setvariable variable=RELEASE_TAG_NAME;]$tagName" + } + displayName: 'Set release tag name as environment variable' + - template: templates/_signBinary.yml + + # Job loop for BuildImages + - ${{ each buildImage in parameters.buildImages }}: + - job: Job_BuildImage_${{ buildImage.key }} + displayName: 'Build & Test ${{ buildImage.key }} Build image' + #dependsOn: Job_SignBinaries condition: succeeded() timeoutInMinutes: 480 pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable skipComponentGovernanceDetection: true + steps: - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=BuildBuildImages;]true" + echo "##vso[task.setvariable variable=TestBuildImages;]true" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=PushBuildImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + if [[ "${{ buildImage.value }}" =~ "cli-builder" ]]; then + echo "##vso[task.setvariable variable=PushBuilderImages;]true" + fi displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml parameters: + imageType: ${{ buildImage.value }} + + - job: Job_Buster_RuntimeImages + displayName: Build and Test Buster Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: imageType: buster - - job: Job_Bullseye_RuntimeImages - displayName: Build and Test Bullseye Runtime Images - dependsOn: Job_SignBinaries - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: + + - job: Job_Bullseye_RuntimeImages + displayName: Build and Test Bullseye Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] - skipComponentGovernanceDetection: true - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self - parameters: + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml + parameters: imageType: bullseye - - job: Job_Bookworm_RuntimeImages - displayName: Build and Test Bookworm Runtime Images - dependsOn: Job_SignBinaries - condition: succeeded() - timeoutInMinutes: 480 - pool: - name: AzurePipelines-EO - demands: + + - job: Job_Bookworm_RuntimeImages + displayName: Build and Test Bookworm Runtime Images + dependsOn: Job_SignBinaries + condition: succeeded() + timeoutInMinutes: 480 + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] - skipComponentGovernanceDetection: true - steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" - echo "##vso[task.setvariable variable=TestRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushRuntimeImages;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self - parameters: - imageType: bookworm - - template: /vsts/pipelines/templates/_integrationJobTemplate.yml@self + variables: + SignType: $[ dependencies.Job_SignBinaries.outputs['setSignTypeVariable.SignType'] ] # map in the signtype variable + skipComponentGovernanceDetection: true + + steps: + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]true" + echo "##vso[task.setvariable variable=TestRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushRuntimeImages;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]true" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + + - template: templates/_setReleaseTag.yml + + - template: templates/_buildTemplate.yml parameters: + imageType: bookworm + + - template: templates/_integrationJobTemplate.yml + parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} - - job: Job_ProductionStorageAccountTest - displayName: Test Production Storage Account - pool: - name: AzurePipelines-EO - demands: + + - job: Job_ProductionStorageAccountTest + displayName: Test Production Storage Account + pool: + name: AzurePipelines-EO + demands: - ImageOverride -equals AzurePipelinesUbuntu20.04compliant - variables: - skipComponentGovernanceDetection: true - steps: - - task: UseDotNet@2 - displayName: 'Use .NET Core sdk 7.x' - inputs: - version: 7.0.306 - - task: ShellScript@2 - displayName: 'Test production storage account' - env: - ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} - inputs: - scriptPath: ./build/testIntegration.sh - args: StorageAccountTests=Prod - - stage: Release - displayName: Release Stage - dependsOn: Build - condition: > - and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) - jobs: - - template: /vsts/pipelines/templates/_releaseJobTemplate.yml@self \ No newline at end of file + variables: + skipComponentGovernanceDetection: true + steps: + - task: UseDotNet@2 + displayName: 'Use .NET Core sdk 7.x' + inputs: + version: 7.0.306 + + - task: ShellScript@2 + displayName: 'Test production storage account' + env: + ORYX_TEST_SDK_STORAGE_URL: ${{ parameters.storageAccountUrl }} + inputs: + scriptPath: ./build/testIntegration.sh + args: StorageAccountTests=Prod + + - stage: Release + displayName: Release Stage + dependsOn: Build + condition: > + and(succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + + jobs: + - template: templates/_releaseJobTemplate.yml + +trigger: none \ No newline at end of file diff --git a/vsts/pipelines/templates/_buildTemplate.yml b/vsts/pipelines/templates/_buildTemplate.yml index 000593473b..db47fbd943 100644 --- a/vsts/pipelines/templates/_buildTemplate.yml +++ b/vsts/pipelines/templates/_buildTemplate.yml @@ -3,6 +3,7 @@ parameters: acrName: oryxdevmcr.azurecr.io imageName: oryxdevmcr.azurecr.io/public/oryx imageType: null + steps: - script: | if [ "$(BuildBuildImages)" != "true" ] && [ "$(BuildRuntimeImages)" != "true" ] && [ "$(TestIntegration)" != "true" ] @@ -12,6 +13,7 @@ steps: exit 1 fi displayName: 'Validate pipeline run' + - script: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc @@ -20,8 +22,10 @@ steps: docker images && docker system prune -fa && docker images && echo displayName: 'clean docker images' condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true')) + - checkout: self clean: true + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact' inputs: @@ -29,19 +33,32 @@ steps: artifactName: 'signed_binaries' targetPath: '$(Build.SourcesDirectory)' condition: > - and(succeeded(), or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + and(succeeded(), + or(in(variables['SIGNTYPE'], 'real', 'Real'), in(variables['SignType'], 'real', 'Real')), + or(startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'],'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/'))) + - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 + - script: | dotnet --version && dotnet msbuild -version && echo displayName: 'Print dotnet and msbuild version' + - task: ShellScript@2 displayName: 'Build Oryx.sln' inputs: scriptPath: ./build/buildSln.sh args: Oryx.sln + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - task: Docker@1 displayName: Container registry logout inputs: @@ -55,19 +72,23 @@ steps: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrName }} + - script: | docker images && docker system prune -fa && df -h && echo displayName: 'Free up space for oryx layers' + - task: ShellScript@2 displayName: 'Test build script generator' inputs: scriptPath: ./build/testBuildScriptGenerator.sh condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) + - task: ShellScript@2 displayName: 'Test startup script generators' inputs: scriptPath: ./build/testStartupScriptGenerators.sh condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) + - task: ShellScript@2 displayName: 'Build build images' env: @@ -76,6 +97,7 @@ steps: scriptPath: ./build/buildBuildImages.sh args: -t ${{ parameters.imageType }} -s $(storageAccountUrl) condition: and(succeeded(), eq(variables['BuildBuildImages'], 'true')) + - task: ShellScript@2 displayName: 'Build runtime images' env: @@ -84,20 +106,24 @@ steps: scriptPath: ./build/buildRunTimeImages.sh args: -s $(storageAccountUrl) ${{ parameters.imageType }} condition: and(succeeded(), eq(variables['BuildRuntimeImages'], 'true')) + + - script: | docker images && docker system prune -f && df -h && echo displayName: 'clean docker unknown layers' + - script: | docker -v - echo "Installing Docker CE version 24.0.5." + echo "Installing Docker CE version 24.0.5................" curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt update sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - echo "After installing ." + echo "After installing ................" docker -v displayName: 'Install Docker 24.0.5' condition: true + - task: ShellScript@2 displayName: 'Test build images' env: @@ -107,6 +133,7 @@ steps: scriptPath: ./build/testBuildImages.sh args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) ${{ parameters.imageType }} condition: and(succeeded(), eq(variables['TestBuildImages'], 'true')) + - task: ShellScript@2 displayName: 'Test runtime images' env: @@ -116,11 +143,13 @@ steps: scriptPath: ./build/testRunTimeImages.sh args: skipBuildingImages ${{ parameters.imageName }} $(Build.DefinitionName).$(RELEASE_TAG_NAME) category=runtime-${{ parameters.imageType }} condition: and(succeeded(), eq(variables['TestRuntimeImages'], 'true')) + - task: DownloadBuildArtifacts@0 displayName: 'Download Build Artifacts' inputs: artifactName: drop condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) + - task: Shellpp@0 displayName: 'Pull and Retag recently built oryx build and runtime images' inputs: @@ -128,12 +157,14 @@ steps: scriptPath: ./vsts/scripts/pullAndTag.sh args: $(System.ArtifactsDirectory)/drop/images/build-images-acr.txt condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) + - task: Shellpp@0 displayName: 'Build a build image for tests' inputs: type: FilePath scriptPath: ./build/buildTestBuildImages.sh condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) + - task: ShellScript@2 displayName: 'Test integration' inputs: @@ -147,6 +178,7 @@ steps: ORYX_TEST_SDK_STORAGE_URL: $(storageAccountUrl) ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) condition: and(succeeded(), eq(variables['TestIntegration'], 'true')) + - task: CopyFiles@2 displayName: 'Copy source projects output to artifacts folder' inputs: @@ -157,6 +189,7 @@ steps: overWrite: true flattenFolders: true condition: true + - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -165,47 +198,77 @@ steps: targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true condition: true + +- task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: Generate Software Bill of Materials (SBOM) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)' + AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' + - task: PublishTestResults@2 inputs: testRunner: 'xUnit' testResultsFiles: '$(Build.ArtifactStagingDirectory)/testResults/*.xml' mergeTestResults: true condition: true -- task: 1ES.PushContainerImage@1 + +- task: Docker@1 displayName: 'Push build images to ACR' inputs: - image: ${{ parameters.acrName }}/${{ parameters.ascName }} - remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:${{ parameters.acrName }} -- task: 1ES.PushContainerImage@1 + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + command: 'Push an image' + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/build-images-acr.txt' + includeLatestTag: false + enforceDockerNamingConvention: false + condition: and(succeeded(), or(eq(variables['PushBuildImages'], 'true'), eq(variables['PushBuilderImages'], 'true')), eq(variables['BuildBuildImages'], 'true')) + +- task: Docker@1 displayName: 'Push runtime images to ACR' inputs: - image: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} - remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:${{ parameters.imageType }} + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + command: 'Push an image' + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/runtime-images-acr.${{ parameters.imageType }}.txt' + includeLatestTag: false + enforceDockerNamingConvention: false condition: and(succeeded(), eq(variables['PushRuntimeImages'], 'true'), eq(variables['BuildRuntimeImages'], 'true')) + - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: scriptPath: ./vsts/scripts/cleanDocker.sh condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) + - task: ShellScript@2 displayName: 'Clean up Docker mounted directories' inputs: scriptPath: ./vsts/scripts/removeDockerArtifacts.sh condition: or(eq(variables['TestBuildImages'], 'true'), eq(variables['TestRuntimeImages'], 'true'), eq(variables['TestIntegration'], 'true')) + - script: | docker images && docker system prune -fa && df -h && docker images && echo displayName: 'clean docker images and stopped containers' + - task: ShellScript@2 displayName: 'Generate release notes' inputs: scriptPath: ./vsts/scripts/generateReleaseNotes.sh condition: and(succeeded(), eq(variables['PushBuildImages'], 'true'), eq(variables['BuildBuildImages'], 'true')) + - task: ArchiveFiles@2 displayName: 'Archive docker files and scripts for Oryx build and runtime images' inputs: rootFolderOrFile: images archiveFile: '$(Build.ArtifactStagingDirectory)/images/dockerFiles.zip' condition: true + +- task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: drop' + condition: true + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' - condition: true \ No newline at end of file + condition: true diff --git a/vsts/pipelines/templates/_builderTemplate.yml b/vsts/pipelines/templates/_builderTemplate.yml index 0890946854..57af09174d 100644 --- a/vsts/pipelines/templates/_builderTemplate.yml +++ b/vsts/pipelines/templates/_builderTemplate.yml @@ -1,27 +1,32 @@ parameters: ascName: oryx-new-service-connection acrName: oryxdevmcr.azurecr.io + steps: - script: | curl -sSL "https://github.com/buildpacks/pack/releases/download/v0.28.0/pack-v0.28.0-linux.tgz" | tar -C /usr/local/bin/ --no-same-owner -xzv pack displayName: 'Install Pack CLI' condition: true + - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrName }} + - task: ShellScript@2 displayName: 'Build base builder images' inputs: scriptPath: ./builders/base/buildBaseBuilder.sh args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -v $(Build.DefinitionName).$(Build.BuildNumber) + - task: ShellScript@2 displayName: 'Build container apps builder images' inputs: scriptPath: ./builders/container-apps/buildCappsBuilder.sh args: -f ${{ parameters.acrName }} -r "public/oryx/builder" -t "capps-$(Build.DefinitionName).$(Build.BuildNumber)" -b "${{ parameters.acrName }}/public/oryx/builder:$(Build.DefinitionName).$(Build.BuildNumber)" + - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -29,13 +34,25 @@ steps: contents: '**/*.*' targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true -- task: 1ES.PushContainerImage@1 + +- task: Docker@1 displayName: 'Push base build images to ACR' inputs: - image: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) - remoteImage: ${{ parameters.acrName }}/${{ parameters.acrName }}:$(Build.BuildNumber) -- task: 1ES.PushContainerImage@1 + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + command: 'Push an image' + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/builder-images-acr.txt' + includeLatestTag: false + enforceDockerNamingConvention: false + +- task: Docker@1 displayName: 'Push container apps builder images to ACR' inputs: - image: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt - remoteImage: ${{ parameters.acrName }}/${{ parameters.ascName }}:$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt \ No newline at end of file + azureSubscriptionEndpoint: ${{ parameters.ascName }} + azureContainerRegistry: ${{ parameters.acrName }} + command: 'Push an image' + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/capps-builder-images-acr.txt' + includeLatestTag: false + enforceDockerNamingConvention: false \ No newline at end of file diff --git a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml index 812bc2ced5..701537907a 100644 --- a/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml +++ b/vsts/pipelines/templates/_buildimageBasesJobTemplate.yml @@ -1,13 +1,25 @@ +parameters: + displayName: '' + imageDir: '' + imageDebianFlavor: '' + scriptPath: '' + artifactsFileName: '' + jobName: '' + jobs: - job: ${{ parameters.jobName }} displayName: ${{ parameters.displayName }} + pool: + name: AzurePipelines-EO + demands: + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant timeoutInMinutes: 250 - templateContext: - outputs: - - output: pipelineArtifact - displayName: 'Publish build artifacts' - targetPath: $(Build.ArtifactStagingDirectory) - steps: + steps: + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - task: ShellScript@2 displayName: Build images inputs: @@ -16,6 +28,13 @@ jobs: env: ORYX_SDK_STORAGE_ACCOUNT_ACCESS_TOKEN: $(ORYX-SDK-STAGING-PRIVATE-SAS-TOKEN) DOTNET_PRIVATE_STORAGE_ACCOUNT_ACCESS_TOKEN: $(DotnetPrivateStorageAccountAccessToken) + + - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 + displayName: Generate Software Bill of Materials (SBOM) + inputs: + BuildDropPath: '$(Build.ArtifactStagingDirectory)' + AdditionalComponentDetectorArgs: '--DirectoryExclusionList **/SampleApps/**' + - task: CopyFiles@2 displayName: Copy artifacts to staging directory inputs: @@ -24,17 +43,29 @@ jobs: targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true condition: true - - task: 1ES.PushContainerImage@1 + + - task: Docker@1 displayName: Push built base images to dev ACR inputs: - image: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' - remoteImage: $(acrname)/$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }} + command: push + azureSubscriptionEndpoint: $(ascName) + azureContainerRegistry: $(acrName) + pushMultipleImages: true + imageNamesPath: '$(Build.ArtifactStagingDirectory)/images/${{ parameters.artifactsFileName }}' + enforceDockerNamingConvention: false + - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: scriptPath: ./vsts/scripts/cleanDocker.sh + + - task: PublishBuildArtifacts@1 + displayName: Publish build artifacts + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: scriptPath: ./vsts/scripts/cleanDocker.sh - condition: true \ No newline at end of file + condition: true diff --git a/vsts/pipelines/templates/_integrationJobTemplate.yml b/vsts/pipelines/templates/_integrationJobTemplate.yml index 7f14fafcb1..6d3edee76c 100644 --- a/vsts/pipelines/templates/_integrationJobTemplate.yml +++ b/vsts/pipelines/templates/_integrationJobTemplate.yml @@ -1,17 +1,25 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + jobs: -- template: /vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml@self +# Python integration tests +- template: integrationTests/_pythonIntegrationJobTemplate.yml parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} -- template: /vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml@self + +# DotNetCore integration tests +- template: integrationTests/_dotnetcoreIntegrationJobTemplate.yml parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} -- template: /vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml@self + +# NodeJS integration tests +- template: integrationTests/_nodeIntegrationJobTemplate.yml parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} -- template: /vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml@self + +# PHP integration tests +- template: integrationTests/_phpIntegrationJobTemplate.yml parameters: storageAccountUrl: ${{ parameters.storageAccountUrl }} \ No newline at end of file diff --git a/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml b/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml index 461a771033..df85c79267 100644 --- a/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml +++ b/vsts/pipelines/templates/_platformBinariesReleaseTemplate.yml @@ -1,12 +1,21 @@ parameters: destinationSdkStorageAccountName: 'oryxsdksstaging' + steps: + - checkout: self clean: true + - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts' inputs: artifactName: drop + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - task: ShellScript@2 displayName: Upload files to Azure Storage env: @@ -18,10 +27,12 @@ steps: inputs: scriptPath: ./vsts/scripts/publishFilesToAzureStorage.sh args: ${{ parameters.destinationSdkStorageAccountName }} + - task: UseDotNet@2 displayName: 'Use .NET Core SDK 7.x' inputs: version: 7.0.306 + - task: ShellScript@2 displayName: 'Test Dev storage account' env: diff --git a/vsts/pipelines/templates/_platformBinariesTemplate.yml b/vsts/pipelines/templates/_platformBinariesTemplate.yml index 294e83bf79..4ae719caff 100644 --- a/vsts/pipelines/templates/_platformBinariesTemplate.yml +++ b/vsts/pipelines/templates/_platformBinariesTemplate.yml @@ -1,14 +1,22 @@ parameters: - platformName: '' - debianFlavor: '' - destinationSdkStorageAccountName: '' + platformName: '' + debianFlavor: '' + destinationSdkStorageAccountName: '' + steps: - checkout: self clean: true + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 + - task: ShellScript@2 displayName: 'Building platform binaries' env: @@ -17,6 +25,7 @@ steps: inputs: scriptPath: ./build/buildPlatformBinaries.sh args: ${{ parameters.platformName }} ${{ parameters.debianFlavor }} https://${{ parameters.destinationSdkStorageAccountName }}.blob.core.windows.net + - task: CopyFiles@2 displayName: 'Copy artifacts from source repo to agent artifacts folder' inputs: @@ -24,4 +33,10 @@ steps: contents: '**/*.*' targetFolder: $(Build.ArtifactStagingDirectory) overWrite: true - condition: true \ No newline at end of file + condition: true + +- task: PublishBuildArtifacts@1 + displayName: Publish build artifacts + inputs: + pathtoPublish: $(Build.ArtifactStagingDirectory) + condition: true diff --git a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml index 6f050f5dd6..145a846779 100644 --- a/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseBaseImagesStepTemplate.yml @@ -4,49 +4,64 @@ parameters: acrProdName: oryxmcr acrPmeProdName: oryxprodmcr acrPmeProdSrvConnection: Oryx-PME-ACR-Production - baseImageName: '' + baseImageName: '' # defaults baseImageRepository: public/oryx/base + steps: - checkout: self clean: true + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }}.azurecr.io + - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts for release' inputs: artifactName: drop + - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(System.ArtifactsDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' + - task: Shellpp@0 displayName: 'Pull and create release tags for PME staging ACR' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBaseImagesForRelease.sh args: '${{ parameters.baseImageName }} ${{ parameters.acrPmeProdName }}' + - task: Docker@2 displayName: Login to production PME ACR inputs: command: login containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} -- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} + +- script: echo base image is ${{ parameters.baseImageName }} and repository is ${{ parameters.baseImageRepository }} + - task: Shellpp@0 displayName: 'Push images to PME staging ACR' inputs: type: FilePath scriptPath: ./vsts/scripts/pushBaseImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.baseImageName }}/${{ parameters.acrPmeProdName }}' + - task: Docker@2 displayName: Logout from PME ACR inputs: command: logout containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} + - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: diff --git a/vsts/pipelines/templates/_releaseJobTemplate.yml b/vsts/pipelines/templates/_releaseJobTemplate.yml index 39b0528908..4064b0cd6e 100644 --- a/vsts/pipelines/templates/_releaseJobTemplate.yml +++ b/vsts/pipelines/templates/_releaseJobTemplate.yml @@ -4,7 +4,7 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 @@ -13,13 +13,15 @@ jobs: echo "##vso[task.setvariable variable=ReleaseBuildImages;]true" echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]false" displayName: 'Set variables' - - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self + + - template: _releaseStepTemplate.yml + - job: Release_RuntimeImages displayName: Push Runtime Images to MCR pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 400 @@ -28,25 +30,35 @@ jobs: echo "##vso[task.setvariable variable=ReleaseBuildImages;]false" echo "##vso[task.setvariable variable=ReleaseRuntimeImages;]true" displayName: 'Set variables' - - template: /vsts/pipelines/templates/_releaseStepTemplate.yml@self + + - template: _releaseStepTemplate.yml + - job: Release_GitHub displayName: Create GitHub release - dependsOn: - - Release_BuildImages - - Release_RuntimeImages + dependsOn: + - Release_BuildImages + - Release_RuntimeImages pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true + steps: - task: DownloadPipelineArtifact@2 displayName: 'Download Oryx binaries to publish to release' inputs: artifactName: 'signed_binaries' path: $(Build.SourcesDirectory)/artifacts - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self + + - task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + + - template: _setReleaseTag.yml + - task: GitHubRelease@0 displayName: 'GitHub release (create)' inputs: diff --git a/vsts/pipelines/templates/_releaseStepTemplate.yml b/vsts/pipelines/templates/_releaseStepTemplate.yml index 671c653163..5288df3fe1 100644 --- a/vsts/pipelines/templates/_releaseStepTemplate.yml +++ b/vsts/pipelines/templates/_releaseStepTemplate.yml @@ -5,7 +5,14 @@ parameters: acrPmeProdName: oryxprodmcr acrPmeProdSrvConnection: Oryx-PME-ACR-Production prodImageName: oryxmcr.azurecr.io/public/oryx + steps: + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - script: | if [ "$(ReleaseBuildImages)" != "true" ] && [ "$(ReleaseRuntimeImages)" != "true" ] then @@ -14,17 +21,19 @@ steps: exit 1 fi displayName: 'Validate release pipeline run' + - script: | docker -v - echo "Installing Docker CE version 24.0.5." + echo "Installing Docker CE version 24.0.5................" curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null sudo apt update sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - echo "After installing ." + echo "After installing ................" docker -v displayName: 'Install Docker 24.0.5' condition: true + - script: | sudo rm -rf /usr/share/dotnet sudo rm -rf /opt/ghc @@ -32,65 +41,78 @@ steps: sudo rm -rf "$AGENT_TOOLSDIRECTORY" docker images && docker system prune -fa && docker images && echo displayName: 'clean docker images' -- task: UseDotNet@2 - displayName: 'Use .NET Core sdk 7.x' - inputs: - version: 7.0.306 + - script: | - curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg - echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null - sudo apt update - sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io - docker -v + curl -fsSL https://download.docker.com/linux/ubuntu/gpg | sudo gpg --dearmor -o /usr/share/keyrings/docker-archive-keyring.gpg + echo "deb [arch=amd64 signed-by=/usr/share/keyrings/docker-archive-keyring.gpg] https://download.docker.com/linux/ubuntu $(lsb_release -cs) stable" | sudo tee /etc/apt/sources.list.d/docker.list > /dev/null + sudo apt update + sudo apt-get install docker-ce=5:24.0.5~3-0~ubuntu-focal docker-ce-cli=5:24.0.5~3-0~ubuntu-focal containerd.io + docker -v displayName: 'Install Docker 24.0.5' + - checkout: self clean: true + - task: Docker@1 displayName: Container registry login inputs: command: login azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }} + - task: DownloadBuildArtifacts@0 displayName: 'Download Artifacts for release' inputs: artifactName: drop + - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)' inputs: SourceFolder: '$(System.ArtifactsDirectory)' TargetFolder: '$(Build.ArtifactStagingDirectory)' -- template: /vsts/pipelines/templates/_setReleaseTag.yml@self + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- template: _setReleaseTag.yml + - task: Shellpp@0 displayName: 'Pull and create release tags for build images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagBuildImagesForRelease.sh - args: '' + args: condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) + - task: Shellpp@0 displayName: 'Pull and create release tags for runtime images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagRunTimeImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) + - task: Shellpp@0 displayName: 'Pull and create release tags for CLI images' inputs: type: FilePath scriptPath: ./vsts/scripts/tagCliImagesForRelease.sh condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) + - task: Docker@1 displayName: Dev Container registry logout inputs: command: logout azureSubscriptionEndpoint: ${{ parameters.ascName }} azureContainerRegistry: ${{ parameters.acrDevName }} + - task: Docker@2 displayName: Login to PME ACR - inputs: + inputs: command: login containerRegistry: ${{ parameters.acrPmeProdSrvConnection }} + - task: Shellpp@0 displayName: 'Push build images to PME staging ACR' inputs: @@ -98,6 +120,7 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-build-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) + - task: Shellpp@0 displayName: 'Push CLI image to PME staging ACR' inputs: @@ -105,6 +128,7 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-cli-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseBuildImages'], 'true')) + - task: Shellpp@0 displayName: 'Push runtime images to PME staging ACR' inputs: @@ -112,11 +136,13 @@ steps: scriptPath: ./vsts/scripts/pushImagesToRegistry.sh args: '$(Build.ArtifactStagingDirectory)/drop/images/${{ parameters.acrPmeProdName }}-runtime-images-mcr.txt' condition: and(succeeded(), eq(variables['ReleaseRuntimeImages'], 'true')) + - task: Docker@2 displayName: Logout from PME ACR - inputs: + inputs: command: logout containerRegistry: '${{ parameters.acrPmeProdSrvConnection }}' + - task: ShellScript@2 displayName: 'Clean up Docker containers and images' inputs: diff --git a/vsts/pipelines/templates/_securityChecks.yml b/vsts/pipelines/templates/_securityChecks.yml index 1f9d7e63d9..e5ab02583a 100644 --- a/vsts/pipelines/templates/_securityChecks.yml +++ b/vsts/pipelines/templates/_securityChecks.yml @@ -1,21 +1,38 @@ steps: - checkout: self clean: true + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-policheck.PoliCheck@1 + displayName: 'Run PoliCheck' + inputs: + targetType: F + condition: always() + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-credscan.CredScan@2 + displayName: 'Run CredScan' + inputs: + debugMode: false + condition: always() + - task: NuGetToolInstaller@1 inputs: versionSpec: 5.8.x + - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 + - script: | dotnet --version && dotnet msbuild -version && echo displayName: 'Print dotnet and msbuild version' + - task: NuGetCommand@2 displayName: 'Run "nuget restore" on Oryx solution' inputs: command: 'restore' restoreSolution: '$(Build.SourcesDirectory)\Oryx.sln' + - task: VSBuild@1 displayName: 'Build the Oryx solution' inputs: @@ -23,11 +40,37 @@ steps: configuration: 'debug' createLogFile: true logFileVerbosity: diagnostic + - task: PowerShell@2 displayName: 'Wait 30 seconds after building solution' inputs: targetType: 'inline' script: 'Start-Sleep -Seconds 30' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-roslynanalyzers.RoslynAnalyzers@2 + displayName: 'Run Roslyn Analyzers' + condition: always() + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-binskim.BinSkim@3 + displayName: 'Run BinSkim ' + inputs: + arguments: 'analyze Microsoft.Oryx*.dll --recurse --output $(Build.BinariesDirectory)\binskim.sarif --verbose' + condition: always() + +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + +- task: securedevelopmentteam.vss-secure-development-tools.build-task-postanalysis.PostAnalysis@1 + displayName: 'Post Analysis' + inputs: + BinSkim: true + CredScan: true + PoliCheck: true + RoslynAnalyzers: true + continueOnError: false + - task: AntiMalware@3 displayName: 'Run Antimalware Scan' inputs: diff --git a/vsts/pipelines/templates/_setReleaseTag.yml b/vsts/pipelines/templates/_setReleaseTag.yml index 91b71548c2..e08e805b14 100644 --- a/vsts/pipelines/templates/_setReleaseTag.yml +++ b/vsts/pipelines/templates/_setReleaseTag.yml @@ -4,6 +4,7 @@ steps: inputs: targetPath: $(Build.SourcesDirectory)/artifacts/releaseTag artifactName: 'releaseTag' + - script: | set -e file="$(Build.SourcesDirectory)/artifacts/releaseTag/releaseTag.txt" diff --git a/vsts/pipelines/templates/_signBinary.yml b/vsts/pipelines/templates/_signBinary.yml index f9d054900d..bd27bef72e 100644 --- a/vsts/pipelines/templates/_signBinary.yml +++ b/vsts/pipelines/templates/_signBinary.yml @@ -1,25 +1,41 @@ steps: +- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0 + displayName: 'Component Detection - OSS Compliance' + inputs: + ignoreDirectories: '$(Build.SourcesDirectory)/tests' + - powershell: | - Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" + Write-Host "##vso[task.setvariable variable=SignType;isOutput=true]real" name: setSignTypeVariable displayName: 'Set sign binary variable' condition: > - and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), or( - startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), - startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), - startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) + and(in(variables['Build.Reason'], 'Schedule', 'Manual'), eq(variables['Build.DefinitionName'], 'Oryx-CI'), + or( + startsWith(variables['Build.SourceBranch'], 'refs/heads/main'), + startsWith(variables['Build.SourceBranch'], 'refs/heads/patch/'), + startsWith(variables['Build.SourceBranch'],'refs/heads/exp/' ))) + - script: | echo $(setSignTypeVariable.SignType) name: SignType + - task: NuGetToolInstaller@0 displayName: 'Use NuGet 4.6.2' inputs: versionSpec: 4.6.2 condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + +- task: ms-vseng.MicroBuildTasks.30666190-6959-11e5-9f96-f56098202fef.MicroBuildSigningPlugin@1 + displayName: 'Install Signing Plugin' + inputs: + signType: '$(setSignTypeVariable.SignType)' + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: UseDotNet@2 displayName: 'Use .NET Core sdk 7.x' inputs: version: 7.0.306 + - powershell: | Write-Host "Setting up git_commit and build_number as env variable" $env:GIT_COMMIT=$(git rev-parse HEAD) @@ -28,6 +44,7 @@ steps: dotnet publish -r linux-x64 -c Release src\BuildServer\BuildServer.csproj displayName: 'dotnet publish and after setting git_commit and build_number as env variable' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: VSBuild@1 displayName: 'Sign Oryx Binaries' inputs: @@ -35,6 +52,7 @@ steps: msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' configuration: '$(BuildConfiguration)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: VSBuild@1 displayName: 'Sign Oryx Build Server Binaries' inputs: @@ -42,6 +60,7 @@ steps: msbuildArgs: '/t:SignFiles /p:RuntimeIdentifier=linux-x64 /p:MicroBuild_SigningEnabled=true' configuration: '$(BuildConfiguration)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: ArchiveFiles@2 displayName: 'Create compressed file of Oryx binaries' inputs: @@ -53,6 +72,7 @@ steps: replaceExistingArchive: true verbose: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: ArchiveFiles@2 displayName: 'Create compressed file of Oryx Build Server binaries' inputs: @@ -64,6 +84,7 @@ steps: replaceExistingArchive: true verbose: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: CopyFiles@2 displayName: 'Copy Files to: $(Build.ArtifactStagingDirectory)/binaries' inputs: @@ -72,6 +93,7 @@ steps: TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' CleanTargetFolder: true condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + - task: CopyFiles@2 displayName: 'Copy BuildServer Files to: $(Build.ArtifactStagingDirectory)/binaries' inputs: @@ -79,4 +101,11 @@ steps: Contents: '**\*' TargetFolder: '$(Build.ArtifactStagingDirectory)/binaries' CleanTargetFolder: false + condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) + +- task: PublishPipelineArtifact@0 + displayName: 'Publish Pipeline Artifact' + inputs: + artifactName: 'signed_binaries' + targetPath: '$(Build.ArtifactStagingDirectory)' condition: and(succeeded(), eq(variables['setSignTypeVariable.SignType'], 'real')) \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml index 78b2aed23b..55a82b60bd 100644 --- a/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_dotnetcoreIntegrationJobTemplate.yml @@ -1,25 +1,26 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net -- name: testMappings - type: object - default: - - category: '6.0' - buildTag: 'debian-buster' - - category: '6.0' - buildTag: 'github-actions-debian-buster' - - category: '7.0' - buildTag: 'github-actions-debian-buster' - - category: '8.0' - buildTag: 'github-actions-debian-bookworm' - - category: 'dynamic' - buildTag: 'debian-buster' - - category: 'dynamic' - buildTag: 'github-actions-debian-buster' + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: testMappings + type: object + default: + - category: '6.0' + buildTag: 'debian-buster' + - category: '6.0' + buildTag: 'github-actions-debian-buster' + - category: '7.0' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-bookworm' + - category: 'dynamic' + buildTag: 'debian-buster' + - category: 'dynamic' + buildTag: 'github-actions-debian-buster' + jobs: - ${{ each mapping in parameters.testMappings }}: - - job: '' + - job: displayName: 'Run .NET Core ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -30,22 +31,22 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=dotnetcore-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: ../_setReleaseTag.yml + - template: ../_buildTemplate.yml \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml index 36443745c1..2c620f7126 100644 --- a/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_golangIntegrationJobTemplate.yml @@ -1,25 +1,26 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net -- name: testMappings - type: object - default: - - category: '1.17' - buildTag: 'full-debian-buster' - - category: '1.17' - buildTag: 'full-debian-bullseye' - - category: '1.18' - buildTag: 'full-debian-buster' - - category: '1.18' - buildTag: 'full-debian-bullseye' - - category: '1.19' - buildTag: 'full-debian-buster' - - category: '1.19' - buildTag: 'full-debian-bullseye' + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: testMappings + type: object + default: + - category: '1.17' + buildTag: 'full-debian-buster' + - category: '1.17' + buildTag: 'full-debian-bullseye' + - category: '1.18' + buildTag: 'full-debian-buster' + - category: '1.18' + buildTag: 'full-debian-bullseye' + - category: '1.19' + buildTag: 'full-debian-buster' + - category: '1.19' + buildTag: 'full-debian-bullseye' + jobs: - ${{ each mapping in parameters.testMappings }}: - - job: '' + - job: displayName: 'Run Golang ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Full @@ -29,22 +30,22 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=golang-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: ../_setReleaseTag.yml + - template: ../_buildTemplate.yml \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml index d79da61eae..cc99344a3d 100644 --- a/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_nodeIntegrationJobTemplate.yml @@ -1,23 +1,24 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net -- name: testMappings - type: object - default: - - category: '14-gh-buster' - buildTag: 'github-actions-debian-buster' - - category: '16' - buildTag: 'debian-buster' - - category: '18' - buildTag: 'github-actions-debian-bullseye' - - category: '20' - buildTag: 'github-actions-debian-bookworm' - - category: '20-bullseye' - buildTag: 'github-actions-debian-bullseye' + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: testMappings + type: object + default: + - category: '14-gh-buster' + buildTag: 'github-actions-debian-buster' + - category: '16' + buildTag: 'debian-buster' + - category: '18' + buildTag: 'github-actions-debian-bullseye' + - category: '20' + buildTag: 'github-actions-debian-bookworm' + - category: '20-bullseye' + buildTag: 'github-actions-debian-bullseye' + jobs: - ${{ each mapping in parameters.testMappings }}: - - job: '' + - job: displayName: 'Run Node ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -28,22 +29,22 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=node-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: ../_setReleaseTag.yml + - template: ../_buildTemplate.yml \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml index 871fd35362..4b0c85605f 100644 --- a/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_phpIntegrationJobTemplate.yml @@ -1,23 +1,34 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net -- name: testMappings - type: object - default: - - category: '7.4' - buildTag: 'github-actions-debian-buster' - - category: '8.0' - buildTag: 'github-actions-debian-buster' - - category: '8.1' - buildTag: 'github-actions-debian-buster' - - category: '8.2' - buildTag: 'github-actions-debian-buster' - - category: '8.2' - buildTag: 'github-actions-debian-bullseye' + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: testMappings + type: object + default: + - category: '7.4' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-buster' + - category: '8.0' + buildTag: 'github-actions-debian-bullseye' + - category: '8.1' + buildTag: 'github-actions-debian-buster' + - category: '8.1' + buildTag: 'github-actions-debian-bullseye' + - category: '8.2' + buildTag: 'github-actions-debian-buster' + - category: '8.2' + buildTag: 'github-actions-debian-bullseye' + - category: '8.3' + buildTag: 'github-actions-debian-buster' + - category: '8.3' + buildTag: 'github-actions-debian-bullseye' + - category: '8.3' + buildTag: 'github-actions-debian-bookworm' + jobs: - ${{ each mapping in parameters.testMappings }}: - - job: '' + - job: displayName: 'Run PHP ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -28,22 +39,22 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=php-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: ../_setReleaseTag.yml + - template: ../_buildTemplate.yml \ No newline at end of file diff --git a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml index 4342f98679..4361135a71 100644 --- a/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml +++ b/vsts/pipelines/templates/integrationTests/_pythonIntegrationJobTemplate.yml @@ -1,31 +1,32 @@ parameters: -- name: storageAccountUrl - type: string - default: https://oryxsdksstaging.blob.core.windows.net -- name: testMappings - type: object - default: - - category: '3.7' - buildTag: 'github-actions-debian-bullseye' - - category: '3.8' - buildTag: 'github-actions-debian-bullseye' - - category: '3.9' - buildTag: 'debian-buster' - - category: '3.9' - buildTag: 'github-actions-debian-buster' - - category: '3.10' - buildTag: 'github-actions-debian-bullseye' - - category: '3.11' - buildTag: 'github-actions-debian-bullseye' - - category: '3.11' - buildTag: 'github-actions-debian-bookworm' - - category: '3.12' - buildTag: 'github-actions-debian-bullseye' - - category: '3.12' - buildTag: 'github-actions-debian-bookworm' + - name: storageAccountUrl + type: string + default: https://oryxsdksstaging.blob.core.windows.net + - name: testMappings + type: object + default: + - category: '3.7' + buildTag: 'github-actions-debian-bullseye' + - category: '3.8' + buildTag: 'github-actions-debian-bullseye' + - category: '3.9' + buildTag: 'debian-buster' + - category: '3.9' + buildTag: 'github-actions-debian-buster' + - category: '3.10' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bullseye' + - category: '3.11' + buildTag: 'github-actions-debian-bookworm' + - category: '3.12' + buildTag: 'github-actions-debian-bullseye' + - category: '3.12' + buildTag: 'github-actions-debian-bookworm' + jobs: - ${{ each mapping in parameters.testMappings }}: - - job: '' + - job: displayName: 'Run Python ${{ mapping.category }} Integration Tests using build image tag ${{ mapping.buildTag }}' dependsOn: - Job_BuildImage_Latest @@ -36,22 +37,22 @@ jobs: pool: name: AzurePipelines-EO demands: - - ImageOverride -equals AzurePipelinesUbuntu20.04compliant + - ImageOverride -equals AzurePipelinesUbuntu20.04compliant variables: skipComponentGovernanceDetection: true timeoutInMinutes: 300 steps: - - script: | - echo "##vso[task.setvariable variable=BuildBuildImages;]false" - echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestBuildImages;]false" - echo "##vso[task.setvariable variable=TestRuntimeImages;]false" - echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" - echo "##vso[task.setvariable variable=TestIntegration;]true" - echo "##vso[task.setvariable variable=PushBuildImages;]false" - echo "##vso[task.setvariable variable=PushRuntimeImages;]false" - echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" - echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" - displayName: 'Set variables' - - template: /vsts/pipelines/templates/_setReleaseTag.yml@self - - template: /vsts/pipelines/templates/_buildTemplate.yml@self \ No newline at end of file + - script: | + echo "##vso[task.setvariable variable=BuildBuildImages;]false" + echo "##vso[task.setvariable variable=BuildRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestBuildImages;]false" + echo "##vso[task.setvariable variable=TestRuntimeImages;]false" + echo "##vso[task.setvariable variable=TestIntegrationCaseFilter;]category=python-${{ mapping.category }}&build-image=${{ mapping.buildTag }}" + echo "##vso[task.setvariable variable=TestIntegration;]true" + echo "##vso[task.setvariable variable=PushBuildImages;]false" + echo "##vso[task.setvariable variable=PushRuntimeImages;]false" + echo "##vso[task.setvariable variable=EmbedBuildContextInImages;]false" + echo "##vso[task.setvariable variable=storageAccountUrl;]${{ parameters.storageAccountUrl }}" + displayName: 'Set variables' + - template: ../_setReleaseTag.yml + - template: ../_buildTemplate.yml \ No newline at end of file