From 766d459216add535e94ee83c89e282007323c729 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 10 Jul 2024 10:35:40 +0800 Subject: [PATCH 1/8] Init --- azure-pipelines.yml | 46 +++++++++++++++++++++++---------------------- 1 file changed, 24 insertions(+), 22 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 0b26aa5cc11..5dc0bed4688 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -403,7 +403,7 @@ jobs: - job: BuildDockerImageAzureLinux displayName: Build Docker Image Azure Linux - dependsOn: BuildRpmPackageMariner + dependsOn: BuildRpmPackageAzureLinux strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -766,19 +766,20 @@ jobs: displayName: 'Test Homebrew Package' -- job: BuildRpmPackageMariner +- job: BuildRpmPackageAzureLinux displayName: Build Rpm Package Mariner condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: - 2.0 AMD64: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-amd64 - pool: ${{ variables.ubuntu_pool }} - 2.0 ARM64: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-arm64 - pool: ${{ variables.ubuntu_arm64_pool }} + ${{ each arch in parameters.architectures }}: + Mariner 2.0 ${{ arch.name }}: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0-${{ arch.value }} + pool: ${{ arch.pool }} + Azure Linux 3.0 ${{ arch.name }}: + image: azurelinuxpreview.azurecr.io/public/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} pool: name: $(pool) steps: @@ -802,25 +803,26 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifact) -- job: TestRpmPackageMariner +- job: TestRpmPackageAzureLinux displayName: Test Rpm Package Mariner timeoutInMinutes: 120 dependsOn: - - BuildRpmPackageMariner + - BuildRpmPackageAzureLinux - ExtractMetadata condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: name: $(pool) strategy: matrix: - 2.0 AMD64: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-amd64 - pool: ${{ variables.ubuntu_pool }} - 2.0 ARM64: - image: mcr.microsoft.com/cbl-mariner/base/core:2.0 - artifact: rpm-mariner2.0-arm64 - pool: ${{ variables.ubuntu_arm64_pool }} + ${{ each arch in parameters.architectures }}: + Mariner 2.0 ${{ arch.name }}: + image: mcr.microsoft.com/cbl-mariner/base/core:2.0 + artifact: rpm-mariner2.0-${{ arch.value }} + pool: ${{ arch.pool }} + Azure Linux 3.0 ${{ arch.name }}: + image: azurelinuxpreview.azurecr.io/public/azurelinux/base/core:3.0 + artifact: rpm-azurelinux3.0-${{ arch.value }} + pool: ${{ arch.pool }} steps: - task: DownloadPipelineArtifact@1 displayName: 'Download Metadata' @@ -841,7 +843,7 @@ jobs: set -ex CLI_VERSION=`cat $SYSTEM_ARTIFACTSDIRECTORY/metadata/version` - RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.cm2.*.rpm" -printf '%f\n') + RPM_NAME=$(find $SYSTEM_ARTIFACTSDIRECTORY/rpm/ -type f -name "azure-cli-$CLI_VERSION-1.*.rpm" -printf '%f\n') echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE @@ -1253,7 +1255,7 @@ jobs: - BuildHomebrewFormula - TestHomebrewFormula - TestHomebrewPackage - - BuildRpmPackageMariner + - BuildRpmPackageAzureLinux - BuildRpmPackages - TestRpmPackage - BuildDebPackages From fcde981cf45067a223f942a6a84193912a563c31 Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Wed, 10 Jul 2024 11:33:27 +0800 Subject: [PATCH 2/8] Minor fix --- scripts/release/rpm/mariner.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/mariner.dockerfile index 3de26923a20..736434660fd 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/mariner.dockerfile @@ -21,8 +21,8 @@ COPY . . RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ - cp /usr/src/mariner/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ - mkdir /out && cp /usr/src/mariner/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ + cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /azure-cli-dev.rpm && \ + mkdir /out && cp /usr/src/*/RPMS/*/azure-cli-${cli_version}-1.*.rpm /out/ FROM ${image} AS execution-env From 6abf9133896702692d0de755348b650b97caf53b Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Fri, 2 Aug 2024 14:50:17 +0800 Subject: [PATCH 3/8] Minor fix --- azure-pipelines.yml | 2 +- scripts/ci/build.sh | 1 + scripts/release/rpm/build.sh | 13 ------------- scripts/release/rpm/test_mariner_in_docker.sh | 2 +- scripts/release/rpm/test_rpm_in_docker.sh | 4 ++-- 5 files changed, 5 insertions(+), 17 deletions(-) delete mode 100755 scripts/release/rpm/build.sh diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5dc0bed4688..df88f9f6258 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -804,7 +804,7 @@ jobs: ArtifactName: $(artifact) - job: TestRpmPackageAzureLinux - displayName: Test Rpm Package Mariner + displayName: Test Rpm Package timeoutInMinutes: 120 dependsOn: - BuildRpmPackageAzureLinux diff --git a/scripts/ci/build.sh b/scripts/ci/build.sh index 671b5d62ec7..706992fe4a5 100755 --- a/scripts/ci/build.sh +++ b/scripts/ci/build.sh @@ -102,6 +102,7 @@ CLASSIFIERS = [ 'Programming Language :: Python :: 3.9', 'Programming Language :: Python :: 3.10', 'Programming Language :: Python :: 3.11', + 'Programming Language :: Python :: 3.12', 'License :: OSI Approved :: MIT License', ] diff --git a/scripts/release/rpm/build.sh b/scripts/release/rpm/build.sh deleted file mode 100755 index d59e9664b9c..00000000000 --- a/scripts/release/rpm/build.sh +++ /dev/null @@ -1,13 +0,0 @@ -#!/bin/bash - -: "${CLI_VERSION:?CLI_VERSION environment variable not set.}" - -yum check-update -yum install -y gcc rpm-build rpm-level rpmlint make bash corutils diffutils \ - path rpmdevtools python libffi-devel python3-devel openssl-devel \ - wget - -set -ev - -export REPO_PATH=`cd $(dirname $0); cd ../../../; pwd` -rpmbuild -v -bb --clean $REPO_PATH/scripts/release/rpm/azure-cli.spec && cp /root/rpmbuild/RPMS/x86_64/* /mnt/output diff --git a/scripts/release/rpm/test_mariner_in_docker.sh b/scripts/release/rpm/test_mariner_in_docker.sh index f979f3052ba..3f9bce55552 100644 --- a/scripts/release/rpm/test_mariner_in_docker.sh +++ b/scripts/release/rpm/test_mariner_in_docker.sh @@ -14,7 +14,7 @@ time az self-test time az --version cd /azure-cli/ -pip install wheel +python -m pip install --upgrade pip setuptools ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. diff --git a/scripts/release/rpm/test_rpm_in_docker.sh b/scripts/release/rpm/test_rpm_in_docker.sh index 5f1d21d8423..be4cc9a4e66 100644 --- a/scripts/release/rpm/test_rpm_in_docker.sh +++ b/scripts/release/rpm/test_rpm_in_docker.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This script should be run in a centos7 docker. +# This script should be run in a ubi8, ubi9 docker. set -exv export USERNAME=azureuser @@ -15,7 +15,7 @@ time az self-test time az --version cd /azure-cli/ -pip install wheel +python -m pip install --upgrade pip setuptools ./scripts/ci/build.sh # From Fedora36, when using `pip install --prefix` with root privileges, the package is installed into `{prefix}/local/lib`. From 7dcbc9a9b5ea808fbb95d2a5a53755cf46af87bb Mon Sep 17 00:00:00 2001 From: Hang Lei Date: Fri, 2 Aug 2024 15:08:47 +0800 Subject: [PATCH 4/8] Minor fix --- azure-pipelines.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index df88f9f6258..504a0eea565 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -767,7 +767,7 @@ jobs: - job: BuildRpmPackageAzureLinux - displayName: Build Rpm Package Mariner + displayName: Build Rpm Package condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: matrix: From 794348c1a43a93a93ca07c1596a810300f5a0101 Mon Sep 17 00:00:00 2001 From: Hang Date: Mon, 12 Aug 2024 16:09:16 +0800 Subject: [PATCH 5/8] Rename to azurelinux --- azure-pipelines.yml | 16 ++++++++-------- ...ariner.dockerfile => azurelinux.dockerfile} | 4 ++-- ...eline_mariner.sh => pipeline_azurelinux.sh} | 18 +++++++++--------- ..._docker.sh => test_azurelinux_in_docker.sh} | 0 4 files changed, 19 insertions(+), 19 deletions(-) rename scripts/release/rpm/{mariner.dockerfile => azurelinux.dockerfile} (84%) rename scripts/release/rpm/{pipeline_mariner.sh => pipeline_azurelinux.sh} (68%) rename scripts/release/rpm/{test_mariner_in_docker.sh => test_azurelinux_in_docker.sh} (100%) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 504a0eea565..922b0ac1f1d 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -403,7 +403,7 @@ jobs: - job: BuildDockerImageAzureLinux displayName: Build Docker Image Azure Linux - dependsOn: BuildRpmPackageAzureLinux + dependsOn: BuildRpmPackagesAzureLinux strategy: matrix: ${{ each arch in parameters.architectures }}: @@ -766,7 +766,7 @@ jobs: displayName: 'Test Homebrew Package' -- job: BuildRpmPackageAzureLinux +- job: BuildRpmPackagesAzureLinux displayName: Build Rpm Package condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) strategy: @@ -787,10 +787,10 @@ jobs: displayName: Install Docker - task: Bash@3 - displayName: 'Build Rpm Package: Mariner' + displayName: 'Build Rpm Package: Azure Linux' inputs: targetType: 'filePath' - filePath: scripts/release/rpm/pipeline_mariner.sh + filePath: scripts/release/rpm/pipeline_azurelinux.sh - task: AzureArtifacts.manifest-generator-task.manifest-generator-task.ManifestGeneratorTask@0 displayName: 'SBOM' @@ -803,11 +803,11 @@ jobs: TargetPath: $(Build.ArtifactStagingDirectory) ArtifactName: $(artifact) -- job: TestRpmPackageAzureLinux +- job: TestRpmPackagesAzureLinux displayName: Test Rpm Package timeoutInMinutes: 120 dependsOn: - - BuildRpmPackageAzureLinux + - BuildRpmPackagesAzureLinux - ExtractMetadata condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI', 'Manual', 'Schedule')) pool: @@ -847,7 +847,7 @@ jobs: echo "== Test rpm package on ${IMAGE} ==" docker pull $IMAGE - docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_mariner_in_docker.sh" + docker run --rm -e RPM_NAME=$RPM_NAME -v $SYSTEM_ARTIFACTSDIRECTORY/rpm:/mnt/rpm -v $(pwd):/azure-cli $IMAGE /bin/bash "/azure-cli/scripts/release/rpm/test_azurelinux_in_docker.sh" displayName: 'Test Rpm Package Mariner' @@ -1255,7 +1255,7 @@ jobs: - BuildHomebrewFormula - TestHomebrewFormula - TestHomebrewPackage - - BuildRpmPackageAzureLinux + - BuildRpmPackagesAzureLinux - BuildRpmPackages - TestRpmPackage - BuildDebPackages diff --git a/scripts/release/rpm/mariner.dockerfile b/scripts/release/rpm/azurelinux.dockerfile similarity index 84% rename from scripts/release/rpm/mariner.dockerfile rename to scripts/release/rpm/azurelinux.dockerfile index 736434660fd..5f36aa0ad48 100644 --- a/scripts/release/rpm/mariner.dockerfile +++ b/scripts/release/rpm/azurelinux.dockerfile @@ -16,8 +16,8 @@ WORKDIR /azure-cli COPY . . -# Mariner 2.0 only has 'python3' package, which is currently (2022-12-09) Python 3.9. -# It has no version-specific packages like 'python39'. +# Mariner 2.0's default Python is 3.9, the rpm path is /usr/src/mariner/RPMS/x86_64/azure-cli-2.63.0-1.cm2.x86_64.rpm +# Azure Linux 3's default Python is 3.12, the rpm path is /usr/src/azl/RPMS/x86_64/azure-cli-2.62.0-1.azl3.x86_64.rpm RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ diff --git a/scripts/release/rpm/pipeline_mariner.sh b/scripts/release/rpm/pipeline_azurelinux.sh similarity index 68% rename from scripts/release/rpm/pipeline_mariner.sh rename to scripts/release/rpm/pipeline_azurelinux.sh index c7274a3f5cb..8a89780d0e8 100644 --- a/scripts/release/rpm/pipeline_mariner.sh +++ b/scripts/release/rpm/pipeline_azurelinux.sh @@ -1,12 +1,12 @@ #!/usr/bin/env bash -# Build assets related to Linux Distributions that use RPM/yum for installation specific for CBL-Mariner. +# Build assets related to Linux Distributions that use RPM/yum for installation specific for Azure Linux set -exv : "${BUILD_STAGINGDIRECTORY:?BUILD_STAGINGDIRECTORY environment variable not set.}" -# IMAGE should be Mariner docker image url, such as mcr.microsoft.com/cbl-mariner/base/core:2.0 +# IMAGE should be Azure Linux docker image url, such as mcr.microsoft.com/cbl-mariner/base/core:2.0 : "${IMAGE:?IMAGE environment variable not set.}" CLI_VERSION=`cat src/azure-cli/azure/cli/__main__.py | grep __version__ | sed s/' '//g | sed s/'__version__='// | sed s/\"//g` @@ -16,21 +16,21 @@ docker build \ --target build-env \ --build-arg cli_version=${CLI_VERSION} \ --build-arg image=${IMAGE} \ - -f ./scripts/release/rpm/mariner.dockerfile \ - -t azure/azure-cli:mariner-builder \ + -f ./scripts/release/rpm/azurelinux.dockerfile \ + -t azure/azure-cli:azurelinux-builder \ . # Continue the previous build, and create a container that has the current azure-cli build but not the source code. docker build \ --build-arg cli_version=${CLI_VERSION} \ --build-arg image=${IMAGE} \ - -f ./scripts/release/rpm/mariner.dockerfile \ - -t azure/azure-cli:mariner \ + -f ./scripts/release/rpm/azurelinux.dockerfile \ + -t azure/azure-cli:azurelinux \ . # Extract the built RPM so that it can be distributed independently. # The RPM file looks like azure-cli-2.32.0-1.x86_64.rpm -id=$(docker create azure/azure-cli:mariner-builder bash) +id=$(docker create azure/azure-cli:azurelinux-builder bash) # https://docs.docker.com/engine/reference/commandline/cp/ # Append /. so that the x86_64 folder's content is copied, instead of x86_64 folder itself. docker cp $id:/out/. ${BUILD_STAGINGDIRECTORY} @@ -40,6 +40,6 @@ docker cp $id:/out/. ${BUILD_STAGINGDIRECTORY} # # The products of `docker save` can be rehydrated using `docker load`. # mkdir -p ${BUILD_STAGINGDIRECTORY}/docker -# docker save azure/azure-cli:mariner-builder | gzip > ${BUILD_STAGINGDIRECTORY}/docker/azure_azure-cli_mariner-builder.tar.gz & -# docker save azure/azure-cli:mariner | gzip > ${BUILD_STAGINGDIRECTORY}/docker/azure_azure-cli_mariner.tar.gz & +# docker save azure/azure-cli:azurelinux-builder | gzip > ${BUILD_STAGINGDIRECTORY}/docker/azure_azure-cli_azurelinux-builder.tar.gz & +# docker save azure/azure-cli:azurelinux | gzip > ${BUILD_STAGINGDIRECTORY}/docker/azure_azure-cli_azurelinux.tar.gz & # wait diff --git a/scripts/release/rpm/test_mariner_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh similarity index 100% rename from scripts/release/rpm/test_mariner_in_docker.sh rename to scripts/release/rpm/test_azurelinux_in_docker.sh From d0e003bf5ac2c6dd05ca71455f6655128197c576 Mon Sep 17 00:00:00 2001 From: Hang Date: Mon, 12 Aug 2024 16:17:11 +0800 Subject: [PATCH 6/8] Minor fix --- scripts/release/rpm/azurelinux.dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/azurelinux.dockerfile b/scripts/release/rpm/azurelinux.dockerfile index 5f36aa0ad48..bdd93868d89 100644 --- a/scripts/release/rpm/azurelinux.dockerfile +++ b/scripts/release/rpm/azurelinux.dockerfile @@ -16,8 +16,8 @@ WORKDIR /azure-cli COPY . . -# Mariner 2.0's default Python is 3.9, the rpm path is /usr/src/mariner/RPMS/x86_64/azure-cli-2.63.0-1.cm2.x86_64.rpm -# Azure Linux 3's default Python is 3.12, the rpm path is /usr/src/azl/RPMS/x86_64/azure-cli-2.62.0-1.azl3.x86_64.rpm +# Mariner 2.0's python3 is 3.9, the rpm path is /usr/src/mariner/RPMS/x86_64/azure-cli-2.63.0-1.cm2.x86_64.rpm +# Azure Linux 3's python3 is 3.12, the rpm path is /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \ From f3f4f24fb039d1d10fb2fa6d375f2590c6594697 Mon Sep 17 00:00:00 2001 From: Hang Date: Tue, 13 Aug 2024 15:36:02 +0800 Subject: [PATCH 7/8] Minor fix --- scripts/release/rpm/test_azurelinux_in_docker.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/release/rpm/test_azurelinux_in_docker.sh b/scripts/release/rpm/test_azurelinux_in_docker.sh index 3f9bce55552..6161d65a87e 100644 --- a/scripts/release/rpm/test_azurelinux_in_docker.sh +++ b/scripts/release/rpm/test_azurelinux_in_docker.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -# This script should be run in a Mariner 2.0 docker container. +# This script should be run in a Azure Linux docker container. set -exv export USERNAME=azureuser From 3e499694e0de4cb68c2a746b291249d1010a9402 Mon Sep 17 00:00:00 2001 From: Hang Date: Wed, 14 Aug 2024 14:28:10 +0800 Subject: [PATCH 8/8] Update scripts/release/rpm/azurelinux.dockerfile Co-authored-by: Jiashuo Li <4003950+jiasli@users.noreply.github.com> --- scripts/release/rpm/azurelinux.dockerfile | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/scripts/release/rpm/azurelinux.dockerfile b/scripts/release/rpm/azurelinux.dockerfile index bdd93868d89..4ee528ea955 100644 --- a/scripts/release/rpm/azurelinux.dockerfile +++ b/scripts/release/rpm/azurelinux.dockerfile @@ -16,8 +16,12 @@ WORKDIR /azure-cli COPY . . -# Mariner 2.0's python3 is 3.9, the rpm path is /usr/src/mariner/RPMS/x86_64/azure-cli-2.63.0-1.cm2.x86_64.rpm -# Azure Linux 3's python3 is 3.12, the rpm path is /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm +# Mariner 2.0's python3 is 3.9, the rpm paths are +# /usr/src/mariner/RPMS/x86_64/azure-cli-2.63.0-1.cm2.x86_64.rpm +# /usr/src/mariner/RPMS/aarch64/azure-cli-2.63.0-1.cm2.aarch64.rpm +# Azure Linux 3's python3 is 3.12, the rpm paths are +# /usr/src/azl/RPMS/x86_64/azure-cli-2.63.0-1.azl3.x86_64.rpm +# /usr/src/azl/RPMS/aarch64/azure-cli-2.63.0-1.azl3.aarch64.rpm RUN dos2unix ./scripts/release/rpm/azure-cli.spec && \ REPO_PATH=$(pwd) CLI_VERSION=$cli_version PYTHON_PACKAGE=python3 PYTHON_CMD=python3 \ rpmbuild -v -bb --clean scripts/release/rpm/azure-cli.spec && \