Refine Azure pipelines for PR/Official/UpgradeVersion builds

.azure-pipelines/azure-pipelines-UpgrateVersion.yml

@@ -18,10 +18,25 @@ pool: sonicbld
 
 stages:
 - stage: Build
+  variables:
+    CACHE_MODE: wcache
+    VERSION_CONTROL_OPTIONS: 'SONIC_VERSION_CONTROL_COMPONENTS='
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
-      buildSlave: y
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      preSteps:
+      - script: |
+          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
+          if [ ! -z "$containers" ]; then
+            docker container kill $containers || true
+            sleep 5
+          fi
+          images=$(docker images 'sonic-slave-*' -a -q)
+          if [ ! -z "$images" ]; then
+            docker rmi -f $images
+          fi
+        displayName: 'Cleanup sonic slave'
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions
@@ -47,12 +62,16 @@ stages:
         echo "artifacts$artifacts"
         cp -r $(Pipeline.Workspace)/sonic-buildimage.${default_platform}/versions target/
         make freeze FREEZE_VERSION_OPTIONS=-r
-        find files/build/versions 
-        for artifact in $artifacts
+        find files/build/versions
+        ordered_artifacts=$(echo "$artifacts" | grep -v -E "arm64|armhf" && echo "$artifacts" | grep -E "arm64|armhf")
+        for artifact in $ordered_artifacts
         do
+          [[ "$artifact" == *arm64* || "$artifact" == *armhf* ]] && continue
           rm -rf target/versions
           cp -r $artifact/versions target/
-          make freeze FREEZE_VERSION_OPTIONS="-a -d"
+          OPTIONS="-a -d"
+          [[ "$artifact" == *arm64* || "$artifact" == *arm64* ]] && OPTIONS="-d"
+          make freeze FREEZE_VERSION_OPTIONS="$OPTIONS"
         done
         git diff files/build/versions
       displayName: 'Freeze Versions'

.azure-pipelines/azure-pipelines-build.yml

@@ -1,72 +1,56 @@
+# The azure pipeline template for Official build, and upgrade version build
+
 parameters:
 - name: 'jobFilters'
   type: object
   default: ''
 - name: 'buildOptions'
   type: string
   default: 'SONIC_CONFIG_BUILD_JOBS=1'
-- name: 'buildSlave'
-  type: string
-  default: 'n'
+- name: 'preSteps'
+  type: stepList
+  default: []
 - name: 'postSteps'
   type: stepList
   default: []
 
 jobs:
-- template: azure-pipelines-job-groups.yml
+- template: build-template.yml
   parameters:
     jobFilters: ${{ parameters.jobFilters }}
-    preSteps:
-      - script: |
-          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-          if [ ! -z "$containers" ]; then
-            docker container kill $containers || true
-            sleep 5
-          fi
-          if [ "${{ parameters.buildSlave }}" == "y" ]; then
-            images=$(docker images 'sonic-slave-*' -a -q)
-            [ ! -z "$images" ] && docker rmi -f $images
-          fi
-          sudo rm -rf $(ls -A1)
-        displayName: 'Init'
-      - checkout: self
-        submodules: recursive
-        displayName: 'Checkout code'
-      - script: |
-          make ${{ parameters.buildOptions }} PLATFORM=$GROUP_NAME configure
-        displayName: 'Make configure'
-    postSteps:
-      - ${{ parameters.postSteps }}
-      - publish: $(System.DefaultWorkingDirectory)/target
-        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
-        displayName: "Archive sonic image"
+    buildOptions: ${{ parameters.buildOptions }}
+    preSteps: ${{ parameters.preSteps }}
+    postSteps: ${{ parameters.postSteps }}
     jobGroups:
       - name: vs
         script: |
-          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
-          make ${{ parameters.buildOptions }} target/sonic-vs.img.gz
+          make $(BUILD_OPTIONS) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
       - name: broadcom
         script: |
-          make ${{ parameters.buildOptions }} target/sonic-broadcom.bin target/sonic-aboot-broadcom.swi
-      - name: barefoot
+          make $(BUILD_OPTIONS) target/sonic-broadcom.bin target/sonic-aboot-broadcom.swi
+      - name: mellanox
         script: |
-          make ${{ parameters.buildOptions }} target/sonic-barefoot.bin target/sonic-aboot-barefoot.swi
-      - name: centec
+          make $(BUILD_OPTIONS) ENABLE_SYNCD_RPC=y all
+      - name: barefoot
         script: |
-          make ${{ parameters.buildOptions }} INSTALL_DEBUG_TOOLS=y target/sonic-centec.bin
-          mv target/sonic-centec.bin target/sonic-centec-dbg.bin
-          make ${{ parameters.buildOptions }} target/sonic-centec.bin
-          make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y target/docker-syncd-centec-rpc.gz
+          make $(BUILD_OPTIONS) target/sonic-barefoot.bin target/sonic-aboot-barefoot.swi
       - name: innovium
         script: |
-          make ${{ parameters.buildOptions }} SONIC_CONFIG_BUILD_JOBS=1 target/sonic-innovium.bin
-      - name: mellanox
-        script: |
-          make ${{ parameters.buildOptions }} target/sonic-mellanox.bin
-      - name: mellanox
-        extName: _rpc
-        script: |
-          make ${{ parameters.buildOptions }} ENABLE_SYNCD_RPC=y all
+          make $(BUILD_OPTIONS) SONIC_CONFIG_BUILD_JOBS=1 target/sonic-innovium.bin
       - name: nephos
         script: |
-          make ${{ parameters.buildOptions }} target/sonic-nephos.bin
+          make $(BUILD_OPTIONS) target/sonic-nephos.bin
+      - name: marvell-armhf
+        pool: sonicbld_8c
+        timeoutInMinutes: 3600
+        variables:
+          PLATFORM_ARCH: armhf
+        script: |
+          make $(BUILD_OPTIONS) target/sonic-marvell-armhf.bin
+      - name: centec-arm64
+        pool: sonicbld_8c
+        timeoutInMinutes: 3600
+        variables:
+          PLATFORM_ARCH: arm64
+        script: |
+          make $(BUILD_OPTIONS) target/sonic-centec-arm64.bin

.azure-pipelines/azure-pipelines-job-groups.yml

@@ -23,7 +23,7 @@ parameters:
 
 jobs:
 - ${{ each jobGroup in parameters.jobGroups }}:
-  - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name)) }}:
+  - ${{ if or(eq(parameters.jobFilters, ''), containsValue(parameters.jobFilters, jobGroup.name), endswith(variables['Build.DefinitionName'], format('.{0}{1}', jobGroup.name, jobGroup.extName))) }}:
     - job: ${{ replace(format('{0}{1}', jobGroup.name, jobGroup.extName), '-', '_') }}
       ${{ each pair in jobGroup }}:
         ${{ if not(in(pair.key, 'job', 'name', 'extName', 'variables', 'steps', 'script', 'scriptEnv')) }}:
@@ -43,6 +43,7 @@ jobs:
       - ${{ parameters.preSteps }}
       - ${{ if ne(jobGroup.script, '') }}:
         - script: |
+            set -e
             ${{ jobGroup.script }}
           env:
             ${{ if ne(parameters.scriptEnv, '') }}:

.azure-pipelines/build-template.yml

@@ -1,116 +1,54 @@
-parameters:
-- name: platform
-  type: string
-  values:
-  - broadcom
-  - centec-arm64
-  - marvell-armhf
-  - mellanox
-  - vs
+# The azure pipeline template for PR build, Official build, and upgrade version build
 
-- name: platform_arch
-  type: string
-  values:
-  - amd64
-  - armhf
-  - arm64
-  default: amd64
-
-- name: platform_short
-  type: string
-  values:
-  - brcm
-  - centec-arm64
-  - marvell-armhf
-  - mlnx
-  - vs
-
-- name: cache_mode
-  type: string
-  values:
-  - wcache
-  - rcache
-  - cache
-
-- name: pool
+parameters:
+- name: 'jobFilters'
+  type: object
+  default: ''
+- name: 'buildOptions'
   type: string
-  values:
-  - sonicbld
-  - sonicbld_8c
-  default: sonicbld
-
-- name: dbg_image
-  type: boolean
-  default: false
-
-- name: swi_image
-  type: boolean
-  default: false
-
-- name: raw_image
-  type: boolean
-  default: false
-
-- name: sync_rpc_image
-  type: boolean
-  default: false
-
-- name: timeout
-  type: number
-  default: 600
+  default: 'SONIC_CONFIG_BUILD_JOBS=1'
+- name: 'preSteps'
+  type: stepList
+  default: []
+- name: 'postSteps'
+  type: stepList
+  default: []
+- name: jobGroups
+  type: object
+  default: []
 
 jobs:
-- job:
-  pool: ${{ parameters.pool }}
-  displayName: ${{ parameters.platform }}
-  timeoutInMinutes: ${{ parameters.timeout }}
-  steps:
-  - template: cleanup.yml
-  - checkout: self
-    clean: true
-    submodules: recursive
-    displayName: 'Checkout code'
-  - script: |
-      git submodule foreach --recursive 'git clean -xfdf || true'
-      git submodule foreach --recursive 'git reset --hard || true'
-      git submodule foreach --recursive 'git remote update || true'
-      git submodule update --init --recursive
-    displayName: 'Reset submodules'
-  - script: |
-      set -e
-      sudo modprobe overlay
-      sudo apt-get install -y acl
-      export DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker
-      CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=${{ parameters.cache_mode }} SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/${{ parameters.platform }}"
-      ENABLE_DOCKER_BASE_PULL=y make configure PLATFORM=${{ parameters.platform }} PLATFORM_ARCH=${{ parameters.platform_arch }}
-      trap "sudo rm -rf fsroot" EXIT
-
-      if [ ${{ parameters.platform }} == vs ]; then
-        if [ ${{ parameters.dbg_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && \
-            mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz
-        fi
-
-        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/docker-sonic-vs.gz target/sonic-vs.img.gz target/docker-ptf.gz
-      else
-        if [ ${{ parameters.dbg_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) INSTALL_DEBUG_TOOLS=y target/sonic-${{ parameters.platform }}.bin && \
-            mv target/sonic-${{ parameters.platform }}.bin target/sonic-${{ parameters.platform }}-dbg.bin
-        fi
-        if [ ${{ parameters.swi_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y target/sonic-aboot-${{ parameters.platform }}.swi
-        fi
-        if [ ${{ parameters.raw_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.raw
-        fi
-        if [ ${{ parameters.sync_rpc_image }} == true ]; then
-          make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) ENABLE_SYNCD_RPC=y target/docker-syncd-${{ parameters.platform_short }}-rpc.gz
-        fi
-
-        make USERNAME=admin $CACHE_OPTIONS SONIC_BUILD_JOBS=$(nproc) target/sonic-${{ parameters.platform }}.bin
-      fi
-    displayName: 'Build sonic image'
-  - template: cleanup.yml
-  - publish: $(System.DefaultWorkingDirectory)/
-    artifact: sonic-buildimage.${{ parameters.platform }}
-    displayName: "Archive sonic image"
+- template: azure-pipelines-job-groups.yml
+  parameters:
+    jobFilters: ${{ parameters.jobFilters }}
+    jobVariables:
+      PLATFORM: $(GROUP_NAME)
+      PLATFORM_ARCH: amd64
+      BUILD_OPTIONS: ${{ parameters.buildOptions }}
+    preSteps:
+      - template: cleanup.yml
+      - ${{ parameters. preSteps }}
+      - script: |
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+            CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM)"
+            BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
+            echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"
+          fi
+        displayName: "Make build options"
+      - checkout: self
+        submodules: recursive
+        displayName: 'Checkout code'
+      - script: |
+          sudo modprobe overlay
+          sudo apt-get install -y acl
+          export DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker
+          sudo bash -c "echo 1 > /proc/sys/vm/compact_memory"
+          make $BUILD_OPTIONS PLATFORM=$(PLATFORM) PLATFORM_ARCH=$(PLATFORM_ARCH) configure
+        displayName: 'Make configure'
+    postSteps:
+      - publish: $(System.DefaultWorkingDirectory)/target
+        artifact: 'sonic-buildimage.$(GROUP_NAME)$(GROUP_EXTNAME)'
+        displayName: "Archive sonic image"
+      - ${{ parameters.postSteps }}
+      - template: cleanup.yml
+    jobGroups: ${{ parameters.jobGroups }}