Merge branch 'master' of http://github.com/azure/sonic-buildimage

Azure · Apr 1, 2022 · 63b2362 · 63b2362
2 parents 47a8caf + eb141f2
commit 63b2362
Show file tree

Hide file tree

Showing 687 changed files with 50,324 additions and 20,033 deletions.
diff --git a/.azure-pipelines/azure-pipelines-UpgrateVersion.yml b/.azure-pipelines/azure-pipelines-UpgrateVersion.yml
@@ -18,8 +18,30 @@ schedules:
     - 202006
   always: true
 
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
 pool: sonicbld
 
+parameters:
+- name: 'jobFilters'
+  type: object
+  default:
+  - vs
+  - barefoot
+  - broadcom
+  - centec
+  - centec-arm64
+  - generic
+  - innovium
+  - marvell-armhf
+  - mellanox
+
 stages:
 - stage: Build
   variables:
@@ -28,19 +50,10 @@ stages:
   jobs:
   - template: azure-pipelines-build.yml
     parameters:
-      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
+      jobFilters: ${{ parameters.jobFilters }}
+      buildOptions: '${{ variables.VERSION_CONTROL_OPTIONS }} ENABLE_DOCKER_BASE_PULL=n SONIC_BUILD_JOBS=$(nproc) ENABLE_IMAGE_SIGNATURE=y'
       preSteps:
-      - script: |
-          containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-          if [ ! -z "$containers" ]; then
-            docker container kill $containers || true
-            sleep 5
-          fi
-          images=$(docker images 'sonic-slave-*' -a -q)
-          if [ ! -z "$images" ]; then
-            docker rmi -f $images
-          fi
-        displayName: 'Cleanup sonic slave'
+      - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
 - stage: UpgradeVersions
   jobs:
   - job: UpgradeVersions

diff --git a/.azure-pipelines/azure-pipelines-image-template.yml b/.azure-pipelines/azure-pipelines-image-template.yml
@@ -28,7 +28,7 @@ jobs:
       - template: cleanup.yml
       - ${{ parameters.preSteps }}
       - script: |
-          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox)$"; then
+          if [ -n "$(CACHE_MODE)" ] && echo $(PLATFORM_AZP) | grep -E -q "^(vs|broadcom|mellanox|marvell-armhf)$"; then
             CACHE_OPTIONS="SONIC_DPKG_CACHE_METHOD=$(CACHE_MODE) SONIC_DPKG_CACHE_SOURCE=/nfs/dpkg_cache/$(PLATFORM_AZP)"
             BUILD_OPTIONS="$(BUILD_OPTIONS) $CACHE_OPTIONS"
             echo "##vso[task.setvariable variable=BUILD_OPTIONS]$BUILD_OPTIONS"

diff --git a/.azure-pipelines/build-commonlib.yml b/.azure-pipelines/build-commonlib.yml
@@ -0,0 +1,19 @@
+pr: none
+trigger: none
+schedules:
+- cron: "0 0 * * *"
+  displayName: Daily build
+  branches:
+    include:
+      - master
+      - 202???
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
+
+jobs:
+- template: .azure-pipelines/template-commonlib.yml@buildimage
diff --git a/.azure-pipelines/docker-sonic-slave-arm64.yml b/.azure-pipelines/docker-sonic-slave-arm64.yml
@@ -23,6 +23,7 @@ pr:
     - sonic-slave-stretch
     - sonic-slave-buster
     - sonic-slave-bullseye
+    - .azure-pipelines
 
 parameters:
 - name: 'dists'

diff --git a/.azure-pipelines/docker-sonic-slave-armhf.yml b/.azure-pipelines/docker-sonic-slave-armhf.yml
@@ -23,6 +23,7 @@ pr:
     - sonic-slave-stretch
     - sonic-slave-buster
     - sonic-slave-bullseye
+    - .azure-pipelines
 
 parameters:
 - name: 'dists'

diff --git a/.azure-pipelines/docker-sonic-slave-template.yml b/.azure-pipelines/docker-sonic-slave-template.yml
@@ -3,6 +3,13 @@
 # Add steps that build, run tests, deploy, and more:
 # https://aka.ms/yaml
 # Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+resources:
+  repositories:
+  - repository: buildimage
+    type: github
+    name: Azure/sonic-buildimage
+    ref: master
+    endpoint: build
 
 parameters:
 - name: arch
@@ -38,6 +45,7 @@ jobs:
   pool: ${{ parameters.pool }}
   steps:
   - template: cleanup.yml
+  - template: .azure-pipelines/template-clean-sonic-slave.yml@buildimage
   - checkout: self
     clean: true
     submodules: recursive
@@ -68,16 +76,6 @@ jobs:
         BUILD_OPTIONS = 'SONIC_VERSION_CONTROL_COMPONENTS=deb,py2,py3,web,git,docker'
       fi
 
-      containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
-      if [ ! -z "$containers" ]; then
-        docker container kill $containers || true
-        sleep 5
-      fi
-      images=$(docker images 'sonic-slave-*' -a -q)
-      if [ ! -z "$images" ]; then
-        docker rmi -f $images
-      fi
-
       tmpfile=$(mktemp)
 
       echo ${{ parameters.arch }} > .arch
@@ -90,6 +88,10 @@ jobs:
 
       docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:latest
       docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE_UPLOAD:$SLAVE_BASE_TAG
+      if [ "$SLAVE_BASE_IMAGE_UPLOAD" != "$SLAVE_DIR" ]; then
+        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:latest-${{ parameters.arch }}
+        docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_DIR:$SLAVE_BASE_TAG
+      fi
       set +x
       echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE_UPLOAD"
       echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
@@ -98,6 +100,7 @@ jobs:
     displayName: Build sonic-slave-${{ parameters.dist }}-${{ parameters.arch }}
 
   - task: Docker@2
+    condition: ne(variables['Build.Reason'], 'PullRequest')
     displayName: Upload image
     inputs:
       containerRegistry: ${{ parameters.registry_conn }}
@@ -106,3 +109,13 @@ jobs:
       tags: |
         $(VARIABLE_SLAVE_BASE_TAG)
         latest
+  - ${{ if ne(parameters.arch, 'amd64') }}:
+    - task: Docker@2
+      condition: ne(variables['Build.Reason'], 'PullRequest')
+      displayName: Upload image ${{ parameters.dist }}
+      inputs:
+        containerRegistry: ${{ parameters.registry_conn }}
+        repository: "sonic-slave-${{ parameters.dist }}"
+        command: push
+        tags: |
+          $(VARIABLE_SLAVE_BASE_TAG) 
diff --git a/.azure-pipelines/docker-sonic-slave.yml b/.azure-pipelines/docker-sonic-slave.yml
@@ -24,6 +24,7 @@ pr:
     - sonic-slave-buster
     - sonic-slave-bullseye
     - src/sonic-build-hooks
+    - .azure-pipelines
 
 parameters:
 - name: 'arches'

diff --git a/.azure-pipelines/official-build-cache.yml b/.azure-pipelines/official-build-cache.yml
@@ -37,3 +37,8 @@ stages:
           variables:
             docker_syncd_rpc_image: yes
             platform_rpc: mlnx
+        - name: marvell-armhf
+          pool: sonicbld-armhf
+          timeoutInMinutes: 1200
+          variables:
+            PLATFORM_ARCH: armhf
diff --git a/.azure-pipelines/official-build-cisco-8000.yml b/.azure-pipelines/official-build-cisco-8000.yml
@@ -22,13 +22,19 @@ resources:
     name: Cisco-8000-sonic/platform-cisco-8000
     endpoint: cisco-connection
 
+variables:
+- group: SONIC-AKV-STROAGE-1
+- name: StorageSASKey
+  value: $(sonicstorage-SasToken)
+
 stages:
 - stage: Build
   pool: sonic
   variables:
     CACHE_MODE: wcache
     SKIP_CHECKOUT: true
     TERM: ''
+    PACKAGE_URL: "https://sonicstorage.blob.core.windows.net/packages"
 
   jobs:
   - template: azure-pipelines-build.yml
@@ -60,5 +66,29 @@ stages:
           make PLATFORM=cisco-8000 platform/cisco-8000
           tar xfz $(System.ArtifactsDirectory)/artifactory-*.tar.gz -C platform/cisco-8000
         displayName: 'Setup cisco artifacts'
+      - script: |
+          set -ex
+          filename=$(find platform/cisco-8000/artifactory/sonic -name cisco-* -type f | head -n 1)
+          if [ -z "$filename" ]; then
+            echo "Cisco sai package not found" 1>&2
+            exit 1
+          fi
+          cd $(dirname $filename)
+          echo "PWD=$(pwd)"
+          ls -l *.deb
+          while read -r package; do
+            # Cisco version format: <VERSION>-sai-<sai-ver>-<distribution>-<COMMIT HASH>
+            # The <sai-ver> may contain several values in one build, the part is skipped when publishing to storage
+            # See https://github.com/Cisco-8000-sonic/sdk/blob/master/azure-pipelines.yml
+            # The $PACKAGE_URL is only accessible for AZP
+            version=$(echo $package | awk -F_ '{print $(NF-1)}' | cut -d- -f1,2,4,5)
+            package_url="$PACKAGE_URL/sai/ciscosai/master/$version/$package"
+            echo "Override package $package from $package_url"
+            wget "$package_url$StorageSASKey" -O "$package"
+          done < <(ls *.deb)
+        env:
+          StorageSASKey: $(StorageSASKey)
+        condition: ne(variables['Build.Reason'], 'PullRequest')
+        displayName: "Override cisco sai packages"
       jobGroups:
       - name: cisco-8000
diff --git a/.azure-pipelines/run-test-template.yml b/.azure-pipelines/run-test-template.yml
@@ -7,6 +7,9 @@ parameters:
   type: string
 - name: ptf_name
   type: string
+- name: section
+  type: string
+  default: ''
 - name: image
   type: string
   default: sonic-vs.img.gz
@@ -43,7 +46,7 @@ steps:
 
 - script: |
     rm -rf $(Build.ArtifactStagingDirectory)/*
-    docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }}"
+    docker exec sonic-mgmt bash -c "/data/sonic-mgmt/tests/kvmtest.sh -en -T ${{ parameters.tbtype }} ${{ parameters.tbname }} ${{ parameters.dut }} ${{ parameters.section }}"
   displayName: "Run tests"
 
 - script: |
@@ -75,17 +78,17 @@ steps:
   condition: succeededOrFailed()
 
 - publish: $(Build.ArtifactStagingDirectory)/kvmdump
-  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype}}.memdump@$(System.JobAttempt)
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype}}${{ parameters.section }}.memdump@$(System.JobAttempt)
   displayName: "Archive sonic kvm memdump"
   condition: failed()
 
 - publish: $(Build.ArtifactStagingDirectory)/logs
-  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype }}.log@$(System.JobAttempt)
+  artifact: sonic-buildimage.kvmtest.${{ parameters.tbtype }}${{ parameters.section }}.log@$(System.JobAttempt)
   displayName: "Archive sonic kvm logs"
   condition: succeededOrFailed()
 
 - task: PublishTestResults@2
   inputs:
     testResultsFiles: '$(Build.ArtifactStagingDirectory)/logs/**/*.xml'
-    testRunTitle: kvmtest.${{ parameters.tbtype }}
+    testRunTitle: kvmtest.${{ parameters.tbtype }}${{ parameters.section }}
   condition: succeededOrFailed()
diff --git a/.azure-pipelines/template-clean-sonic-slave.yml b/.azure-pipelines/template-clean-sonic-slave.yml
@@ -0,0 +1,8 @@
+steps:
+- script: |
+    containers=$(docker container ls -a | grep "sonic-slave" | awk '{ print $1 }')
+    [ -n "$containers" ] && docker container rm -f containers
+    docker images | grep "^<none>" | awk '{print$3}' | xargs -i docker rmi {}
+    images=$(docker images 'sonic-slave-*' -a -q)
+    [ -n "$images" ] && docker rmi -f $images
+  displayName: 'Cleanup sonic slave'
diff --git a/.azure-pipelines/template-commonlib.yml b/.azure-pipelines/template-commonlib.yml
@@ -0,0 +1,34 @@
+jobs:
+- job: Build
+  timeoutInMinutes: 120
+  pool: sonicbld
+  steps:
+  - checkout: self
+    clean: true
+    submodules: recursive
+  - script: |
+      set -ex
+      case $(Build.SourceBranchName) in
+        202012 | 202106)
+          bldenv=buster
+          ;;
+        *)
+          bldenv=bullseye
+          ;;
+      esac
+      BLDENV=$bldenv make -f Makefile.work configure PLATFORM=vs ENABLE_DOCKER_BASE_PULL=y
+      echo "##vso[task.setvariable variable=bldenv;]$bldenv"
+    displayName: Make configure
+  - script: |
+      set -ex
+      LIBNL3_VERSION_BASE=$(grep "LIBNL3_VERSION_BASE =" rules/libnl3.mk | awk '{print$3}')
+      LIBNL3_VERSION=$(grep "LIBNL3_VERSION =" rules/libnl3.mk | awk '{print$3}' | sed -e "s/(//" -e "s/)//" -e "s/\\$//" -e "s/LIBNL3_VERSION_BASE/$LIBNL3_VERSION_BASE/")
+      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libnl-3-200_${LIBNL3_VERSION}_amd64.deb ENABLE_DOCKER_BASE_PULL=y
+
+      LIBYANG_VERSION_BASE=$(grep "LIBYANG_VERSION_BASE =" rules/libyang.mk | awk '{print$3}')
+      LIBYANG_VERSION=$(grep "LIBYANG_VERSION =" rules/libyang.mk | awk '{print$3}' | sed -e "s/\\$//" -e "s/(//" -e "s/)//" -e "s/LIBYANG_VERSION_BASE/$LIBYANG_VERSION_BASE/")
+      BLDENV=$(bldenv) make -f Makefile.work target/debs/$(bldenv)/libyang_${LIBYANG_VERSION}_amd64.deb
+      find target -name *.deb | xargs -i cp {} $(Build.ArtifactStagingDirectory)
+    displayName: Make common lib packages
+  - publish: $(Build.ArtifactStagingDirectory)
+    artifact: common-lib
diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md
@@ -30,13 +30,20 @@
 - [ ] 202006
 - [ ] 202012
 - [ ] 202106
+- [ ] 202111
 
 #### Description for the changelog
 <!--
 Write a short (one line) summary that describes the changes in this
 pull request for inclusion in the changelog:
 -->
 
+#### Link to config_db schema for YANG module changes
+<!--
+Provide a link to config_db schema for the table for which YANG model
+is defined
+Link should point to correct section on https://github.com/Azure/SONiC/wiki/Configuration.
+-->
 
 #### A picture of a cute animal (not mandatory but encouraged)
 
diff --git a/.gitmodules b/.gitmodules
@@ -99,4 +99,7 @@
 	url = https://github.com/Azure/sonic-linkmgrd.git
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
-	url = https://github.com/Azure/sonic-pins.git
+	url = https://github.com/Azure/sonic-pins.git
+[submodule "src/thrift_0_14_1/thrift"]
+	path = src/thrift_0_14_1/thrift
+	url = https://github.com/apache/thrift.git
diff --git a/Makefile.work b/Makefile.work
@@ -115,6 +115,10 @@ rules/config.user:
 include rules/config
 -include rules/config.user
 
+ifneq ($(DEFAULT_CONTAINER_REGISTRY),)
+override DEFAULT_CONTAINER_REGISTRY := $(DEFAULT_CONTAINER_REGISTRY)/
+endif
+
 ifeq ($(ENABLE_DOCKER_BASE_PULL),)
 	override ENABLE_DOCKER_BASE_PULL = n
 endif
@@ -139,9 +143,9 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
-$(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
+$(shell BUILD_SLAVE=y DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 
 # Add the versions in the tag, if the version change, need to rebuild the slave
 SLAVE_BASE_TAG = $(shell cat $(SLAVE_DIR)/Dockerfile $(SLAVE_DIR)/buildinfo/versions/versions-* src/sonic-build-hooks/hooks/* | sha1sum | awk '{print substr($$1,0,11);}')
@@ -270,6 +274,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            SONIC_ENABLE_PFCWD_ON_START=$(ENABLE_PFCWD_ON_START) \
                            SONIC_ENABLE_SYNCD_RPC=$(ENABLE_SYNCD_RPC) \
                            SONIC_INSTALL_DEBUG_TOOLS=$(INSTALL_DEBUG_TOOLS) \
+                           SONIC_SAITHRIFT_V2=$(SAITHRIFT_V2) \
                            MDEBUG=$(MDEBUG) \
                            PASSWORD=$(PASSWORD) \
                            USERNAME=$(USERNAME) \
@@ -290,10 +295,12 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \
                            BUILD_LOG_TIMESTAMP=$(BUILD_LOG_TIMESTAMP) \
                            SONIC_ENABLE_IMAGE_SIGNATURE=$(ENABLE_IMAGE_SIGNATURE) \
+                           SONIC_DEFAULT_CONTAINER_REGISTRY=$(DEFAULT_CONTAINER_REGISTRY) \
                            ENABLE_HOST_SERVICE_ON_START=$(ENABLE_HOST_SERVICE_ON_START) \
                            SLAVE_DIR=$(SLAVE_DIR) \
                            ENABLE_AUTO_TECH_SUPPORT=$(ENABLE_AUTO_TECH_SUPPORT) \
                            BUILD_MULTIASIC_KVM=$(BUILD_MULTIASIC_KVM) \
+                           ENABLE_ASAN=$(ENABLE_ASAN) \
                            $(SONIC_OVERRIDE_BUILD_VARS)
 
 .PHONY: sonic-slave-build sonic-slave-bash init reset