Merge branch 'master' of github.com:azure/sonic-buildimage into sae_p…

…2_warm

Signed-off-by: Stepan Blyschak <stepanb@nvidia.com>

.azure-pipelines/azure-pipelines-build.yml

@@ -41,49 +41,61 @@ jobs:
         - name: vs
           variables:
             dbg_image: yes
+
         - name: barefoot
           variables:
+            sync_rpc_image: yes
             swi_image: yes
+
         - name: broadcom
           variables:
             dbg_image: yes
             swi_image: yes
             raw_image: yes
             sync_rpc_image: yes
             platform_rpc: brcm
+
         - name: centec
           variables:
             dbg_image: yes
             sync_rpc_image: yes
             platform_rpc: centec
+
         - name: centec-arm64
           pool: sonicbld_8c
-          timeoutInMinutes: 1800
+          timeoutInMinutes: 2160
           variables:
             PLATFORM_ARCH: arm64
+
         - name: generic
           variables:
             dbg_image: yes
+
         - name: innovium
           variables:
             swi_image: yes
+
         - name: marvell-armhf
           pool: sonicbld_8c
-          timeoutInMinutes: 1800
+          timeoutInMinutes: 2160
           variables:
             PLATFORM_ARCH: armhf
+
         - name: mellanox
           variables:
             dbg_image: yes
             sync_rpc_image: yes
             platform_rpc: mlnx
+
         - name: nephos
           variables:
             dbg_image: yes
             sync_rpc_image: yes
             platform_rpc: nephos
+
     buildSteps:
       - bash: |
+          set -ex
           if [ $(GROUP_NAME) == vs ]; then
             if [ $(dbg_image) == yes ]; then
               make $BUILD_OPTIONS INSTALL_DEBUG_TOOLS=y target/sonic-vs.img.gz && mv target/sonic-vs.img.gz target/sonic-vs-dbg.img.gz

.azure-pipelines/azure-pipelines-job-groups.yml

@@ -19,7 +19,7 @@ parameters:
   default: ''
 - name: 'timeoutInMinutes'
   type: 'number'
-  default: 600
+  default: 720
 - name: 'jobFilters'
   type: object
   default: ''

.azure-pipelines/docker-sonic-mgmt.yml

@@ -0,0 +1,57 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-mgmt image
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - dockers/docker-sonic-mgmt
+
+parameters:
+- name: registry_url
+  type: string
+  default: sonicdev-microsoft.azurecr.io
+- name: registry_conn
+  type: string
+  default: sonicdev
+
+stages:
+- stage: Build
+  jobs:
+  - job: Build
+    pool: sonicbld
+    timeoutInMinutes: 360
+    steps:
+    - template: cleanup.yml
+    - checkout: self
+      clean: true
+      submodules: recursive
+    - bash: |
+        set -xe
+        make configure PLATFORM=generic
+        make target/docker-sonic-mgmt.gz
+        docker load -i target/docker-sonic-mgmt.gz
+        docker tag docker-sonic-mgmt $REGISTRY_SERVER/docker-sonic-mgmt:latest
+      env:
+        REGISTRY_SERVER: ${{ parameters.registry_url }}
+      displayName: Build docker-sonic-mgmt.gz
+    - task: Docker@2
+      displayName: Upload image
+      inputs:
+        containerRegistry: ${{ parameters.registry_conn }}
+        repository: docker-sonic-mgmt
+        command: push
+        tags: latest

.azure-pipelines/docker-sonic-slave.yml

@@ -0,0 +1,107 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Build and push sonic-slave-[buster|jessie|stretch] images for amd64/armhf/arm64
+
+schedules:
+- cron: "0 8 * * *"
+  branches:
+    include:
+    - master
+    - 202012
+  always: true
+
+trigger: none
+pr:
+  branches:
+    include:
+    - master
+  paths:
+    include:
+    - sonic-slave-jessie
+    - sonic-slave-stretch
+    - sonic-slave-buster
+
+parameters:
+- name: 'arches'
+  type: object
+  default:
+  - amd64
+  - armhf
+  - arm64
+- name: 'dists'
+  type: object
+  default:
+  - buster
+  - stretch
+  - jessie
+- name: registry_url
+  type: string
+  default: sonicdev-microsoft.azurecr.io
+- name: registry_conn
+  type: string
+  default: sonicdev
+
+stages:
+- stage: Build
+  jobs:
+  - ${{ each dist in parameters.dists }}:
+    - ${{ if endswith(variables['Build.DefinitionName'], dist) }}:
+      - ${{ each arch in parameters.arches }}:
+        - job: Build_${{ dist }}_${{ arch }}
+          timeoutInMinutes: 360
+          pool: sonicbld
+          steps:
+          - template: cleanup.yml
+          - checkout: self
+            clean: true
+            submodules: recursive
+          - bash: |
+              set -ex
+
+              containers=$(docker container ls | grep "sonic-slave" | awk '{ print $1 }')
+              if [ ! -z "$containers" ]; then
+                docker container kill $containers || true
+                sleep 5
+              fi
+              images=$(docker images 'sonic-slave-*' -a -q)
+              if [ ! -z "$images" ]; then
+                docker rmi -f $images
+              fi
+
+              SLAVE_DIR=sonic-slave-${{ dist }}
+              if [ x${{ arch }} == x"amd64" ]; then
+                SLAVE_BASE_IMAGE=${SLAVE_DIR}
+              else
+                SLAVE_BASE_IMAGE=${SLAVE_DIR}-march-${{ arch }}
+              fi
+
+              tmpfile=$(mktemp)
+
+              echo ${{ arch }} > .arch
+
+              DOCKER_DATA_ROOT_FOR_MULTIARCH=/data/march/docker BLDENV=${{ dist }} make -f Makefile.work sonic-slave-build | tee $tmpfile
+              SLAVE_BASE_TAG=$(grep "^Checking sonic-slave-base image:" $tmpfile | awk -F ':' '{print $3}')
+              SLAVE_TAG=$(grep "^Checking sonic-slave image:" $tmpfile | awk -F ':' '{print $3}')
+
+              mkdir -p target
+
+              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:latest
+              docker tag $SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG $REGISTRY_SERVER/$SLAVE_BASE_IMAGE:$SLAVE_BASE_TAG
+              set +x
+              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_IMAGE]$SLAVE_BASE_IMAGE"
+              echo "##vso[task.setvariable variable=VARIABLE_SLAVE_BASE_TAG]$SLAVE_BASE_TAG"
+            env:
+              REGISTRY_SERVER: ${{ parameters.registry_url }}
+            displayName: Build sonic-slave-${{ dist }}-${{ arch }}
+
+          - task: Docker@2
+            displayName: Upload image
+            inputs:
+              containerRegistry: ${{ parameters.registry_conn }}
+              repository: $(VARIABLE_SLAVE_BASE_IMAGE)
+              command: push
+              tags: |
+                $(VARIABLE_SLAVE_BASE_TAG)
+                latest

.azure-pipelines/dpkg-cache-cleanup.yml

@@ -0,0 +1,27 @@
+# Starter pipeline
+# Start with a minimal pipeline that you can customize to build and deploy your code.
+# Add steps that build, run tests, deploy, and more:
+# https://aka.ms/yaml
+# Clean up the cache 30 days ago
+
+schedules:
+- cron: "0 0 * * *"
+  branches:
+    include:
+    - master
+  always: true
+
+trigger: none
+pr: none
+
+jobs:
+- job: Build
+  pool: sonicbld
+  timeoutInMinutes: 5
+  steps:
+  - checkout: none
+  - script: |
+      set -xe
+      sudo find /nfs/dpkg_cache/ -name *.tgz -mtime +30 -type f -delete
+    displayName: clean dpkg cache
+    

.azure-pipelines/official-build.yml

@@ -11,6 +11,12 @@ schedules:
     - master
     - 202012
   always: true
+- cron: "0 4 * * *"
+  displayName: nightly build for release
+  branches:
+    include:
+    - 201911
+    - 201811
 
 trigger: none
 pr: none

.gitmodules

@@ -16,10 +16,6 @@
 [submodule "src/p4-hlir/p4-hlir"]
 	path = platform/p4/p4-hlir/p4-hlir
 	url = https://github.com/p4lang/p4-hlir
-[submodule "quagga"]
-	path = src/sonic-quagga
-	url = https://github.com/Azure/sonic-quagga
-	branch = debian/0.99.24.1
 [submodule "sonic-dbsyncd"]
 	path = src/sonic-dbsyncd
 	url = https://github.com/Azure/sonic-dbsyncd

Makefile.work

@@ -38,6 +38,7 @@
 #  * SONIC_DPKG_CACHE_METHOD: Specifying method of obtaining the Debian packages from cache: none or cache
 #  * SONIC_DPKG_CACHE_SOURCE: Debian package cache location when cache enabled for debian packages
 #  * BUILD_LOG_TIMESTAMP: Set timestamp in the build log (simple/none)
+#  * DOCKER_EXTRA_OPTS: Extra command line arguments for dockerd running in slave container.
 #
 ###############################################################################
 
@@ -99,6 +100,11 @@ else
 SLAVE_DIR = sonic-slave-jessie
 endif
 
+# Define a do-nothing target for rules/config.user so that when
+# the file is missing, make won't try to rebuld everything.
+rules/config.user:
+	@echo -n ""
+
 include rules/config
 -include rules/config.user
 
@@ -126,7 +132,7 @@ $(shell SONIC_VERSION_CONTROL_COMPONENTS=$(SONIC_VERSION_CONTROL_COMPONENTS) \
     scripts/generate_buildinfo_config.sh)
 
 # Generate the slave Dockerfile, and prepare build info for it
-$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
+$(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) DOCKER_EXTRA_OPTS=$(DOCKER_EXTRA_OPTS) j2 $(SLAVE_DIR)/Dockerfile.j2 > $(SLAVE_DIR)/Dockerfile)
 $(shell CONFIGURED_ARCH=$(CONFIGURED_ARCH) MULTIARCH_QEMU_ENVIRON=$(MULTIARCH_QEMU_ENVIRON) j2 $(SLAVE_DIR)/Dockerfile.user.j2 > $(SLAVE_DIR)/Dockerfile.user)
 $(shell BUILD_SLAVE=y scripts/prepare_docker_buildinfo.sh $(SLAVE_BASE_IMAGE) $(SLAVE_DIR)/Dockerfile $(CONFIGURED_ARCH) "" $(BLDENV))
 

azure-pipelines.yml

@@ -44,6 +44,8 @@ stages:
       jobGroups:
       - name: vs
       - name: broadcom
+        variables:
+          swi_image: yes
       - name: mellanox
 
 - stage: Test

build_debian.sh

@@ -474,16 +474,15 @@ fi
 
 ## Version file
 sudo mkdir -p $FILESYSTEM_ROOT/etc/sonic
-sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml > /dev/null <<EOF
-build_version: '${SONIC_IMAGE_VERSION}'
-debian_version: '$(cat $FILESYSTEM_ROOT/etc/debian_version)'
-kernel_version: '$kversion'
-asic_type: $sonic_asic_platform
-commit_id: '$(git rev-parse --short HEAD)'
-build_date: $(date -u)
-build_number: ${BUILD_NUMBER:-0}
-built_by: $USER@$BUILD_HOSTNAME
-EOF
+export build_version="${SONIC_IMAGE_VERSION}"
+export debian_version="$(cat $FILESYSTEM_ROOT/etc/debian_version)"
+export kernel_version="${kversion}"
+export asic_type="${sonic_asic_platform}"
+export commit_id="$(git rev-parse --short HEAD)"
+export build_date="$(date -u)"
+export build_number="${BUILD_NUMBER:-0}"
+export built_by="$USER@$BUILD_HOSTNAME"
+j2 files/build_templates/sonic_version.yml.j2 | sudo tee $FILESYSTEM_ROOT/etc/sonic/sonic_version.yml
 
 ## Copy over clean-up script
 sudo cp ./files/scripts/core_cleanup.py $FILESYSTEM_ROOT/usr/bin/core_cleanup.py
@@ -578,9 +577,8 @@ sudo rm -f $ONIE_INSTALLER_PAYLOAD $FILESYSTEM_SQUASHFS
 ## Note: -x to skip directories on different file systems, such as /proc
 sudo du -hsx $FILESYSTEM_ROOT
 sudo mkdir -p $FILESYSTEM_ROOT/var/lib/docker
-sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR
-
 scripts/collect_host_image_version_files.sh $TARGET_PATH $FILESYSTEM_ROOT
+sudo mksquashfs $FILESYSTEM_ROOT $FILESYSTEM_SQUASHFS -e boot -e var/lib/docker -e $PLATFORM_DIR
 
 # Ensure admin gid is 1000
 gid_user=$(sudo LANG=C chroot $FILESYSTEM_ROOT id -g $USERNAME) || gid_user="none"