Merge branch 'master' into dualtor-neigh-miss

.azure-pipelines/build-docker-sonic-vs-template.yml

@@ -36,41 +36,47 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 12
+      pipeline: Azure.sonic-sairedis
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: ${{ parameters.swss_artifact_name }}
-    displayName: "Download sonic swss artifact"
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download pre-stage built ${{ parameters.swss_artifact_name }}"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 1
+      pipeline: Azure.sonic-buildimage.official.vs
       artifact: sonic-buildimage.vs
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-    displayName: "Download sonic buildimage"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: '**/target/docker-sonic-vs.gz'
+    displayName: "Download sonic-buildimage docker-sonic-vs"
   - script: |
+      set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < ../target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
-      cp -v ../*.deb .azure-pipelines/docker-sonic-vs/debs
+      cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines
 
@@ -79,7 +85,8 @@ jobs:
       popd
 
       docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
-
+      rm -rf $(Build.ArtifactStagingDirectory)/download
+    displayName: "Build docker-sonic-vs"
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive sonic docker vs image"

.azure-pipelines/build-template.yml

@@ -23,12 +23,6 @@ parameters:
 - name: sonic_slave
   type: string
 
-- name: buildimage_artifact_name
-  type: string
-
-- name: buildimage_pipeline
-  type: number
-
 - name: sairedis_artifact_name
   type: string
 
@@ -46,6 +40,9 @@ parameters:
   type: boolean
   default: false
 
+- name: common_lib_artifact_name
+  type: string
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -54,7 +51,7 @@ jobs:
   pool:
     ${{ if ne(parameters.pool, 'default') }}:
       name: ${{ parameters.pool }}
-    ${{ if eq(parameters.pool, 'default') }}:
+    ${{ else }}:
       vmImage: 'ubuntu-20.04'
 
   container:
@@ -79,77 +76,63 @@ jobs:
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: ${{ parameters.swss_common_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-      path: '$(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libswsscommon_1.0.0_${{ parameters.arch }}.deb
+        libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
     displayName: "Download sonic swss common deb packages"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 12
+      pipeline: Azure.sonic-sairedis
       artifact: ${{ parameters.sairedis_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-      path: '$(Build.SourcesDirectory)/${{ parameters.sairedis_artifact_name }}'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        libsaivs_*.deb
+        libsaivs-dev_*.deb
+        libsairedis_*.deb
+        libsairedis-dev_*.deb
+        libsaimetadata_*.deb
+        libsaimetadata-dev_*.deb
+        syncd-vs_*.deb
     displayName: "Download sonic sairedis deb packages"
   - task: DownloadPipelineArtifact@2
-    ${{ if eq(parameters.buildimage_pipeline, 141) }}:
-      continueOnError: True
     inputs:
       source: specific
       project: build
-      pipeline: ${{ parameters.buildimage_pipeline }}
-      artifact: ${{ parameters.buildimage_artifact_name }}
+      pipeline: Azure.sonic-buildimage.common_libs
+      artifact: ${{ parameters.common_lib_artifact_name }}
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
-      path: '$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}'
-    displayName: "Download sonic buildimage deb packages"
-  - script: |
-      buildimage_artifact_downloaded=n
-      [ -d "$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}/target" ] && buildimage_artifact_downloaded=y
-      echo "buildimage_artifact_downloaded=$buildimage_artifact_downloaded"
-      echo "##vso[task.setvariable variable=buildimage_artifact_downloaded]$buildimage_artifact_downloaded"
-    condition: eq(${{ parameters.buildimage_pipeline }}, 141)
-    displayName: "Check if sonic buildimage deb packages downloaded"
-  - task: DownloadPipelineArtifact@2
-    condition: and(eq(variables.buildimage_artifact_downloaded, 'n'), eq(${{ parameters.buildimage_pipeline }}, 141))
-    inputs:
-      source: specific
-      project: build
-      pipeline: ${{ parameters.buildimage_pipeline }}
-      artifact: 'sonic-buildimage.marvell-armhf1'
-      runVersion: specific
-      runId: 80637
-      path: '$(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}'
-    displayName: "Download sonic buildimage deb packages from 80637"
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
+      patterns: |
+        target/debs/buster/libnl-3-200_*.deb
+        target/debs/buster/libnl-3-dev_*.deb
+        target/debs/buster/libnl-genl-3-200_*.deb
+        target/debs/buster/libnl-genl-3-dev_*.deb
+        target/debs/buster/libnl-route-3-200_*.deb
+        target/debs/buster/libnl-route-3-dev_*.deb
+        target/debs/buster/libnl-nf-3-200_*.deb
+        target/debs/buster/libnl-nf-3-dev_*.deb
+    displayName: "Download common libs"
   - script: |
-      cd $(Build.SourcesDirectory)/${{ parameters.buildimage_artifact_name }}
-      sudo dpkg -i target/debs/buster/libnl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-genl-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-route-3-dev_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-200_*.deb
-      sudo dpkg -i target/debs/buster/libnl-nf-3-dev_*.deb
-      cd $(Build.SourcesDirectory)/${{ parameters.swss_common_artifact_name }}
-      sudo dpkg -i libswsscommon_1.0.0_${{ parameters.arch }}.deb
-      sudo dpkg -i libswsscommon-dev_1.0.0_${{ parameters.arch }}.deb
-      cd $(Build.SourcesDirectory)/${{ parameters.sairedis_artifact_name }}
-      sudo dpkg -i libsaivs_*.deb
-      sudo dpkg -i libsaivs-dev_*.deb
-      sudo dpkg -i libsairedis_*.deb
-      sudo dpkg -i libsairedis-dev_*.deb
-      sudo dpkg -i libsaimetadata_*.deb
-      sudo dpkg -i libsaimetadata-dev_*.deb
-      sudo dpkg -i syncd-vs_*.deb
-    workingDirectory: $(Pipeline.Workspace)
+      set -ex
+      cd download
+      sudo dpkg -i $(find target/debs/buster -type f)
+      sudo dpkg -i $(ls *.deb)
+      cd ..
+      rm -rf download
+    workingDirectory: $(Build.ArtifactStagingDirectory)
     displayName: "Install libnl3, sonic swss common and sairedis"
   - script: |
-      set -x
+      set -ex
       tar czf pytest.tgz tests
       cp -r pytest.tgz $(Build.ArtifactStagingDirectory)/
       if [ '${{ parameters.archive_gcov }}' == True ]; then

.azure-pipelines/docker-sonic-vs/Dockerfile

@@ -4,18 +4,12 @@ ARG docker_container_name
 
 ADD ["debs", "/debs"]
 
-RUN dpkg --purge python-swsscommon
-RUN dpkg --purge python3-swsscommon
-RUN dpkg --purge swss
-RUN dpkg --purge libsairedis
-RUN dpkg --purge libswsscommon
-RUN dpkg --purge libsaimetadata
-RUN dpkg --purge libsaivs
-RUN dpkg --purge syncd-vs
+RUN dpkg --purge python-swsscommon python3-swsscommon swss libsairedis sonic-db-cli libswsscommon libsaimetadata libsaivs syncd-vs
 
 RUN dpkg -i /debs/libswsscommon_1.0.0_amd64.deb
 RUN dpkg -i /debs/python-swsscommon_1.0.0_amd64.deb
 RUN dpkg -i /debs/python3-swsscommon_1.0.0_amd64.deb
+RUN dpkg -i /debs/sonic-db-cli_1.0.0_amd64.deb
 
 RUN dpkg -i /debs/libsaimetadata_1.0.0_amd64.deb
 RUN dpkg -i /debs/libsairedis_1.0.0_amd64.deb

.azure-pipelines/gcov.yml

@@ -14,7 +14,7 @@ parameters:
 
 - name: timeout
   type: number
-  default: 180
+  default: 240
 
 - name: sonic_slave
   type: string

.azure-pipelines/test-docker-sonic-vs-template.yml

@@ -31,25 +31,26 @@ jobs:
   - task: DownloadPipelineArtifact@2
     inputs:
       artifact: docker-sonic-vs
-    displayName: "Download docker sonic vs image"
-
+      path: $(Build.ArtifactStagingDirectory)/download
+    displayName: "Download pre-stage built docker-sonic-vs"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
       project: build
-      pipeline: 9
+      pipeline: Azure.sonic-swss-common
       artifact: sonic-swss-common.amd64.ubuntu20_04
       runVersion: 'latestFromBranch'
-      runBranch: 'refs/heads/master'
+      runBranch: 'refs/heads/$(BUILD_BRANCH)'
+      path: $(Build.ArtifactStagingDirectory)/download
     displayName: "Download sonic swss common deb packages"
 
   - script: |
-      set -x
+      set -ex
       sudo .azure-pipelines/build_and_install_module.sh
 
       sudo apt-get install -y libhiredis0.14
-      sudo dpkg -i --force-confask,confnew ../libswsscommon_1.0.0_amd64.deb || apt-get install -f
-      sudo dpkg -i ../python3-swsscommon_1.0.0_amd64.deb
+      sudo dpkg -i --force-confask,confnew $(Build.ArtifactStagingDirectory)/download/libswsscommon_1.0.0_amd64.deb || apt-get install -f
+      sudo dpkg -i $(Build.ArtifactStagingDirectory)/download/python3-swsscommon_1.0.0_amd64.deb
 
       # install packages for vs test
       sudo apt-get install -y net-tools bridge-utils vlan
@@ -58,8 +59,8 @@ jobs:
     displayName: "Install dependencies"
 
   - script: |
-      set -x
-      sudo docker load -i ../docker-sonic-vs.gz
+      set -ex
+      sudo docker load -i $(Build.ArtifactStagingDirectory)/download/docker-sonic-vs.gz
       docker ps
       ip netns list
       uname -a
@@ -68,15 +69,27 @@ jobs:
       pushd tests
 
       if [ '${{ parameters.archive_gcov }}' == True ]; then
-        sudo py.test -v --force-flaky --junitxml=tr.xml --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+        all_tests=$(ls test_*.py)
+        all_tests="${all_tests} p4rt"
+        for test in ${all_tests}; do
+          test_name=$(echo "${test}" | cut -d "." -f 1)
+          sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) ${test}
+          container_count=$(docker ps -q -a | wc -l)
+          if [ ${container_count} -gt 0 ]; then
+            ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
+            docker stop $(docker ps -q -a)
+            docker rm $(docker ps -q -a)
+          fi
+        done
       else
-        sudo py.test -v --force-flaky --junitxml=tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+        sudo py.test -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
       fi
+      rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
 
   - task: PublishTestResults@2
     inputs:
-      testResultsFiles: '**/tr.xml'
+      testResultsFiles: '**/*_tr.xml'
       testRunTitle: vstest
     condition: always()
 
@@ -85,17 +98,17 @@ jobs:
 
       if [ '${{ parameters.archive_gcov }}' == True ]; then
         sudo apt-get install -y lcov
-        ./tests/gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
-        docker stop $(docker ps -q -a)
-        docker rm $(docker ps -q -a)
+        cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
+        tar -zcvf sonic-gcov.tar.gz sonic-gcov/
+        rm -rf sonic-gcov
       fi
     displayName: "Collect logs"
     condition: always()
 
   - publish: $(Build.ArtifactStagingDirectory)/gcov_tmp
     artifact: ${{ parameters.gcov_artifact_name }}
     displayName: "Publish gcov output"
-    condition: eq('${{ parameters.archive_gcov }}', true)
+    condition: and(succeeded(), eq('${{ parameters.archive_gcov }}', true))
 
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)