[ci][asan] add DVS tests run with ASAN (sonic-net#2363)

This adds 3 new stages: 1. BuildAsan: same as 'Build' but with ENABLE_ASAN=y 2. BuildDockerAsan: same as 'BuildDocker' but uses asan-enabled swss and adds swss-dbg package. It also uses asan-enabled docker-sonic-vs from the sonic-buildimage ci. 3. TestAsan: same as 'Test' but uses asan-enabled docker and publishes the asan reports if any. - What I did Added new CI stages to run the DVS tests with ASAN and check the ASAN reports. - Why I did it To utilize ASAN for catching memory usage issues on PR. - How I verified it Run the CI on my fork Signed-off-by: Yakiv Huryk <yhuryk@nvidia.com>
tshalvi · Aug 24, 2022 · ca9edca · ca9edca
1 parent dec4570
commit ca9edca
Show file tree

Hide file tree

Showing 5 changed files with 97 additions and 11 deletions.
diff --git a/.azure-pipelines/build-docker-sonic-vs-template.yml b/.azure-pipelines/build-docker-sonic-vs-template.yml
@@ -23,6 +23,10 @@ parameters:
 - name: artifact_name
   type: string
 
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -66,27 +70,32 @@ jobs:
       runVersion: 'latestFromBranch'
       runBranch: 'refs/heads/$(BUILD_BRANCH)'
       path: $(Build.ArtifactStagingDirectory)/download
-      patterns: '**/target/docker-sonic-vs.gz'
-    displayName: "Download sonic-buildimage docker-sonic-vs"
+      patterns: '**/target/${{ parameters.artifact_name }}.gz'
+    displayName: "Download sonic-buildimage ${{ parameters.artifact_name }}"
   - script: |
       set -ex
       echo $(Build.DefinitionName).$(Build.BuildNumber)
 
-      docker load < $(Build.ArtifactStagingDirectory)/download/target/docker-sonic-vs.gz
+      docker load < $(Build.ArtifactStagingDirectory)/download/target/${{ parameters.artifact_name }}.gz
 
       mkdir -p .azure-pipelines/docker-sonic-vs/debs
 
       cp -v $(Build.ArtifactStagingDirectory)/download/*.deb .azure-pipelines/docker-sonic-vs/debs
 
       pushd .azure-pipelines
 
-      docker build --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) docker-sonic-vs
+      build_args=""
+      if [ '${{ parameters.asan }}' == True ]; then
+        build_args="--build-arg need_dbg=y"
+      fi
+
+      docker build $build_args --no-cache -t docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} docker-sonic-vs
 
       popd
 
-      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber) | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
+      docker save docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }} | gzip -c > $(Build.ArtifactStagingDirectory)/docker-sonic-vs.gz
       rm -rf $(Build.ArtifactStagingDirectory)/download
-    displayName: "Build docker-sonic-vs"
+    displayName: "Build ${{ parameters.artifact_name }}"
   - publish: $(Build.ArtifactStagingDirectory)/
     artifact: ${{ parameters.artifact_name }}
     displayName: "Archive sonic docker vs image"
diff --git a/.azure-pipelines/build-template.yml b/.azure-pipelines/build-template.yml
@@ -43,6 +43,10 @@ parameters:
 - name: common_lib_artifact_name
   type: string
 
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: ${{ parameters.arch }}
@@ -138,6 +142,9 @@ jobs:
       if [ '${{ parameters.archive_gcov }}' == True ]; then
          export ENABLE_GCOV=y
       fi
+      if [ '${{ parameters.asan }}' == True ]; then
+         export ENABLE_ASAN=y
+      fi
       ./autogen.sh
       dpkg-buildpackage -us -uc -b -j$(nproc) && cp ../*.deb .
     displayName: "Compile sonic swss"

diff --git a/.azure-pipelines/docker-sonic-vs/Dockerfile b/.azure-pipelines/docker-sonic-vs/Dockerfile
@@ -1,6 +1,7 @@
 FROM docker-sonic-vs
 
 ARG docker_container_name
+ARG need_dbg
 
 ADD ["debs", "/debs"]
 
@@ -18,6 +19,7 @@ RUN dpkg -i /debs/syncd-vs_1.0.0_amd64.deb
 
 RUN dpkg --purge swss
 RUN dpkg -i /debs/swss_1.0.0_amd64.deb
+RUN if [ "$need_dbg" = "y" ] ; then dpkg -i /debs/swss-dbg_1.0.0_amd64.deb ; fi
 
 RUN apt-get update
 

diff --git a/.azure-pipelines/test-docker-sonic-vs-template.yml b/.azure-pipelines/test-docker-sonic-vs-template.yml
@@ -16,6 +16,14 @@ parameters:
   type: boolean
   default: false
 
+- name: docker_sonic_vs_name
+  type: string
+  default: docker-sonic-vs
+
+- name: asan
+  type: boolean
+  default: false
+
 jobs:
 - job:
   displayName: vstest
@@ -30,9 +38,9 @@ jobs:
   - checkout: self
   - task: DownloadPipelineArtifact@2
     inputs:
-      artifact: docker-sonic-vs
+      artifact: ${{ parameters.docker_sonic_vs_name }}
       path: $(Build.ArtifactStagingDirectory)/download
-    displayName: "Download pre-stage built docker-sonic-vs"
+    displayName: "Download pre-stage built ${{ parameters.docker_sonic_vs_name }}"
   - task: DownloadPipelineArtifact@2
     inputs:
       source: specific
@@ -77,7 +85,7 @@ jobs:
           test_set+=("${test}")
           if [ ${#test_set[@]} -ge 20 ]; then
             test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-            echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+            echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
             container_count=$(docker ps -q -a | wc -l)
             if [ ${container_count} -gt 0 ]; then
               ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
@@ -89,7 +97,7 @@ jobs:
         done
         if [ ${#test_set[@]} -gt 0 ]; then
           test_name=$(echo "${test_set[0]}" | cut -d "." -f 1)
-          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+          echo "${test_set[*]}" | xargs sudo py.test -v --force-flaky --junitxml="${test_name}_tr.xml" --keeptb --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
           container_count=$(docker ps -q -a | wc -l)
           if [ ${container_count} -gt 0 ]; then
             ./gcov_support.sh set_environment $(Build.ArtifactStagingDirectory)
@@ -98,8 +106,13 @@ jobs:
           fi
         fi
       else
-        sudo py.test -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber)
+        test_args=""
+        if [ '${{ parameters.asan }}' == True ]; then
+          test_args="--graceful-stop"
+        fi
+        sudo py.test $test_args -v --force-flaky --junitxml=tests_tr.xml --imgname=docker-sonic-vs:$(Build.DefinitionName).$(Build.BuildNumber).asan-${{ parameters.asan }}
       fi
+
       rm -rf $(Build.ArtifactStagingDirectory)/download
     displayName: "Run vs tests"
 
@@ -112,6 +125,10 @@ jobs:
   - script: |
       cp -r tests/log $(Build.ArtifactStagingDirectory)/
 
+      if [ '${{ parameters.asan }}' == True ]; then
+        cp -r tests/log/*/log/asan $(Build.ArtifactStagingDirectory)/
+      fi
+
       if [ '${{ parameters.archive_gcov }}' == True ]; then
         sudo apt-get install -y lcov
         cd $(Build.ArtifactStagingDirectory)/gcov_tmp/
@@ -130,3 +147,17 @@ jobs:
     artifact: ${{ parameters.log_artifact_name }}@$(System.JobAttempt)
     displayName: "Publish logs"
     condition: always()
+
+  - publish: $(Build.ArtifactStagingDirectory)/asan
+    artifact: asan-reports
+    displayName: "Publish ASAN reports"
+    condition: eq('${{ parameters.asan }}', true)
+
+  - script: |
+      if [ "$(ls -A $(Build.ArtifactStagingDirectory)/asan)" ]; then
+        echo "There are issues reported by ASAN"
+      else
+        echo "No issues reported by ASAN"
+      fi
+    displayName: "Check ASAN reports"
+    condition: eq('${{ parameters.asan }}', true)
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
@@ -49,6 +49,19 @@ stages:
       archive_pytests: true
       archive_gcov: true
 
+- stage: BuildAsan
+
+  jobs:
+  - template: .azure-pipelines/build-template.yml
+    parameters:
+      arch: amd64
+      sonic_slave: sonic-slave-buster
+      common_lib_artifact_name: common-lib
+      swss_common_artifact_name: sonic-swss-common
+      sairedis_artifact_name: sonic-sairedis
+      artifact_name: sonic-swss-asan
+      asan: true
+
 - stage: BuildArm
   dependsOn: Build
   condition: succeeded('Build')
@@ -88,6 +101,18 @@ stages:
       swss_artifact_name: sonic-swss
       artifact_name: docker-sonic-vs
 
+- stage: BuildDockerAsan
+  dependsOn: BuildAsan
+  condition: succeeded('BuildAsan')
+  jobs:
+  - template: .azure-pipelines/build-docker-sonic-vs-template.yml
+    parameters:
+      swss_common_artifact_name: sonic-swss-common
+      sairedis_artifact_name: sonic-sairedis
+      swss_artifact_name: sonic-swss-asan
+      artifact_name: docker-sonic-vs-asan
+      asan: true
+
 - stage: Test
   dependsOn: BuildDocker
   condition: succeeded('BuildDocker')
@@ -99,6 +124,18 @@ stages:
       sonic_slave: sonic-slave-buster
       archive_gcov: true
 
+- stage: TestAsan
+  dependsOn: BuildDockerAsan
+  condition: succeeded('BuildDockerAsan')
+  jobs:
+  - template: .azure-pipelines/test-docker-sonic-vs-template.yml
+    parameters:
+      log_artifact_name: log-asan
+      gcov_artifact_name: sonic-gcov
+      sonic_slave: sonic-slave-buster
+      docker_sonic_vs_name: docker-sonic-vs-asan
+      asan: true
+
 - stage: Gcov
   dependsOn: Test
   condition: always()