[kvm] Use run_tests.sh script in sonic-mgmt to run PR tests (sonic-ne…

…t#139) Signed-off-by: Danny Allen <daall@microsoft.com>
xumia · Jul 31, 2020 · b52807c · b52807c
1 parent d3c45ad
commit b52807c
Show file tree

Hide file tree

Showing 8 changed files with 51 additions and 78 deletions.
diff --git a/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile
@@ -39,8 +39,8 @@ pipeline {
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

diff --git a/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile
@@ -36,8 +36,8 @@ pipeline {
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+                    archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

diff --git a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile
@@ -67,8 +67,8 @@ pipeline {
     post {
 
         always {
-            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
         }
 
         fixed {

diff --git a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile
@@ -87,8 +87,8 @@ sudo cp ../target/sonic-vs.bin /nfs/jenkins/sonic-vs-${JOB_NAME##*/}.${BUILD_NUM
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
         }

diff --git a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile
@@ -67,8 +67,8 @@ pipeline {
     post {
 
         always {
-            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**, kvmdump/**, ptfdump/**')
-            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
+            archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+            junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
         }
 
         fixed {

diff --git a/jenkins/vs/buildimage-vs-image/Jenkinsfile b/jenkins/vs/buildimage-vs-image/Jenkinsfile
@@ -100,8 +100,8 @@ sudo cp ../target/sonic-vs-dbg.bin /nfs/jenkins/sonic-vs-dbg-${JOB_NAME##*/}.${B
 
             post {
                 always {
-                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml')
-                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
+                    junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml')
+                    archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**')
                 }
             }
 

diff --git a/scripts/vs/buildimage-vs-image/runtest.sh b/scripts/vs/buildimage-vs-image/runtest.sh
@@ -1,25 +1,7 @@
 #!/bin/bash -xe
 
 tbname=$1
-
-run_pytest()
-{
-    tgname=$1
-    shift
-    tests=$@
-
-    echo "run tests: $tests"
-
-    mkdir -p logs/$tgname
-    for tn in ${tests}; do
-        tdir=$(dirname $tn)
-        if [ $tdir != "." ]; then
-            mkdir -p logs/$tgname/$tdir
-            mkdir -p results/$tgname/$tdir
-        fi
-        py.test $PYTEST_COMMON_OPTS --log-file logs/$tgname/$tn.log --junitxml=results/$tgname/$tn.xml $tn.py
-    done
-}
+dut=$2
 
 cd $HOME
 mkdir -p .ssh
@@ -40,68 +22,59 @@ export ANSIBLE_LIBRARY=/data/sonic-mgmt/ansible/library/
 # workaround for issue https://github.com/Azure/sonic-mgmt/issues/1659
 export export ANSIBLE_KEEP_REMOTE_FILES=1
 
-PYTEST_COMMON_OPTS="--inventory veos.vtb \
-                    --host-pattern all \
-                    --user admin \
-                    -vvv \
-                    --show-capture stdout \
-                    --testbed $tbname \
-                    --testbed_file vtestbed.csv \
-                    --disable_loganalyzer \
-                    --log-file-level debug"
+PYTEST_CLI_COMMON_OPTS="\
+    -i veos.vtb \
+    -d $dut \
+    -n $tbname \
+    -f vtestbed.csv \
+    -k debug \
+    -l warning \
+    -m group \
+    -e --disable_loganalyzer
+"
 
-# Check testbed health
 cd /data/sonic-mgmt/tests
-rm -rf logs results
+rm -rf logs
 mkdir -p logs
-mkdir -p results
-py.test $PYTEST_COMMON_OPTS --log-file logs/test_nbr_health.log --junitxml=results/tr.xml test_nbr_health.py
-
-# Run anounce route test case in order to populate BGP route
-py.test $PYTEST_COMMON_OPTS --log-file logs/test_announce_routes.log --junitxml=results/tr.xml test_announce_routes.py
 
-# Tests to run using one vlan configuration
+# Run tests_1vlan on vlab-01 virtual switch
+# TODO: Use a marker to select these tests rather than providing a hard-coded list here.
 tgname=1vlan
 tests="\
-    test_interfaces \
-    pc/test_po_update \
-    bgp/test_bgp_fact \
-    lldp/test_lldp \
-    route/test_default_route \
-    bgp/test_bgp_speaker \
-    bgp/test_bgp_gr_helper \
-    dhcp_relay/test_dhcp_relay \
-    syslog/test_syslog \
-    tacacs/test_rw_user \
-    tacacs/test_ro_user \
-    ntp/test_ntp \
-    cacl/test_cacl_application \
-    cacl/test_cacl_function \
-    telemetry/test_telemetry \
-    snmp/test_snmp_cpu \
-    snmp/test_snmp_interfaces \
-    snmp/test_snmp_lldp \
-    snmp/test_snmp_pfc_counters \
-    snmp/test_snmp_queue
-"
+test_interfaces.py \
+bgp/test_bgp_fact.py \
+bgp/test_bgp_gr_helper.py \
+bgp/test_bgp_speaker.py \
+cacl/test_cacl_application.py \
+cacl/test_cacl_function.py \
+dhcp_relay/test_dhcp_relay.py \
+lldp/test_lldp.py \
+ntp/test_ntp.py \
+pc/test_po_update.py \
+route/test_default_route.py \
+snmp/test_snmp_cpu.py \
+snmp/test_snmp_interfaces.py \
+snmp/test_snmp_lldp.py \
+snmp/test_snmp_pfc_counters.py \
+snmp/test_snmp_queue.py \
+syslog/test_syslog.py \
+tacacs/test_rw_user.py \
+tacacs/test_ro_user.py \
+telemetry/test_telemetry.py"
 
-# Run tests_1vlan on vlab-01 virtual switch
 pushd /data/sonic-mgmt/tests
-run_pytest $tgname $tests
+./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname
 popd
 
 # Create and deploy two vlan configuration (two_vlan_a) to the virtual switch
 cd /data/sonic-mgmt/ansible
 ./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg $tbname lab password.txt -e vlan_config=two_vlan_a
 sleep 180
 
-# Tests to run using two vlan configuration
+# Run tests_2vlans on vlab-01 virtual switch
 tgname=2vlans
-tests="\
-    dhcp_relay/test_dhcp_relay \
-"
+tests="dhcp_relay/test_dhcp_relay.py"
 
-# Run tests_2vlans on vlab-01 virtual switch
 pushd /data/sonic-mgmt/tests
-run_pytest $tgname $tests
+./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname
 popd
diff --git a/scripts/vs/buildimage-vs-image/test.sh b/scripts/vs/buildimage-vs-image/test.sh
@@ -28,7 +28,7 @@ cd sonic-mgmt/ansible
 sed -i s:use_own_value:johnar: veos.vtb
 echo abc > password.txt
 cd ../../
-docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname
+docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname $dut
 
 # save dut state if test fails
 if [ $? != 0 ]; then