From b52807c44d692fa29185ec928a09cf7470aee7a2 Mon Sep 17 00:00:00 2001 From: Danny Allen Date: Fri, 31 Jul 2020 10:06:24 -0700 Subject: [PATCH] [kvm] Use run_tests.sh script in sonic-mgmt to run PR tests (#139) Signed-off-by: Danny Allen --- jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile | 4 +- jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile | 4 +- .../Jenkinsfile | 4 +- jenkins/vs/buildimage-vs-image-pr/Jenkinsfile | 4 +- .../vs/buildimage-vs-image-test/Jenkinsfile | 4 +- jenkins/vs/buildimage-vs-image/Jenkinsfile | 4 +- scripts/vs/buildimage-vs-image/runtest.sh | 103 +++++++----------- scripts/vs/buildimage-vs-image/test.sh | 2 +- 8 files changed, 51 insertions(+), 78 deletions(-) diff --git a/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile index a51ae50bcd88..062aff6ca2c6 100644 --- a/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile +++ b/jenkins/mgmt/sonic-mgmt-canary/Jenkinsfile @@ -39,8 +39,8 @@ pipeline { post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') - archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') } } } diff --git a/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile b/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile index 202dd8b8e279..63be124f4952 100644 --- a/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile +++ b/jenkins/mgmt/sonic-mgmt-pr/Jenkinsfile @@ -36,8 +36,8 @@ pipeline { post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') - archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') } } } diff --git a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile index 0a301a091081..c99839db2751 100644 --- a/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-201911-test/Jenkinsfile @@ -67,8 +67,8 @@ pipeline { post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') - archiveArtifacts(artifacts: 'sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') } fixed { diff --git a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile index 71c7edd16a72..b998f8eafa28 100644 --- a/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-pr/Jenkinsfile @@ -87,8 +87,8 @@ sudo cp ../target/sonic-vs.bin /nfs/jenkins/sonic-vs-${JOB_NAME##*/}.${BUILD_NUM post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') - archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') + archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') } } } diff --git a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile index 3ecd1004cd20..4e65ee326c8c 100644 --- a/jenkins/vs/buildimage-vs-image-test/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image-test/Jenkinsfile @@ -67,8 +67,8 @@ pipeline { post { always { - archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, sonic-mgmt/tests/results/**, kvmdump/**, ptfdump/**') - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') + archiveArtifacts(artifacts: 'sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') } fixed { diff --git a/jenkins/vs/buildimage-vs-image/Jenkinsfile b/jenkins/vs/buildimage-vs-image/Jenkinsfile index 7c0e454e99db..5c89e75fda26 100644 --- a/jenkins/vs/buildimage-vs-image/Jenkinsfile +++ b/jenkins/vs/buildimage-vs-image/Jenkinsfile @@ -100,8 +100,8 @@ sudo cp ../target/sonic-vs-dbg.bin /nfs/jenkins/sonic-vs-dbg-${JOB_NAME##*/}.${B post { always { - junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/results/**/*.xml') - archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/results/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') + junit(allowEmptyResults: true, keepLongStdio: true, testResults: 'sonic-mgmt/tests/logs/**/tr.xml') + archiveArtifacts(artifacts: 'target/**, sonic-mgmt/tests/logs/**, kvmdump/**, ptfdump/**') } } diff --git a/scripts/vs/buildimage-vs-image/runtest.sh b/scripts/vs/buildimage-vs-image/runtest.sh index 7be83398b1e3..b96f5c7cff6b 100755 --- a/scripts/vs/buildimage-vs-image/runtest.sh +++ b/scripts/vs/buildimage-vs-image/runtest.sh @@ -1,25 +1,7 @@ #!/bin/bash -xe tbname=$1 - -run_pytest() -{ - tgname=$1 - shift - tests=$@ - - echo "run tests: $tests" - - mkdir -p logs/$tgname - for tn in ${tests}; do - tdir=$(dirname $tn) - if [ $tdir != "." ]; then - mkdir -p logs/$tgname/$tdir - mkdir -p results/$tgname/$tdir - fi - py.test $PYTEST_COMMON_OPTS --log-file logs/$tgname/$tn.log --junitxml=results/$tgname/$tn.xml $tn.py - done -} +dut=$2 cd $HOME mkdir -p .ssh @@ -40,54 +22,48 @@ export ANSIBLE_LIBRARY=/data/sonic-mgmt/ansible/library/ # workaround for issue https://github.com/Azure/sonic-mgmt/issues/1659 export export ANSIBLE_KEEP_REMOTE_FILES=1 -PYTEST_COMMON_OPTS="--inventory veos.vtb \ - --host-pattern all \ - --user admin \ - -vvv \ - --show-capture stdout \ - --testbed $tbname \ - --testbed_file vtestbed.csv \ - --disable_loganalyzer \ - --log-file-level debug" +PYTEST_CLI_COMMON_OPTS="\ + -i veos.vtb \ + -d $dut \ + -n $tbname \ + -f vtestbed.csv \ + -k debug \ + -l warning \ + -m group \ + -e --disable_loganalyzer +" -# Check testbed health cd /data/sonic-mgmt/tests -rm -rf logs results +rm -rf logs mkdir -p logs -mkdir -p results -py.test $PYTEST_COMMON_OPTS --log-file logs/test_nbr_health.log --junitxml=results/tr.xml test_nbr_health.py - -# Run anounce route test case in order to populate BGP route -py.test $PYTEST_COMMON_OPTS --log-file logs/test_announce_routes.log --junitxml=results/tr.xml test_announce_routes.py -# Tests to run using one vlan configuration +# Run tests_1vlan on vlab-01 virtual switch +# TODO: Use a marker to select these tests rather than providing a hard-coded list here. tgname=1vlan tests="\ - test_interfaces \ - pc/test_po_update \ - bgp/test_bgp_fact \ - lldp/test_lldp \ - route/test_default_route \ - bgp/test_bgp_speaker \ - bgp/test_bgp_gr_helper \ - dhcp_relay/test_dhcp_relay \ - syslog/test_syslog \ - tacacs/test_rw_user \ - tacacs/test_ro_user \ - ntp/test_ntp \ - cacl/test_cacl_application \ - cacl/test_cacl_function \ - telemetry/test_telemetry \ - snmp/test_snmp_cpu \ - snmp/test_snmp_interfaces \ - snmp/test_snmp_lldp \ - snmp/test_snmp_pfc_counters \ - snmp/test_snmp_queue -" +test_interfaces.py \ +bgp/test_bgp_fact.py \ +bgp/test_bgp_gr_helper.py \ +bgp/test_bgp_speaker.py \ +cacl/test_cacl_application.py \ +cacl/test_cacl_function.py \ +dhcp_relay/test_dhcp_relay.py \ +lldp/test_lldp.py \ +ntp/test_ntp.py \ +pc/test_po_update.py \ +route/test_default_route.py \ +snmp/test_snmp_cpu.py \ +snmp/test_snmp_interfaces.py \ +snmp/test_snmp_lldp.py \ +snmp/test_snmp_pfc_counters.py \ +snmp/test_snmp_queue.py \ +syslog/test_syslog.py \ +tacacs/test_rw_user.py \ +tacacs/test_ro_user.py \ +telemetry/test_telemetry.py" -# Run tests_1vlan on vlab-01 virtual switch pushd /data/sonic-mgmt/tests -run_pytest $tgname $tests +./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname popd # Create and deploy two vlan configuration (two_vlan_a) to the virtual switch @@ -95,13 +71,10 @@ cd /data/sonic-mgmt/ansible ./testbed-cli.sh -m veos.vtb -t vtestbed.csv deploy-mg $tbname lab password.txt -e vlan_config=two_vlan_a sleep 180 -# Tests to run using two vlan configuration +# Run tests_2vlans on vlab-01 virtual switch tgname=2vlans -tests="\ - dhcp_relay/test_dhcp_relay \ -" +tests="dhcp_relay/test_dhcp_relay.py" -# Run tests_2vlans on vlab-01 virtual switch pushd /data/sonic-mgmt/tests -run_pytest $tgname $tests +./run_tests.sh $PYTEST_CLI_COMMON_OPTS -c "$tests" -p logs/$tgname popd diff --git a/scripts/vs/buildimage-vs-image/test.sh b/scripts/vs/buildimage-vs-image/test.sh index 5bbf09a4ec98..4bb2f8c09b1a 100755 --- a/scripts/vs/buildimage-vs-image/test.sh +++ b/scripts/vs/buildimage-vs-image/test.sh @@ -28,7 +28,7 @@ cd sonic-mgmt/ansible sed -i s:use_own_value:johnar: veos.vtb echo abc > password.txt cd ../../ -docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname +docker run --rm=true -v $(pwd):/data -w /data -i sonicdev-microsoft.azurecr.io:443/docker-sonic-mgmt ./scripts/vs/buildimage-vs-image/runtest.sh $tbname $dut # save dut state if test fails if [ $? != 0 ]; then