From c20704243258e4b82c230dbe3f6602a7c8d9381c Mon Sep 17 00:00:00 2001 From: Hanumanth Pothula Date: Tue, 24 Sep 2024 02:23:05 +0530 Subject: [PATCH] ci: add multiple vf test case Add test case to verify multiple SDP VFs Signed-off-by: Hanumanth Pothula Change-Id: Ib17dd64a6ccb23d1c74fb9e6aaaf9b4fe85ecb78 Reviewed-on: https://sj1git1.cavium.com/c/IP/SW/dataplane/dpu-offload/+/136154 Tested-by: sa_ip-toolkits-Jenkins Reviewed-by: Harman Kalra --- ci/test/dao-test/common/ep_common_ops.sh | 96 ++++++++++++++++- ci/test/dao-test/common/ep_host_utils.sh | 49 +++++++++ ci/test/dao-test/ovs/meson.build | 6 ++ ci/test/dao-test/ovs/ovs_ping.sh | 129 +++++++++++++++++++---- ci/test/dao-test/ovs/ovs_utils.sh | 31 ++++-- 5 files changed, 280 insertions(+), 31 deletions(-) diff --git a/ci/test/dao-test/common/ep_common_ops.sh b/ci/test/dao-test/common/ep_common_ops.sh index 50db68b..aa7a794 100755 --- a/ci/test/dao-test/common/ep_common_ops.sh +++ b/ci/test/dao-test/common/ep_common_ops.sh @@ -88,12 +88,13 @@ function ep_common_if_configure() local vxlan_remote_ip= local vxlan_local_ip= local vxlan_vni= + local num_alias= local vlan_id= local mtu= if ! opts=$(getopt \ - -l "ip:,pcie-addr:,down,vxlan-remote-ip:,vxlan-local-ip:,vxlan-vni:,vlan-id:,mtu:" \ - -- configure_sdp_interface $@); then + -l "ip:,pcie-addr:,down,vxlan-remote-ip:,vxlan-local-ip:,vxlan-vni:,vlan-id:,mtu:,\ + alias:" -- configure_sdp_interface $@); then echo "Failed to parse arguments" exit 1 fi @@ -107,6 +108,7 @@ function ep_common_if_configure() --vxlan-remote-ip) shift; vxlan_remote_ip=$1;; --vxlan-local-ip) shift; vxlan_local_ip=$1;; --vlan-id) shift; vlan_id=$1;; + --alias) shift; num_alias=$1;; --down) down=1;; --mtu) shift; mtu=$1;; *) echo "Invalid argument $1"; exit 1;; @@ -120,10 +122,28 @@ function ep_common_if_configure() exit fi - ep_common_cleanup_interfaces $iface_name + if [[ -z $num_alias ]]; then + ep_common_cleanup_interfaces $iface_name + fi + if [[ -z $down ]]; then - if [[ -n $vlan_id ]]; then + if [[ -n $num_alias ]]; then + IFS='.' read -r -a ip_parts <<< "$ip_addr" + for ((i=0; i<$num_alias; i++)); do + ip="${ip_parts[0]}.${ip_parts[1]}.${ip_parts[2]}.${ip_parts[3]}" + if [[ -n $vlan_id ]]; then + vlan_id=$((vlan_id + 1)) + ip link add link $iface_name name \ + $iface_name.v$vlan_id type vlan id $vlan_id + ip link set dev $iface_name.v$vlan_id up + ip addr add $ip/24 dev $iface_name.v$vlan_id + else + ifconfig $iface_name:$i $ip/24 + fi + ((ip_parts[3]++)) + done + elif [[ -n $vlan_id ]]; then nmcli dev set $iface_name managed no &> /dev/null || true ifconfig $iface_name up ifconfig $iface_name 0 @@ -181,6 +201,31 @@ function ep_common_ping() fi } +function ep_common_multiple_pings() +{ + local host_ip=$1 + local remote_ip=$2 + local num_ifs=$3 + local remote_if + + IFS='.' read -r -a hip <<< "$host_ip" + IFS='.' read -r -a rip <<< "$remote_ip" + + for ((i=0; i<$num_ifs; i++)); do + if [[ $(ep_common_ping $host_ip $remote_ip) != "SUCCESS" ]]; then + echo "FAILURE" + exit 1 + fi + + ((hip[3]++)) + ((rip[3]++)) + host_ip="${hip[0]}.${hip[1]}.${hip[2]}.${hip[3]}" + remote_ip="${rip[0]}.${rip[1]}.${rip[2]}.${rip[3]}" + done + + echo "SUCCESS" +} + ep_common_cleanup_interfaces() { local prefix=$1 @@ -192,10 +237,53 @@ ep_common_cleanup_interfaces() done } +ep_host_clean_sdp_host_ifcs() +{ + local sdp_vfs="$@" + + for vf in $sdp_vfs; do + iface=$(ep_common_if_name_get $vf) + ep_common_cleanup_interfaces $iface + done +} + +ep_common_cleanup_alias_ifcs() +{ + local pci_addr=$1 + local num_alias=$2 + local ip=$3 + local test_type=$4 + local vlan_id=$5 + local iface=$(ep_common_if_name_get $pci_addr) + + IFS='.' read -r -a ip_parts <<< "$ip" + + if [[ $test_type == "vlan" ]]; then + ip link del $iface.v$vlan_id + ((vlan_id++)) + fi + + for ((i=0; i<$num_alias; i++)); do + if [[ $test_type == "plane" ]]; then + ip addr del $ip/24 dev $iface:$i + elif [[ $test_type == "vlan" ]]; then + ip link del $iface.v$vlan_id + ((vlan_id++)) + fi + ((ip_parts[3]++)) + ip="${ip_parts[0]}.${ip_parts[1]}.${ip_parts[2]}.${ip_parts[3]}" + done + + ep_common_cleanup_interfaces $iface +} + function ep_common_set_numvfs() { local dev=$1 local numvfs=$2 + local maxvfs=$(cat /sys/bus/pci/devices/$dev/sriov_totalvfs) + + numvfs=$((numvfs > maxvfs ? maxvfs : numvfs)) echo 0 > /sys/bus/pci/devices/$dev/sriov_numvfs sleep 1 diff --git a/ci/test/dao-test/common/ep_host_utils.sh b/ci/test/dao-test/common/ep_host_utils.sh index df15d97..c9a61f0 100755 --- a/ci/test/dao-test/common/ep_host_utils.sh +++ b/ci/test/dao-test/common/ep_host_utils.sh @@ -342,6 +342,55 @@ function ep_host_shutdown_guest() sleep 10; } +function ep_host_sdp_vf_setup() +{ + local sdp_pf=$1 + local num_vf=$2 + local sdp_vfs + + set +e # Module may be already loaded + if [[ -n ${EP_HOST_MODULE_DIR:-} ]]; then + insmod $EP_HOST_MODULE_DIR/octeon_ep_vf.ko + else + insmod $EP_DIR/ep_files/octeon_ep_vf.ko + fi + set -e + sleep 2 + + max_vfs=$(cat /sys/bus/pci/devices/$sdp_pf/sriov_totalvfs) + num_vf=$((num_vf > max_vfs ? max_vfs : num_vf)) + + echo 0 > /sys/bus/pci/devices/$sdp_pf/sriov_numvfs + sleep 3 + echo $num_vf > /sys/bus/pci/devices/$sdp_pf/sriov_numvfs + sleep 10 + + echo $(ep_common_pcie_addr_get "0xB903" all) +} + +function ep_host_sdp_vf_cleanup() +{ + local sdp_pf=$1 + + echo 0 > /sys/bus/pci/devices/$sdp_pf/sriov_numvfs + rmmod octeon_ep_vf +} + +function ep_host_sdp_vfs_ip_cnf() +{ + local arg1=("$@") + local ip="${!#}" + local sdp_vfs=${arg1[@]:0:${#arg1[@]}-1} + + IFS='.' read -r -a ip_parts <<< "$ip" + for vf in $sdp_vfs; do + vf_ip="${ip_parts[0]}.${ip_parts[1]}.${ip_parts[2]}.${ip_parts[3]}" + ep_common_if_configure --pcie-addr $vf --ip $vf_ip + ((ip_parts[3]++)) + done +} + + # If this script is directly invoked from the shell execute the # op specified if [[ ${BASH_SOURCE[0]} == ${0} ]]; then diff --git a/ci/test/dao-test/ovs/meson.build b/ci/test/dao-test/ovs/meson.build index bee394c..e5d6ce4 100644 --- a/ci/test/dao-test/ovs/meson.build +++ b/ci/test/dao-test/ovs/meson.build @@ -16,6 +16,12 @@ tests = [ ['ovs_plain_ping_jumbo_pkt_hw_offload', 'ovs_ping.sh', ''], ['ovs_vlan_ping_jumbo_pkt_hw_offload', 'ovs_ping.sh', ''], ['ovs_vxlan_ping_jumbo_pkt_hw_offload', 'ovs_ping.sh', ''], + ['ovs_plain_mul_vf_ping', 'ovs_ping.sh', ''], + ['ovs_vlan_mul_vf_ping', 'ovs_ping.sh', ''], + ['ovs_plain_mul_vf_ping_hw_offload', 'ovs_ping.sh', ''], + ['ovs_vlan_mul_vf_ping_hw_offload', 'ovs_ping.sh', ''], + ['ovs_vlan_neg_ping', 'ovs_ping.sh', ''], + ['ovs_vlan_neg_ping_hw_offload', 'ovs_ping.sh', ''], ] test_dir = meson.current_build_dir() diff --git a/ci/test/dao-test/ovs/ovs_ping.sh b/ci/test/dao-test/ovs/ovs_ping.sh index bd09a0e..5ba4584 100755 --- a/ci/test/dao-test/ovs/ovs_ping.sh +++ b/ci/test/dao-test/ovs/ovs_ping.sh @@ -7,17 +7,19 @@ set -euo pipefail OVS_PING_SCRIPT_PATH="$( cd -- "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" source $OVS_PING_SCRIPT_PATH/ovs_utils.sh +NUM_VF_PER_PF=3 + function ovs_ping() { local test_type=$1 - local hw_offload=$2 - local mtu=${3:-1500} - local pktsz=${4:-56} + local num_sdp_ifcs_per_eth=$2 + local hw_offload=$3 + local mtu=${4:-1500} + local pktsz=${5:-56} local eth_pf_ifcs local sdp_eth_vf_pairs local esw_vf_ifcs local num_eth_ifcs=1 - local num_sdp_ifcs_per_eth=2 local num_esw_ifcs=1 local ovs_debug=1 local ssh_ip=$(echo $EP_DEVICE | awk -F '\@' '{print $2}' 2>/dev/null) @@ -32,10 +34,16 @@ function ovs_ping() local vxlan_vni=5001 local vlan_id=100 local maxpktlen + local host_vf_base_ip="50.0.0.9" + local remote_alias_base_ip="50.0.0.99" + local sdp_host_vfs if [[ $test_type == "vlan" ]]; then extra_args_interface_setup="--vlan-id $vlan_id" extra_args_remote_ifconfig="--vlan-id $vlan_id" + elif [[ $test_type == "vlan-neg" ]]; then + extra_args_interface_setup="--vlan-id $vlan_id" + extra_args_remote_ifconfig="--vlan-id $((vlan_id+10))" elif [[ $test_type == "vxlan" ]]; then extra_args_interface_setup="--vxlan-vni $vxlan_vni --vxlan-subnet $vxlan_subnet" extra_args_remote_ifconfig="--vxlan-vni $vxlan_vni \ @@ -89,71 +97,154 @@ function ovs_ping() --ip $remote_ip $extra_args_remote_ifconfig if [[ $(ep_host_op ping $host_ip $remote_ip 32 $pktsz) != "SUCCESS" ]]; then - echo "Ping Failed" - exit 1 + echo "$test_type Failed" + if [[ $test_type == "vlan-neg" ]]; then + extra_args_remote_ifconfig="--vlan-id $vlan_id" + ep_remote_op if_configure --pcie-addr $EP_REMOTE_IFACE \ + --ip $remote_ip $extra_args_remote_ifconfig + if [[ $(ep_host_op ping $host_ip $remote_ip) != "SUCCESS" ]]; then + echo "Ping Failed" + exit 1 + else + echo "Ping Passed" + fi + else + echo "Ping Failed" + exit 1 + fi else - echo "Ping Passed" + if [[ $test_type == "vlan-neg" ]]; then + echo "Ping is successful with invalid VLAN-ID!!" + exit 1 + else + echo "Ping Passed" + fi + fi + + if [ "$num_sdp_ifcs_per_eth" -gt 1 ]; then + echo "Verify VF interfaces" + + num_host_vfs=$((num_sdp_ifcs_per_eth - 1)) + sdp_host_vfs=$(ep_host_op sdp_vf_setup $EP_HOST_SDP_IFACE $num_host_vfs) + + echo "Configure host vfs" + ep_host_op sdp_vfs_ip_cnf "$sdp_host_vfs" $host_vf_base_ip + + echo "Configure remote aliases" + ep_remote_op if_configure --pcie-addr $EP_REMOTE_IFACE \ + --ip $remote_alias_base_ip \ + --alias $num_host_vfs \ + $extra_args_remote_ifconfig + + echo "Start VF pings" + if [[ $(ep_host_op multiple_pings $host_vf_base_ip \ + $remote_alias_base_ip $num_host_vfs) != "SUCCESS" ]]; then + result="Failure" + else + result="SUCCESS" + fi + + ep_host_op clean_sdp_host_ifcs "$sdp_host_vfs" + ep_host_op sdp_vf_cleanup $EP_HOST_SDP_IFACE + ep_remote_op cleanup_alias_ifcs $EP_REMOTE_IFACE $num_host_vfs \ + $remote_alias_base_ip $test_type $vlan_id + + if [[ $result != "SUCCESS" ]]; then + echo "fail to ping VF interfaces" + exit 1 + fi + + echo "Test case passed!" fi } function ovs_plain_ping() { - ovs_ping plain false + ovs_ping plain 1 false } function ovs_vlan_ping() { - ovs_ping vlan false + ovs_ping vlan 1 false } function ovs_vxlan_ping() { - ovs_ping vxlan false + ovs_ping vxlan 1 false } function ovs_plain_ping_hw_offload() { - ovs_ping plain true + ovs_ping plain 1 true } function ovs_vlan_ping_hw_offload() { - ovs_ping vlan true + ovs_ping vlan 1 true } function ovs_vxlan_ping_hw_offload() { - ovs_ping vxlan true + ovs_ping vxlan 1 true } function ovs_plain_ping_jumbo_pkt() { - ovs_ping plain false 9000 8000 + ovs_ping plain 1 false 9000 8000 } function ovs_vlan_ping_jumbo_pkt() { - ovs_ping vlan false 9000 8000 + ovs_ping vlan 1 false 9000 8000 } function ovs_vxlan_ping_jumbo_pkt() { - ovs_ping vxlan false 9000 8000 + ovs_ping vxlan 1 false 9000 8000 } function ovs_plain_ping_jumbo_pkt_hw_offload() { - ovs_ping plain true 9000 8000 + ovs_ping plain 1 true 9000 8000 } function ovs_vlan_ping_jumbo_pkt_hw_offload() { - ovs_ping vlan true 9000 8000 + ovs_ping vlan 1 true 9000 8000 } function ovs_vxlan_ping_jumbo_pkt_hw_offload() { - ovs_ping vxlan true 9000 8000 + ovs_ping vxlan 1 true 9000 8000 +} + +function ovs_plain_mul_vf_ping() +{ + ovs_ping plain $NUM_VF_PER_PF false } +function ovs_vlan_mul_vf_ping() +{ + ovs_ping vlan $NUM_VF_PER_PF false +} + +function ovs_plain_mul_vf_ping_hw_offload() +{ + ovs_ping plain $NUM_VF_PER_PF true +} + +function ovs_vlan_mul_vf_ping_hw_offload() +{ + ovs_ping vlan $NUM_VF_PER_PF true +} + +function ovs_vlan_neg_ping() +{ + ovs_ping vlan-neg 1 false +} + +function ovs_vlan_neg_ping_hw_offload() +{ + ovs_ping vlan-neg 1 true +} test_run ${DAO_TEST} 2 diff --git a/ci/test/dao-test/ovs/ovs_utils.sh b/ci/test/dao-test/ovs/ovs_utils.sh index 3a69d8b..5803a88 100755 --- a/ci/test/dao-test/ovs/ovs_utils.sh +++ b/ci/test/dao-test/ovs/ovs_utils.sh @@ -205,7 +205,10 @@ function ovs_interface_setup() echo "List of bridges" ovs-vsctl list-br if [[ -n $vlan_id ]]; then - ovs-vsctl set port e0_vf_rep0 tag=$vlan_id + for ((i=0; i<$num_sdp_ifcs_per_eth; i++)); do + ovs-vsctl set port e0_vf_rep$i tag=$vlan_id + vlan_id=$((vlan_id + 1)) + done fi ovs-vsctl show } @@ -315,6 +318,7 @@ function ovs_offload_launch() local portconf="" local maxpktlen=0 local tmp + local first_ifc=true if ! opts=$(getopt \ -l "sdp-eth-vf-pair:,esw-vf-ifc:,max-pkt-len:" \ @@ -323,16 +327,23 @@ function ovs_offload_launch() exit 1 fi + max_cores=$num_cores + eval set -- "$opts" while [[ $# -gt 1 ]]; do case $1 in --sdp-eth-vf-pair) shift; # One additional core required for control thread - if [[ $num_cores -le 2 ]]; then - echo "Error: Number of cores: $num_cores not sufficient" - exit 1 + if [[ $num_cores -le 4 ]]; then + echo "Max cores reached, start reassiging cores" + num_cores=$max_cores fi sdp_vf=$(echo $1 | awk -F ',' '{print $1}'); + if [ "$first_ifc" = true ]; then + sdp_vf+=,max_pools=1024 + fi + first_ifc=flase + eth_vf=$(echo $1 | awk -F ',' '{print $2}'); allowlist="$allowlist -a $sdp_vf -a $eth_vf"; portmap="${portmap}(${1}),"; @@ -354,13 +365,17 @@ function ovs_offload_launch() portmap=${portmap::-1} portconf=${portconf::-1} - # 1 extra core for control thread - num_cores=$((num_cores - 1)) - coremask=$((coremask | 1 << num_cores)) + #Max supported coremask + coremask=$(((1 << max_cores) - 1 )) + coremask=$((coremask & ~0x03)) # Convert the coremask to hex coremask=$(printf "%x" $coremask) coremask="0x$coremask" + num_ports=$(((1 << $num_ports) - 1)) + num_ports=$(printf "%x" $num_ports) + num_ports="0x$num_ports" + find_executable "dao-ovs-offload" dao_offload "$OVS_UTILS_SCRIPT_PATH/../../../../app" echo "$dao_offload \ @@ -379,7 +394,7 @@ function ovs_offload_launch() --vfio-vf-token="$VFIO_TOKEN" \ --file-prefix=ep \ -- \ - -p 0xff \ + -p $num_ports \ --portmap="$portmap" \ --max-pkt-len=$maxpktlen \ --config="$portconf" &> $EP_DEVICE_OVS_PATH/var/log/dao-ovs-offload.log &