Add python3 virtual environment for docker-ptf

.gitmodules

@@ -103,3 +103,6 @@
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
 	url = https://github.com/Azure/sonic-pins.git
+[submodule "src/ptf-py3"]
+	path = src/ptf-py3
+	url = https://github.com/p4lang/ptf.git

dockers/docker-ptf/Dockerfile.j2

@@ -7,6 +7,11 @@ FROM {{ prefix }}multiarch/debian-debootstrap:arm64-stretch
 FROM {{ prefix }}debian:buster
 {% endif %}
 
+{% from "dockers/dockerfile-macros.j2" import install_python_wheels, copy_files %}
+
+USER root
+WORKDIR /root
+
 MAINTAINER Pavel Shirshov
 
 RUN echo "deb [arch=amd64] http://debian-archive.trafficmanager.net/debian buster-backports main" >> /etc/apt/sources.list
@@ -51,6 +56,13 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /
         python-libpcap      \
         python-scapy        \
         python-six          \
+        python3             \
+        python3-venv        \
+        python3-pip         \
+        python3-dev         \
+        python3-scapy       \
+        python3-six         \
+        libpcap-dev         \
         tacacs+             \
         rsyslog             \
         ntp                 \
@@ -59,7 +71,9 @@ RUN sed --in-place 's/httpredir.debian.org/debian-archive.trafficmanager.net/' /
         arping              \
         bridge-utils        \
         libteam-utils       \
-        gdb
+        gdb                 \
+        automake            \
+        iproute2
 
 # Install all python modules from pypi. python-scapy is exception, ptf debian package requires python-scapy
 # TODO: Clean up this step
@@ -117,10 +131,61 @@ RUN rm -rf /debs \
     && pip install pybrctl pyro4 rpyc yabgp \
     && pip install unittest-xml-reporting \
     && pip install pyrasite \
+    && pip install retrying \
     && mkdir -p /opt       \
     && cd /opt             \
     && wget https://raw.githubusercontent.com/p4lang/ptf/master/ptf_nn/ptf_nn_agent.py
 
+RUN python3 -m venv env-python3
+
+# Activating a virtualenv. The virtualenv automatically works for RUN, ENV and CMD.
+ENV VIRTUAL_ENV=/root/env-python3
+ARG BACKUP_OF_PATH="$PATH"
+ENV PATH="$VIRTUAL_ENV/bin:$PATH"
+
+ENV LANG=C.UTF-8 LC_ALL=C.UTF-8 PYTHONIOENCODING=UTF-8
+
+RUN python3 -m pip install --upgrade --ignore-installed pip
+
+# Install all python modules from pypi. python3-scapy is exception, ptf debian package requires python3-scapy
+RUN python3 -m pip install setuptools \
+    && pip3 install supervisor \
+    && pip3 install ipython==5.4.1 \
+    && pip3 install Cython \
+    && pip3 install cffi    \
+    && pip3 install nnpy    \
+    && pip3 install dpkt    \
+    && pip3 install ipaddress \
+    && pip3 install pysubnettree \
+    && pip3 install paramiko \
+    && pip3 install Flask   \
+    && pip3 install exabgp \
+    && pip3 install pyaml   \
+    && pip3 install pybrctl pyro4 rpyc yabgp \
+    && pip3 install unittest-xml-reporting \
+    && pip3 install pyrasite \
+    && pip3 install python-libpcap \
+    && pip3 install enum34 \
+    && pip3 install grpcio \
+    && pip3 install grpcio-tools \
+    && pip3 install protobuf \
+    && pip3 install six==1.16.0 \
+    && pip3 install itsdangerous \
+    && pip3 install retrying \
+    && pip3 install jinja2 \
+    && pip3 install scapy==2.4.5
+
+{% if docker_ptf_whls.strip() -%}
+# Copy locally-built Python wheel dependencies
+{{ copy_files("python-wheels/", docker_ptf_whls.split(' '), "/python-wheels/") }}
+
+# Install locally-built Python wheel dependencies
+{{ install_python_wheels(docker_ptf_whls.split(' ')) }}
+{% endif %}
+
+# Deactivating a virtualenv.
+ENV PATH="$BACKUP_OF_PATH"
+
 ## Adjust sshd settings
 RUN mkdir /var/run/sshd \
     && echo 'root:root' | chpasswd \

platform/vs/docker-ptf.mk

@@ -1,6 +1,7 @@
 # docker image for docker-ptf
 
 DOCKER_PTF = docker-ptf.gz
+$(DOCKER_PTF)_PYTHON_WHEELS += $(PTF_PY3)
 $(DOCKER_PTF)_PATH = $(DOCKERS_PATH)/docker-ptf
 $(DOCKER_PTF)_DEPENDS += $(LIBTHRIFT) $(PYTHON_THRIFT) $(PTF) $(PYTHON_SAITHRIFT)
 SONIC_DOCKER_IMAGES += $(DOCKER_PTF)

rules/ptf-py3.dep

@@ -0,0 +1,11 @@
+
+SPATH       := $($(PTF_PY3)_SRC_PATH)
+DEP_FILES   := $(SONIC_COMMON_FILES_LIST) rules/ptf-py3.mk rules/ptf-py3.dep   
+DEP_FILES   += $(SONIC_COMMON_BASE_FILES_LIST)
+SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
+
+$(PTF_PY3)_CACHE_MODE  := GIT_CONTENT_SHA 
+$(PTF_PY3)_DEP_FLAGS   := $(SONIC_COMMON_FLAGS_LIST)
+$(PTF_PY3)_DEP_FILES   := $(DEP_FILES)
+$(PTF_PY3)_SMDEP_FILES := $(SMDEP_FILES)
+$(PTF_PY3)_SMDEP_PATHS := $(SPATH)

rules/ptf-py3.mk

@@ -0,0 +1,7 @@
+# ptf package
+
+PTF_PY3 = ptf-0.9.3-py3-none-any.whl
+$(PTF_PY3)_SRC_PATH = $(SRC_PATH)/ptf-py3
+$(PTF_PY3)_PYTHON_VERSION = 3
+$(PTF_PY3)_TEST = n
+SONIC_PYTHON_WHEELS += $(PTF_PY3)

src/ptf-py3.patch/0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch

@@ -0,0 +1,26 @@
+From a8b13b9fbaa16ddd305ba2df2238ef606ef222a7 Mon Sep 17 00:00:00 2001
+From: Zhaohui Sun <zhaohuisun@microsoft.com>
+Date: Wed, 13 Apr 2022 09:24:46 +0000
+Subject: [PATCH 1/2] Remove ord in get_mac() to avoid TypeError
+
+Signed-off-by: Zhaohui Sun <zhaohuisun@microsoft.com>
+---
+ src/ptf/netutils.py | 2 +-
+ 1 file changed, 1 insertion(+), 1 deletion(-)
+
+diff --git a/src/ptf/netutils.py b/src/ptf/netutils.py
+index 752e76c..6aabe79 100644
+--- a/src/ptf/netutils.py
++++ b/src/ptf/netutils.py
+@@ -54,7 +54,7 @@ def get_if_index(iff):
+
+
+ def get_mac(iff):
+-    return ":".join(["%02x" % ord(char) for char in get_if(iff, SIOCGIFHWADDR)[18:24]])
++    return ":".join(["%02x" % char for char in get_if(iff, SIOCGIFHWADDR)[18:24]])
+
+
+ def set_promisc(s, iff, val=1):
+-- 
+2.25.1
+

src/ptf-py3.patch/0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch

@@ -0,0 +1,36 @@
+From 6e570e00ea05882d2db1e480ed041ea631bf37da Mon Sep 17 00:00:00 2001
+From: Zhaohui Sun <zhaohuisun@microsoft.com>
+Date: Wed, 13 Apr 2022 09:25:28 +0000
+Subject: [PATCH 2/2] Fill byte formatted client mac address in DHCP Discover
+ packet
+
+Signed-off-by: Zhaohui Sun <zhaohuisun@microsoft.com>
+---
+ src/ptf/testutils.py | 9 +++++----
+ 1 file changed, 5 insertions(+), 4 deletions(-)
+
+diff --git a/src/ptf/testutils.py b/src/ptf/testutils.py
+index ab67cea..83a9075 100755
+--- a/src/ptf/testutils.py
++++ b/src/ptf/testutils.py
+@@ -2712,12 +2712,13 @@ def __dhcp_mac_to_chaddr(mac_addr="00:01:02:03:04:05"):
+     """
+     Private helper function to convert a 6-byte MAC address of form:
+       '00:01:02:03:04:05'
+-    into a 16-byte chaddr byte string of form:
+-      '\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
++    into a 16-byte chaddr byte of form:
++      b'\x00\x01\x02\x03\x04\x05\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00'
+
+     """
+-    chaddr = "".join([chr(int(octet, 16)) for octet in mac_addr.split(":")])
+-    chaddr += "\x00" * 10
++    import binascii
++    chaddr = binascii.unhexlify(mac_addr.replace(':', ''))
++    chaddr += b'\x00\x00\x00\x00\x00\x00'
+     return chaddr
+
+
+-- 
+2.25.1
+

src/ptf-py3.patch/series

@@ -0,0 +1,2 @@
+0001-Remove-ord-in-get_mac-to-avoid-TypeError.patch
+0002-Fill-byte-formatted-client-mac-address-in-DHCP-Disco.patch