DOCA Packages Integration (#17)

Signed-off-by: Vivek Reddy Karri <vkarri@nvidia.com>
Co-authored-by: Vivek Reddy Karri <vkarri@nvidia.com>

.gitmodules

@@ -100,6 +100,9 @@
 [submodule "platform/nvidia-bluefield/bfscripts"]
 	path = platform/nvidia-bluefield/bfscripts
 	url = https://github.com/Mellanox/bfscripts
+[submodule "platform/nvidia-bluefield/sdk-src/sonic-bluefield-packages"]
+	path = platform/nvidia-bluefield/sdk-src/sonic-bluefield-packages
+	url = ssh://git@github.com/nvidia-sonic/sonic-bluefield-packages
 [submodule "src/sonic-p4rt/sonic-pins"]
 	path = src/sonic-p4rt/sonic-pins
-	url = https://github.com/Azure/sonic-pins.git
+	url = https://github.com/Azure/sonic-pins.git

platform/nvidia-bluefield/docker-syncd-bluefield/Dockerfile.j2

@@ -27,6 +27,10 @@ ENV DEBIAN_FRONTEND=noninteractive
 
 RUN apt-get update && \
     apt-get install -y \
+        udev \
+        hwdata \
+        pciutils \
+        libpcap0.8 \
         libxml2     \
         python3-pip  \
         python3-dev \

platform/nvidia-bluefield/recipes/docker-syncd-bluefield.mk

@@ -44,7 +44,7 @@ $(DOCKER_SYNCD_BASE)_RUN_OPT += --privileged -t
 $(DOCKER_SYNCD_BASE)_RUN_OPT += -v /host/machine.conf:/etc/machine.conf
 $(DOCKER_SYNCD_BASE)_RUN_OPT += -v /etc/sonic:/etc/sonic:ro
 
-$(DOCKER_SYNCD_BASE)_DEPENDS += $(MFT)
+$(DOCKER_SYNCD_BASE)_DEPENDS += $(MFT) $(SDN_APPL)
 $(DOCKER_SYNCD_BASE)_DBG_DEPENDS += $(LIBSWSSCOMMON_DBG)
 
 $(DOCKER_SYNCD_BASE)_VERSION = 1.0.0

platform/nvidia-bluefield/recipes/sdk.mk

@@ -0,0 +1,182 @@
+#
+# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+DOCA_SDK_BASE_PATH = $(PLATFORM_PATH)/sdk-src/sonic-bluefield-packages/bin
+
+# Place here URL where SDK sources exist
+DOCA_SDK_SOURCE_BASE_URL =
+
+ifneq ($(DOCA_SDK_SOURCE_BASE_URL), )
+SDK_FROM_SRC = y
+else
+SDK_FROM_SRC = n
+endif
+
+### RDMA
+RDMA_CORE_VER=33.1-1
+
+RDMA_CORE = rdma-core_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+$(RDMA_CORE)_SRC_PATH = $(PLATFORM_PATH)/sdk-src/rdma
+$(RDMA_CORE)_DEPENDS = $(LIBNL3_DEV) $(LIBNL_ROUTE3_DEV)
+$(RDMA_CORE)_RDEPENDS = $(LIBNL3) $(ETHTOOL)
+RDMA_CORE_DBGSYM = rdma-core-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(RDMA_CORE_DBGSYM)))
+endif
+
+# Derived debs from rdma-core
+IB_VERBS_PROV = ibverbs-providers_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_VERBS_PROV_DBGSYM = ibverbs-providers-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+$(IB_VERBS_PROV)_DEPENDS = $(IB_VERBS)
+$(IB_VERBS_PROV)_RDEPENDS = $(IB_VERBS)
+$(IB_VERBS_PROV_DBGSYM)_DEPENDS = $(IB_VERBS_DBGSYM)
+$(IB_VERBS_PROV_DBGSYM)_RDEPENDS = $(IB_VERBS_DBGSYM)
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_VERBS_PROV)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_VERBS_PROV_DBGSYM)))
+endif
+
+IB_VERBS = libibverbs1_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_VERBS_DBGSYM = libibverbs1-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_VERBS_DEV = libibverbs-dev_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+$(IB_VERBS)_DEPENDS = $(LIBNL3_DEV) $(LIBNL_ROUTE3_DEV)
+$(IB_VERBS_DBGSYM)_DEPENDS = $(LIBNL3_DEV) $(LIBNL_ROUTE3_DEV)
+$(IB_VERBS_DEV)_DEPENDS = $(IB_VERBS_PROV)
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_VERBS)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_VERBS_DBGSYM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_VERBS_DEV)))
+endif
+
+IB_UMAD = libibumad3_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_UMAD_DBGSYM = libibumad3-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_UMAD_DEV = libibumad-dev_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_UMAD)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_UMAD_DBGSYM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_UMAD_DEV)))
+endif
+
+RDMA_CM = librdmacm1_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+RDMA_CM_DBGSYM = librdmacm1-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+RDMA_CM_DEV = librdmacm-dev_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(RDMA_CM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(RDMA_CM_DBGSYM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(RDMA_CM_DEV)))
+endif
+
+IB_MAD = libibmad5_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+$(IB_MAD)_DEPENDS = $(IB_UMAD)
+$(IB_MAD)_RDEPENDS = $(IB_UMAD)
+IB_MAD_DBGSYM = libibmad5-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+$(IB_MAD_DBGSYM)_DEPENDS = $(IB_UMAD_DBGSYM)
+$(IB_MAD_DBGSYM)_RDEPENDS = $(IB_UMAD_DBGSYM)
+IB_MAD_DEV = libibmad-dev_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_MAD)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_MAD_DBGSYM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_MAD_DEV)))
+endif
+
+IB_NET_DISC = libibnetdisc5_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_NET_DISC_DBGSYM = libibnetdisc5-dbgsym_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+IB_NET_DISC_DEV = libibnetdisc-dev_${RDMA_CORE_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_NET_DISC)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_NET_DISC_DBGSYM)))
+$(eval $(call add_extra_package,$(RDMA_CORE),$(IB_NET_DISC_DEV)))
+endif
+
+RDMA_CORE_DEBS += $(IB_VERBS_PROV) $(IB_VERBS) $(RDMA_CM) \
+				  $(IB_UMAD) $(IB_MAD) $(IB_NET_DISC)
+
+RDMA_CORE_DBGSYM_DEBS += $(RDMA_CORE_DBGSYM) $(IB_VERBS_PROV_DBGSYM) \
+				 $(IB_VERBS_DBGSYM) $(IB_UMAD_DBGSYM) $(RDMA_CM_DBGSYM) \
+				 $(IB_MAD_DBGSYM) $(IB_NET_DISC_DBGSYM)
+
+RDMA_CORE_DEV_DEBS += $(IB_VERBS_DEV) $(IB_UMAD_DEV) $(RDMA_CM_DEV) $(IB_MAD_DEV) $(IB_NET_DISC_DEV)
+
+export RDMA_CORE RDMA_CORE_DEBS RDMA_CORE_DBGSYM_DEBS RDMA_CORE_DEV_DEBS
+
+### DPDK with Hardware Steering
+
+DPDK_VER=21.02-1mlnx1
+
+DPDK_HWS = dpdk.hws_${DPDK_VER}_${CONFIGURED_ARCH}.deb
+$(DPDK_HWS)_SRC_PATH = $(PLATFORM_PATH)/sdk-src/dpdk
+$(DPDK_HWS)_DEPENDS = $(IB_VERBS_PROV) $(IB_VERBS_DEV) $(RDMA_CORE) $(IB_MAD) $(IB_NET_DISC) $(RDMA_CM)
+$(DPDK_HWS)_RDEPENDS = $(IB_VERBS_PROV) $(IB_VERBS) $(RDMA_CORE) $(IB_MAD) $(IB_NET_DISC) $(RDMA_CM)
+
+DPDK_HWS_DEV = dpdk.hws-dev_${DPDK_VER}_${CONFIGURED_ARCH}.deb
+$(DPDK_HWS_DEV)_DEPENDS = $(IB_VERBS_PROV) $(IB_VERBS_DEV) $(RDMA_CORE) $(IB_MAD) $(IB_NET_DISC) $(RDMA_CM)
+$(DPDK_HWS_DEV)_RDEPENDS = $(IB_VERBS_PROV) $(IB_VERBS) $(RDMA_CORE) $(IB_MAD) $(IB_NET_DISC) $(RDMA_CM)
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_extra_package,$(DPDK_HWS),$(DPDK_HWS_DEV)))
+endif
+
+export DPDK_HWS DPDK_HWS_DEV
+
+## DOCA_DPDK
+
+DOCA_DPDK_VER=1.5-1mlnx1
+
+DOCA_DPDK = doca-dpdk_${DOCA_DPDK_VER}_${CONFIGURED_ARCH}.deb
+$(DOCA_DPDK)_SRC_PATH = $(PLATFORM_PATH)/sdk-src/doca-dpdk
+$(DOCA_DPDK)_DEPENDS = $(DPDK_HWS_DEV)
+$(DOCA_DPDK)_RDEPENDS = $(DPDK_HWS)
+
+DOCA_DPDK_DEV = doca-dpdk-dev_${DOCA_DPDK_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_derived_package,$(DOCA_DPDK),$(DOCA_DPDK_DEV)))
+endif
+
+export DOCA_DPDK DOCA_DPDK_DEV
+
+## SDN Appliance 
+
+SDN_APPL_VER=1.5-1mlnx1
+SDN_APPL = sdn-appliance_${SDN_APPL_VER}_${CONFIGURED_ARCH}.deb
+$(SDN_APPL)_SRC_PATH = $(PLATFORM_PATH)/sdk-src/sdn
+$(SDN_APPL)_DEPENDS = $(DOCA_DPDK_DEV) 
+$(SDN_APPL)_RDEPENDS = $(DOCA_DPDK)
+
+SDN_APPL_DEV = sdn-appliance-dev_${DOCA_DPDK_VER}_${CONFIGURED_ARCH}.deb
+ifeq ($(SDK_FROM_SRC),y)
+$(eval $(call add_derived_package,$(SDN_APPL),$(SDN_APPL_DEV)))
+endif
+
+export SDN_APPL SDN_APPL_DEV
+
+define make_path
+	$(1)_PATH = $(DOCA_SDK_BASE_PATH)
+
+endef
+
+$(eval $(foreach deb,$(RDMA_CORE) $(RDMA_CORE_DEBS),$(call make_path,$(deb))))
+$(eval $(foreach deb,$(DPDK_HWS) $(DOCA_DPDK) $(SDN_APPL),$(call make_path,$(deb))))
+
+ifeq ($(SDK_FROM_SRC), y)
+SONIC_MAKE_DEBS += $(RDMA_CORE) $(DPDK_HWS) $(DOCA_DPDK) $(SDN_APPL)
+else
+$(SDN_APPL)_DERIVED_DEBS += $(RDMA_CORE) $(RDMA_CORE_DEBS) $(DPDK_HWS) $(DOCA_DPDK)
+SONIC_COPY_DEBS += $(RDMA_CORE) $(RDMA_CORE_DEBS) $(DPDK_HWS) $(DOCA_DPDK) $(SDN_APPL)
+endif
+
+doca-sdk-packages: $(addprefix $(DEBS_PATH)/, $(RDMA_CORE) $(DPDK_HWS) $(DOCA_DPDK) $(SDN_APPL))
+
+SONIC_PHONY_TARGETS += doca-sdk-packages

platform/nvidia-bluefield/rules.mk

@@ -41,6 +41,7 @@ include $(PLATFORM_PATH)/$(RECIPE_DIR)/bfscripts.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/platform-modules-bf2.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/mft.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/mlx-openipmi.mk
+include $(PLATFORM_PATH)/$(RECIPE_DIR)/sdk.mk
 # include $(PLATFORM_PATH)/$(RECIPE_DIR)/libpka.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/docker-syncd-bluefield.mk
 include $(PLATFORM_PATH)/$(RECIPE_DIR)/installer-image.mk

platform/nvidia-bluefield/sdk-src/doca-dpdk/.gitignore

@@ -0,0 +1,2 @@
+*
+!Makefile

platform/nvidia-bluefield/sdk-src/doca-dpdk/Makefile

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.ONESHELL:
+SHELL = /bin/bash
+
+MAIN_TARGET = $(DOCA_DPDK)
+DERIVED_TARGETS = $(DOCA_DPDK_DEV)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	# get sources
+	rm -rf doca_dpdk/ doca_dpdk.zip
+
+	# get sources
+	wget $(DOCA_SDK_SOURCE_BASE_URL)/doca_dpdk.zip
+	unzip doca_dpdk.zip
+
+	pushd doca_dpdk/
+
+	# Build the Debs
+	PATH=/usr/bin/:$(PATH) dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+
+	popd
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

platform/nvidia-bluefield/sdk-src/dpdk/.gitignore

@@ -0,0 +1,2 @@
+*
+!Makefile

platform/nvidia-bluefield/sdk-src/dpdk/Makefile

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.ONESHELL:
+SHELL = /bin/bash
+
+MAIN_TARGET = $(DPDK_HWS)
+DERIVED_TARGETS = $(DPDK_HWS_DEV)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+
+	rm -rf dpdk.hws/ dpdk.hws.zip
+
+	# get sources
+	wget $(DOCA_SDK_SOURCE_BASE_URL)/dpdk.hws.zip
+	unzip dpdk.hws.zip
+	pushd dpdk.hws/
+
+	# Build the package
+	# PATH variable is required for ninja to find the right version
+	PATH=/usr/bin/:$(PATH) DPDK_CONFIG_OPTIONS='--cross-file ../config/arm/arm64_bluefield_linux_gcc' DEB_BUILD_OPTIONS='dpdk_config_options=-Dmachine=0xd08' dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+
+	popd
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

platform/nvidia-bluefield/sdk-src/rdma/.gitignore

@@ -0,0 +1,2 @@
+*
+!Makefile

platform/nvidia-bluefield/sdk-src/rdma/Makefile

@@ -0,0 +1,41 @@
+#
+# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.ONESHELL:
+SHELL = /bin/bash
+
+MAIN_TARGET = $(RDMA_CORE)
+DERIVED_TARGETS = $(RDMA_CORE_DEBS) $(RDMA_CORE_DBGSYM_DEBS) $(RDMA_CORE_DEV_DEBS)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+
+	rm -rf rdma-core.hws.fdb/ rdma-core.hws.fdb.zip
+
+	# get sources
+	wget $(DOCA_SDK_SOURCE_BASE_URL)/rdma-core.hws.fdb.zip
+	unzip rdma-core.hws.fdb.zip
+
+	# build
+	pushd rdma-core.hws.fdb/
+
+	# Build the Debs
+	PATH=/usr/bin/:$(PATH) dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+
+	popd
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)

platform/nvidia-bluefield/sdk-src/sdn/.gitignore

@@ -0,0 +1,2 @@
+*
+!Makefile

platform/nvidia-bluefield/sdk-src/sdn/Makefile

@@ -0,0 +1,40 @@
+#
+# Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES.
+# Apache-2.0
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+.ONESHELL:
+SHELL = /bin/bash
+
+MAIN_TARGET = $(SDN_APPL)
+DERIVED_TARGETS = $(SDN_APPL_DEV)
+
+$(addprefix $(DEST)/, $(MAIN_TARGET)): $(DEST)/% :
+	# get sources
+	rm -rf SDN_appliance/ SDN_appliance.zip
+
+	# get sources
+	wget $(DOCA_SDK_SOURCE_BASE_URL)/SDN_appliance.zip
+	unzip SDN_appliance.zip
+
+	pushd SDN_appliance/
+
+	# Build the Debs
+	PATH=/usr/bin/:$(PATH) dpkg-buildpackage -us -uc -b -j$(SONIC_CONFIG_MAKE_JOBS) --admindir $(SONIC_DPKG_ADMINDIR)
+
+	popd
+	mv $(DERIVED_TARGETS) $* $(DEST)/
+
+$(addprefix $(DEST)/, $(DERIVED_TARGETS)): $(DEST)/% : $(DEST)/$(MAIN_TARGET)