[sonic-utilities] Build and install as a Python wheel package

build_debian.sh

@@ -317,6 +317,10 @@ sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y in
  cron \
  haveged
 
+# Install "wheel" package so that we can install .whl packages and not
+# encounter a "error: invalid command 'bdist_wheel'" error
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install wheel
+sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip3 install wheel
 
 if [[ $CONFIGURED_ARCH == amd64 ]]; then
 ## Pre-install the fundamental packages for amd64 (x86)
@@ -396,7 +400,7 @@ sudo mkdir -p $FILESYSTEM_ROOT/var/core
 
 # Config sysctl
 sudo augtool --autosave "
-set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/bin/coredump-compress %e %t %p %P'
+set /files/etc/sysctl.conf/kernel.core_pattern '|/usr/local/bin/coredump-compress %e %t %p %P'
 set /files/etc/sysctl.conf/kernel.softlockup_panic 1
 set /files/etc/sysctl.conf/kernel.panic 10
 set /files/etc/sysctl.conf/vm.panic_on_oom 2

dockers/docker-fpm-frr/base_image_files/TSC

@@ -2,4 +2,4 @@
 
 docker exec -i bgp /usr/bin/TSC
 
-/usr/bin/portstat -p 5
+portstat -p 5

files/build_templates/docker_image_ctl.j2

@@ -125,10 +125,10 @@ function postStartAction()
  $SONIC_DB_CLI CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"
  fi
 
- if [[ -x /usr/bin/db_migrator.py ]]; then
+ if [[ -x /usr/local/bin/db_migrator.py ]]; then
  # Migrate the DB to the latest schema version if needed
  if [ -z "$DEV" ]; then
- /usr/bin/db_migrator.py -o migrate
+ /usr/local/bin/db_migrator.py -o migrate
  fi
  fi
  # Add redis UDS to the redis group and give read/write access to the group

files/build_templates/sonic_debian_extension.j2

@@ -169,27 +169,29 @@ sudo cp {{platform_common_py2_wheel_path}} $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2
 sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install $PLATFORM_COMMON_PY2_WHEEL_NAME
 sudo rm -rf $FILESYSTEM_ROOT/$PLATFORM_COMMON_PY2_WHEEL_NAME
 
+# Install Debian packages and their dependencies which are needed by sonic-utilities
+sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f python-m2crypto
+
 # Install built Python Click package (and its dependencies via 'apt-get -y install -f')
 # Do this before installing sonic-utilities so that it doesn't attempt to install
 # an older version as part of its dependencies
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/python-click*_all.deb || \
  sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
-# Install python pexpect used by sonic-utilities consutil
-# using pip install instead to get a more recent version than is available through debian
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install pexpect
-
-# Install tabulate >= 0.8.1 via pip in order to support multi-line row output for sonic-utilities
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install tabulate==0.8.2
-
-# Install packages to support Dynamic Port Breakout config command for sonic-utilities
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install xmltodict==0.12.0
-sudo https_proxy=$https_proxy LANG=C chroot $FILESYSTEM_ROOT pip install jsondiff==1.2.0
+# Install SONiC Utilities Python 2 package
+SONIC_UTILITIES_PY2_WHEEL_NAME=$(basename {{sonic_utilities_py2_wheel_path}})
+sudo cp {{sonic_utilities_py2_wheel_path}} $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY2_WHEEL_NAME
+sudo LANG=C chroot $FILESYSTEM_ROOT pip install $SONIC_UTILITIES_PY2_WHEEL_NAME
+sudo rm -rf $FILESYSTEM_ROOT/$SONIC_UTILITIES_PY2_WHEEL_NAME
 
-# Install SONiC Utilities (and its dependencies via 'apt-get -y install -f')
-sudo dpkg --root=$FILESYSTEM_ROOT -i $python_debs_path/python-sonic-utilities_*.deb || \
+# Install sonic-utilities data files (and any dependencies via 'apt-get -y install -f')
+sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-utilities-data_*.deb || \
  sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 
+# sonic-utilities-data installs bash-completion as a dependency. However, it is disabled by default
+# in bash.bashrc, so we copy a version of the file with it enabled here.
+sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
+
 {% if enable_ztp == "y" %}
 # Install ZTP (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-ztp_*.deb || \
@@ -202,10 +204,6 @@ sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-host-service_*.deb || \
  sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f
 {% endif %}
 
-# SONiC utilities installs bash-completion as a dependency. However, it is disabled by default
-# in bash.bashrc, so we copy a version of the file with it enabled here.
-sudo cp -f $IMAGE_CONFIGS/bash/bash.bashrc $FILESYSTEM_ROOT/etc/
-
 # Install SONiC Device Data (and its dependencies via 'apt-get -y install -f')
 sudo dpkg --root=$FILESYSTEM_ROOT -i $debs_path/sonic-device-data_*.deb || \
  sudo LANG=C DEBIAN_FRONTEND=noninteractive chroot $FILESYSTEM_ROOT apt-get -y install -f

files/image_config/config-setup/config-setup

@@ -188,9 +188,9 @@ load_config()
  return $?
  fi
 
- if [[ -x /usr/bin/db_migrator.py ]]; then
+ if [[ -x /usr/local/bin/db_migrator.py ]]; then
  # Migrate the DB to the latest schema version if needed
- /usr/bin/db_migrator.py -o migrate
+ /usr/local/bin/db_migrator.py -o migrate
  fi
 
  sonic-db-cli CONFIG_DB SET "CONFIG_DB_INITIALIZED" "1"

files/image_config/fstrim/fstrim.service

@@ -3,5 +3,5 @@ Description=Discard unused blocks
 
 [Service]
 Type=oneshot
-ExecStartPre=/usr/bin/log_ssd_health
+ExecStartPre=/usr/local/bin/log_ssd_health
 ExecStart=/sbin/fstrim -av

files/image_config/monit/conf.d/sonic-host

@@ -27,6 +27,6 @@ check process rsyslog with pidfile /var/run/rsyslogd.pid
 # Hence for any discrepancy, there will be log messages for "ERR" level
 # from both route_check.py & monit.
 #
-check program routeCheck with path "/usr/bin/route_check.py"
+check program routeCheck with path "/usr/local/bin/route_check.py"
  every 5 cycles
  if status != 0 then alert

files/image_config/sudoers/sudoers

@@ -19,26 +19,25 @@ Defaults lecture_file = /etc/sudoers.lecture
 
 # Cmnd alias specification
 # Note: bcmcmd is dangerous for users in read only netgroups because it may operate ASIC
-Cmnd_Alias READ_ONLY_CMDS = /sbin/brctl show, \
- /usr/bin/decode-syseeprom, \
- /usr/bin/docker images *, \
+Cmnd_Alias READ_ONLY_CMDS = /bin/cat /var/log/syslog*, \
+ /sbin/brctl show, \
  /usr/bin/docker exec snmp cat /etc/snmp/snmpd.conf, \
  /usr/bin/docker exec bgp cat /etc/quagga/bgpd.conf, \
  /usr/bin/docker exec * ps aux, \
+ /usr/bin/docker images *, \
  /usr/bin/docker ps*, \
- /usr/bin/generate_dump, \
  /usr/bin/lldpctl, \
- /usr/bin/lldpshow, \
- /usr/bin/psuutil *, \
  /usr/bin/sensors, \
- /usr/bin/sonic-installer list, \
- /usr/bin/sfputil show *, \
- /usr/bin/teamshow, \
+ /usr/bin/tail -F /var/log/syslog, \
  /usr/bin/vtysh -c show *, \
- /bin/cat /var/log/syslog*, \
- /usr/bin/tail -F /var/log/syslog
-
-Cmnd_Alias PASSWD_CMDS = /usr/bin/config tacacs passkey *, \
+ /usr/local/bin/decode-syseeprom, \
+ /usr/local/bin/generate_dump, \
+ /usr/local/bin/lldpshow, \
+ /usr/local/bin/psuutil *, \
+ /usr/local/bin/sonic-installer list, \
+ /usr/local/bin/sfputil show *
+
+Cmnd_Alias PASSWD_CMDS = /usr/local/bin/config tacacs passkey *, \
  /usr/sbin/chpasswd *
 
 # User privilege specification

files/image_config/warmboot-finalizer/finalize-warmboot.sh

@@ -6,7 +6,7 @@ VERBOSE=no
 COMP_LIST="orchagent neighsyncd bgp natsyncd"
 EXP_STATE="reconciled"
 
-ASSISTANT_SCRIPT="/usr/bin/neighbor_advertiser"
+ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
 
 
 function debug()

files/image_config/watchdog-control/watchdog-control.sh

@@ -1,7 +1,7 @@
 #! /bin/bash
 
 VERBOSE=no
-WATCHDOG_UTIL="/usr/bin/watchdogutil"
+WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
 
 function debug()
 {

platform/broadcom/sonic-platform-modules-dell/common/fw-updater

@@ -16,7 +16,7 @@ HOST_GRUB_CFG = HOST_GRUB_DIR + '/grub/grub.cfg'
 HOST_GRUB_ENV = HOST_GRUB_DIR + '/grub/grubenv'
 HOST_GRUB_BOOT_DIR = '--boot-directory=' + HOST_GRUB_DIR
 HOST_PLATFORM_INFO = HOST_GRUB_DIR + '/platform'
-dell_reload_tool = '/usr/bin/reboot'
+dell_reload_tool = '/usr/local/bin/reboot'
 
 
 

platform/mellanox/mlnx-onie-fw-update.sh

@@ -94,7 +94,7 @@ system_reboot() {
  sleep 5s
 
  # Use SONiC reboot scenario
- /usr/bin/reboot
+ /usr/local/bin/reboot
 }
 
 terminate_handler() {

platform/p4/docker-sonic-p4.mk

@@ -11,7 +11,7 @@ $(DOCKER_SONIC_P4)_DEPENDS += $(SWSS) \
  $(LIBTEAMDCTL) \
  $(LIBTEAM_UTILS) \
  $(SONIC_DEVICE_DATA) \
- $(SONIC_UTILS) \
+ $(SONIC_UTILITIES_PY2) \
  $(IPROUTE2)
 
 # ifeq ($(ROUTING_STACK), quagga)

platform/vs/docker-sonic-vs.mk

@@ -10,9 +10,8 @@ $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS) \
  $(SONIC_DEVICE_DATA) \
  $(LIBYANG) \
  $(LIBYANG_CPP) \
- $(LIBYANG_PY2)
-
-$(DOCKER_SONIC_VS)_PYTHON_DEBS += $(SONIC_UTILS)
+ $(LIBYANG_PY2) \
+ $(SONIC_UTILITIES_DATA)
 
 # swsssdk is a dependency of sonic-py-common
 # TODO: sonic-py-common should depend on swsscommon instead
@@ -21,7 +20,8 @@ $(DOCKER_SONIC_VS)_PYTHON_WHEELS += $(SWSSSDK_PY2) \
  $(SONIC_PY_COMMON_PY2) \
  $(SONIC_PY_COMMON_PY3) \
  $(SONIC_YANG_MODELS_PY3) \
- $(SONIC_YANG_MGMT_PY)
+ $(SONIC_YANG_MGMT_PY) \
+ $(SONIC_UTILITIES_PY2)
 
 ifeq ($(INSTALL_DEBUG_TOOLS), y)
 $(DOCKER_SONIC_VS)_DEPENDS += $(SWSS_DBG) \

platform/vs/docker-sonic-vs/Dockerfile.j2

@@ -46,7 +46,8 @@ RUN apt-get install -y net-tools \
  conntrack \
  iptables \
  python3-pip \
- jq
+ jq \
+ python-m2crypto
 
 # install redis-server
 RUN curl -o redis-tools_6.0.6-1~bpo10+1_amd64.deb "https://sonicstorage.blob.core.windows.net/packages/redis/redis-tools_6.0.6-1~bpo10+1_amd64.deb?sv=2015-04-05&sr=b&sig=73zbmjkf3pi%2Bn0R8Hy7CWT2EUvOAyzM5aLYJWCLySGM%3D&se=2030-09-06T19%3A44%3A59Z&sp=r"
@@ -56,6 +57,8 @@ RUN rm redis-tools_6.0.6-1~bpo10+1_amd64.deb redis-server_6.0.6-1~bpo10+1_amd64.
 
 RUN pip install setuptools
 RUN pip3 install setuptools
+RUN pip install wheel
+RUN pip3 install wheel
 RUN pip install py2_ipaddress
 RUN pip install six
 RUN pip install pyroute2==0.5.3 netifaces==0.10.7
@@ -102,11 +105,7 @@ COPY python-wheels/{{ whl }} python-wheels/
 # install PKGs after copying all PKGs to avoid dependency failure
 # use py3 to find python3 package, which is forced by wheel as of now
 {%- for whl in docker_sonic_vs_whls.split(' ') %}
-{%- if 'py3' in whl %}
-RUN pip3 install python-wheels/{{ whl }}
-{% else -%}
-RUN pip install python-wheels/{{ whl }}
-{%- endif %}
+RUN pip{% if 'py3' in whl %}3{% endif %} install python-wheels/{{ whl }}
 {%- endfor %}
 {% endif %}
 

rules/sonic-utilities-data.dep

@@ -0,0 +1,9 @@
+SPATH := $($(SONIC_UTILITIES_DATA)_SRC_PATH)
+DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-utilities-data.mk rules/sonic-utilities-data.dep 
+DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
+DEP_FILES += $(shell git ls-files $(SPATH))
+
+$(SONIC_UTILITIES_DATA)_CACHE_MODE := GIT_CONTENT_SHA
+$(SONIC_UTILITIES_DATA)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_UTILITIES_DATA)_DEP_FILES := $(DEP_FILES)
+

rules/sonic-utilities-data.mk

@@ -0,0 +1,5 @@
+# SONiC command line utilities data package 
+
+SONIC_UTILITIES_DATA = sonic-utilities-data_1.0-1_all.deb
+$(SONIC_UTILITIES_DATA)_SRC_PATH = $(SRC_PATH)/sonic-utilities/sonic-utilities-data
+SONIC_DPKG_DEBS += $(SONIC_UTILITIES_DATA)

rules/sonic-utilities.dep

@@ -1,12 +1,10 @@
-
-SPATH := $($(SONIC_UTILS)_SRC_PATH)
+SPATH := $($(SONIC_UTILITIES_PY2)_SRC_PATH)
 DEP_FILES := $(SONIC_COMMON_FILES_LIST) rules/sonic-utilities.mk rules/sonic-utilities.dep 
 DEP_FILES += $(SONIC_COMMON_BASE_FILES_LIST)
 SMDEP_FILES := $(addprefix $(SPATH)/,$(shell cd $(SPATH) && git ls-files))
 
-$(SONIC_UTILS)_CACHE_MODE := GIT_CONTENT_SHA 
-$(SONIC_UTILS)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
-$(SONIC_UTILS)_DEP_FILES := $(DEP_FILES)
-$(SONIC_UTILS)_SMDEP_FILES := $(SMDEP_FILES)
-$(SONIC_UTILS)_SMDEP_PATHS := $(SPATH)
-
+$(SONIC_UTILITIES_PY2)_CACHE_MODE := GIT_CONTENT_SHA
+$(SONIC_UTILITIES_PY2)_DEP_FLAGS := $(SONIC_COMMON_FLAGS_LIST)
+$(SONIC_UTILITIES_PY2)_DEP_FILES := $(DEP_FILES)
+$(SONIC_UTILITIES_PY2)_SMDEP_FILES := $(SMDEP_FILES)
+$(SONIC_UTILITIES_PY2)_SMDEP_PATHS := $(SPATH)

rules/sonic-utilities.mk

@@ -6,19 +6,18 @@
 # added here also. However, the current build system assumes all runtime
 # dependencies are .deb packages.
 #
-# TODO: Create a way to specify both .deb and .whl runtime dependencies
-# then add the aforementioned runtime dependencies here.
-#
 
-SONIC_UTILS = python-sonic-utilities_1.2-1_all.deb
-$(SONIC_UTILS)_SRC_PATH = $(SRC_PATH)/sonic-utilities
-$(SONIC_UTILS)_DEBS_DEPENDS = $(LIBYANG) \
- $(LIBYANG_CPP) \
- $(LIBYANG_PY2) \
- $(LIBYANG_PY3)
-$(SONIC_UTILS)_WHEEL_DEPENDS = $(SONIC_PY_COMMON_PY2) \
- $(SONIC_PY_COMMON_PY3) \
- $(SONIC_CONFIG_ENGINE) \
- $(SONIC_YANG_MGMT_PY) \
- $(SONIC_YANG_MODELS_PY3)
-SONIC_PYTHON_STDEB_DEBS += $(SONIC_UTILS)
+SONIC_UTILITIES_PY2 = sonic_utilities-1.2-py2-none-any.whl
+$(SONIC_UTILITIES_PY2)_SRC_PATH = $(SRC_PATH)/sonic-utilities
+$(SONIC_UTILITIES_PY2)_PYTHON_VERSION = 2
+$(SONIC_UTILITIES_PY2)_DEPENDS += $(SONIC_PY_COMMON_PY2) \
+ $(SONIC_PY_COMMON_PY3) \
+ $(SWSSSDK_PY2) \
+ $(SONIC_CONFIG_ENGINE) \
+ $(SONIC_YANG_MGMT_PY) \
+ $(SONIC_YANG_MODELS_PY3)
+$(SONIC_UTILITIES_PY2)_DEBS_DEPENDS = $(LIBYANG) \
+ $(LIBYANG_CPP) \
+ $(LIBYANG_PY2) \
+ $(LIBYANG_PY3)
+SONIC_PYTHON_WHEELS += $(SONIC_UTILITIES_PY2)

slave.mk

@@ -802,12 +802,13 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  $(LIBPAM_TACPLUS) \
  $(LIBNSS_TACPLUS) \
  $(MONIT) \
- $(PYTHON_SWSSCOMMON)) \
+ $(PYTHON_SWSSCOMMON) \
+ $(SONIC_UTILITIES_DATA)) \
  $$(addprefix $(TARGET_PATH)/,$$($$*_DOCKERS)) \
  $$(addprefix $(FILES_PATH)/,$$($$*_FILES)) \
  $(if $(findstring y,$(ENABLE_ZTP)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_ZTP))) \
  $(if $(findstring y,$(INCLUDE_HOST_SERVICE)),$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(SONIC_HOST_SERVICE))) \
- $(addprefix $(PYTHON_DEBS_PATH)/,$(SONIC_UTILS)) \
+ $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY2)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_PY_COMMON_PY3)) \
  $(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_CONFIG_ENGINE)) \
@@ -858,6 +859,7 @@ $(addprefix $(TARGET_PATH)/, $(SONIC_INSTALLERS)) : $(TARGET_PATH)/% : \
  export multi_instance="false"
  export python_swss_debs="$(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$($(LIBSWSSCOMMON)_RDEPENDS))"
  export python_swss_debs+=" $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(LIBSWSSCOMMON)) $(addprefix $(IMAGE_DISTRO_DEBS_PATH)/,$(PYTHON_SWSSCOMMON))"
+ export sonic_utilities_py2_wheel_path="$(addprefix $(PYTHON_WHEELS_PATH)/,$(SONIC_UTILITIES_PY2))"
 
  $(foreach docker, $($*_DOCKERS),\
  export docker_image="$(docker)"