Support building sonic-utilities as a Python wheel package instead of…

… a Debian package (sonic-net#1122)

Support building sonic-utilities as a Python wheel package rather than a Debian package.

Notes:
- Wheel packaging restricts the installation of files to within the dist-packages directory. Thus, we cannot install data files to system directories. Therefore, I am building a separate 'sonic-utilities-data' package for installing the data files (bash_completion, templates)
- Wheel packages install scripts/entrypoints to the /usr/local/bin directory, whereas Debian packages install to /usr/bin

.gitignore

@@ -1,7 +1,28 @@
+# Compiled Python files
+*.pyc
+scripts/aclshowc
+scripts/neighbor_advertiserc
+scripts/port2aliasc
+
+# Generated by packaging
+*.egg-info/
+.eggs/
 build/
-deb_dist/
 dist/
-*.egg-info/
-*.pyc
-.cache
-*.tar.gz
+
+# Unit test coverage
+.coverage
+coverage.xml
+htmlcov/
+
+# Ignores for sonic-utilities-data
+sonic-utilities-data/debian/*
+!sonic-utilities-data/debian/changelog
+!sonic-utilities-data/debian/compat
+!sonic-utilities-data/debian/control
+!sonic-utilities-data/debian/copyright
+!sonic-utilities-data/debian/install
+!sonic-utilities-data/debian/rules
+sonic-utilities-data_*.deb
+sonic-utilities-data_*.buildinfo
+sonic-utilities-data_*.changes

config/main.py

@@ -1101,7 +1101,7 @@ def reload(db, filename, yes, load_sysinfo, no_service_restart):
         client.set(config_db.INIT_INDICATOR, 1)
 
         # Migrate DB contents to latest version
-        db_migrator='/usr/bin/db_migrator.py'
+        db_migrator='/usr/local/bin/db_migrator.py'
         if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK):
             if namespace is None:
                 command = "{} -o migrate".format(db_migrator)
@@ -1200,7 +1200,7 @@ def load_minigraph(db, no_service_restart):
     clicommon.run_command("config qos reload", display_cmd=True)
 
     # Write latest db version string into db
-    db_migrator='/usr/bin/db_migrator.py'
+    db_migrator='/usr/local/bin/db_migrator.py'
     if os.path.isfile(db_migrator) and os.access(db_migrator, os.X_OK):
         for namespace in namespace_list:
             if namespace is DEFAULT_NAMESPACE:

scripts/fast-reboot

@@ -12,12 +12,12 @@ FORCE=no
 STRICT=no
 REBOOT_METHOD="/sbin/kexec -e"
 ASSISTANT_IP_LIST=""
-ASSISTANT_SCRIPT="/usr/bin/neighbor_advertiser"
-WATCHDOG_UTIL="/usr/bin/watchdogutil"
+ASSISTANT_SCRIPT="/usr/local/bin/neighbor_advertiser"
+WATCHDOG_UTIL="/usr/local/bin/watchdogutil"
 DEVPATH="/usr/share/sonic/device"
 PLATFORM=$(sonic-cfggen -H -v DEVICE_METADATA.localhost.platform)
 PLATFORM_PLUGIN="${REBOOT_TYPE}_plugin"
-LOG_SSD_HEALTH="/usr/bin/log_ssd_health"
+LOG_SSD_HEALTH="/usr/local/bin/log_ssd_health"
 SSD_FW_UPDATE="ssd-fw-upgrade"
 # Require 100M available on the hard drive for warm reboot temp files,
 # Size is in 1K blocks:
@@ -324,7 +324,7 @@ function reboot_pre_check()
     fi
     
     # Make sure ASIC configuration has not changed between images
-    ASIC_CONFIG_CHECK_SCRIPT="/usr/bin/asic_config_check"
+    ASIC_CONFIG_CHECK_SCRIPT="/usr/local/bin/asic_config_check"
     ASIC_CONFIG_CHECK_SUCCESS=0
     if [[ "$REBOOT_TYPE" = "warm-reboot" || "$REBOOT_TYPE" = "fastfast-reboot" ]]; then
         ASIC_CONFIG_CHECK_EXIT_CODE=0
@@ -447,7 +447,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
     CONFIG_DB_FILE=/etc/sonic/config_db.json
     mkdir -p $DUMP_DIR
     FAST_REBOOT_DUMP_RC=0
-    /usr/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$?
+    /usr/local/bin/fast-reboot-dump.py -t $DUMP_DIR || FAST_REBOOT_DUMP_RC=$?
     if [[ FAST_REBOOT_DUMP_RC -ne 0 ]]; then
         error "Failed to run fast-reboot-dump.py. Exit code: $FAST_REBOOT_DUMP_RC"
         unload_kernel
@@ -456,7 +456,7 @@ if [[ "$REBOOT_TYPE" = "fast-reboot" ]]; then
 
     FILTER_FDB_ENTRIES_RC=0
     # Filter FDB entries using MAC addresses from ARP table
-    /usr/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
+    /usr/local/bin/filter_fdb_entries -f $DUMP_DIR/fdb.json -a $DUMP_DIR/arp.json -c $CONFIG_DB_FILE || FILTER_FDB_ENTRIES_RC=$?
     if [[ FILTER_FDB_ENTRIES_RC -ne 0 ]]; then
         error "Failed to filter FDb entries. Exit code: $FILTER_FDB_ENTRIES_RC"
         unload_kernel
@@ -497,7 +497,7 @@ fi
 
 # Kill nat docker after saving the conntrack table
 debug "Stopping nat ..."
-/usr/bin/dump_nat_entries.py
+/usr/local/bin/dump_nat_entries.py
 docker kill nat > /dev/null || true
 systemctl stop nat
 debug "Stopped nat ..."

scripts/generate_dump

@@ -342,7 +342,7 @@ main() {
     $MKDIR $V -p $TARDIR
 
     # Start with this script so its obvious what code is responsible
-    $LN $V -s /usr/bin/generate_dump $TARDIR
+    $LN $V -s /usr/local/bin/generate_dump $TARDIR
     $TAR $V -chf $TARFILE -C $DUMPDIR $BASE
     $RM $V -f $TARDIR/sonic_dump
 

scripts/sonic-kdump-config

@@ -167,7 +167,7 @@ def get_kdump_administrative_mode():
 #
 #  @return The current memory string used for kdump (read from running configuration)
 def get_kdump_memory():
-    (rc, lines, err_str) = run_command("/usr/bin/show kdump memory", use_shell=False)
+    (rc, lines, err_str) = run_command("show kdump memory", use_shell=False)
     try:
         if rc == 0 and len(lines) == 1:
             p = lines[0].find(': ')
@@ -185,7 +185,7 @@ def get_kdump_memory():
 #  @return The maximum number of kernel dump files stored locally
 #          (read from running configuration)
 def get_kdump_num_dumps():
-    (rc, lines, err_str) = run_command("/usr/bin/show kdump num_dumps", use_shell=False)
+    (rc, lines, err_str) = run_command("show kdump num_dumps", use_shell=False)
     try:
         if rc == 0 and len(lines) == 1:
             p = lines[0].find(': ')
@@ -386,7 +386,7 @@ def cmd_kdump_disable(verbose, image=get_current_image()):
 #                 If None, display current value read from running configuration
 def cmd_kdump_memory(verbose, memory):
     if memory is None:
-        (rc, lines, err_str) = run_command("/usr/bin/show kdump memory", use_shell=False);
+        (rc, lines, err_str) = run_command("show kdump memory", use_shell=False);
         print('\n'.join(lines))
     else:
         use_kdump_in_cfg = read_use_kdump()
@@ -403,7 +403,7 @@ def cmd_kdump_memory(verbose, memory):
 #                 If None, display current value read from running configuration
 def cmd_kdump_num_dumps(verbose, num_dumps):
     if num_dumps is None:
-        (rc, lines, err_str) = run_command("/usr/bin/show kdump num_dumps", use_shell=False);
+        (rc, lines, err_str) = run_command("show kdump num_dumps", use_shell=False);
         print('\n'.join(lines))
     else:
         write_num_dumps(num_dumps)

setup.py

@@ -14,7 +14,6 @@
     pkg_resources.require('fastentrypoints')
     import fastentrypoints
 
-import glob
 from setuptools import setup
 
 setup(
@@ -118,10 +117,6 @@
         'scripts/watermarkcfg',
         'scripts/sonic-kdump-config'
     ],
-    data_files=[
-        ('/etc/bash_completion.d', glob.glob('data/etc/bash_completion.d/*')),
-        ('/usr/share/sonic/templates', ['sonic_installer/templates/sonic-environment.j2']),
-    ],
     entry_points={
         'console_scripts': [
             'acl-loader = acl_loader.main:cli',
@@ -151,21 +146,18 @@
             'watchdogutil = watchdogutil.main:watchdogutil',
         ]
     },
-    # NOTE: sonic-utilities also depends on other packages that are either only
-    # available as .whl files or the latest available Debian packages are
-    # out-of-date and we must install newer versions via pip. These
-    # dependencies cannot be listed here, as this package is built as a .deb,
-    # therefore all dependencies will be assumed to also be available as .debs.
-    # These unlistable dependencies are as follows:
-    # - sonic-config-engine
-    # - sonic-py-common
-    # - sonic-py-swsssdk
-    # - tabulate
     install_requires=[
         'click',
         'ipaddress',
+        'jsondiff==1.2.0',
+        'm2crypto',
         'natsort',
-        'm2crypto'
+        'pexpect',
+        'sonic-config-engine',
+        'sonic-py-common',
+        'swsssdk>=2.0.1',
+        'tabulate==0.8.2',
+        'xmltodict==0.12.0'
     ],
     setup_requires= [
         'pytest-runner'

sonic-utilities-data/LICENSE

@@ -0,0 +1,15 @@
+Copyright (C) 2020 Microsoft
+
+This program is free software; you can redistribute it and/or
+modify it under the terms of the GNU General Public License
+as published by the Free Software Foundation; either version 2
+of the License, or (at your option) any later version.
+
+This program is distributed in the hope that it will be useful,
+but WITHOUT ANY WARRANTY; without even the implied warranty of
+MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+GNU General Public License for more details.
+
+You should have received a copy of the GNU General Public License
+along with this program; if not, write to the Free Software
+Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.

sonic-utilities-data/MAINTAINERS

@@ -0,0 +1,7 @@
+# This file describes the maintainers for sonic-device-data
+# See the SONiC project governance document for more information
+
+Name = "Joe LeVeque"
+Email = "jolevequ@microsoft.com"
+Github = jleveque
+Mailinglist = sonicproject@googlegroups.com

sonic-utilities-data/README.md

@@ -0,0 +1,19 @@
+# sonic-utilities-data
+Data files required for SONiC command line utilities
+
+
+## To build
+
+```
+dpkg-buildpackage -rfakeroot -b -us -uc
+```
+
+## To clean
+
+```
+dpkg-buildpackage -rfakeroot -Tclean
+```
+
+---
+
+See the [SONiC Website](http://azure.github.io/SONiC/) for more information about the SONiC project.

sonic-utilities-data/debian/changelog

@@ -0,0 +1,5 @@
+sonic-utilities-data (1.0-1) UNRELEASED; urgency=low
+
+  * Initial release
+
+ -- Joe LeVeque <jolevequ@microsoft.com>  Thu, 17 Sep 2020 01:48:49 +0000

sonic-utilities-data/debian/compat

@@ -0,0 +1 @@
+9

sonic-utilities-data/debian/control

@@ -0,0 +1,11 @@
+Source: sonic-utilities-data
+Maintainer: Joe LeVeque <jolevequ@microsoft.com>
+Section: misc
+Priority: optional
+Standards-Version: 0.1
+Build-Depends: debhelper (>=9)
+
+Package: sonic-utilities-data
+Architecture: all
+Depends: ${misc:Depends} bash-completion
+Description: Data files required for SONiC command line utilities

sonic-utilities-data/debian/install

@@ -0,0 +1,2 @@
+bash_completion.d/    /etc/
+templates/*.j2        /usr/share/sonic/templates/

sonic-utilities-data/debian/rules

@@ -0,0 +1,6 @@
+#!/usr/bin/make -f
+
+build:
+
+%:
+	dh $@