Use 'importlib' module in lieu of deprecated 'imp' module (#1450)

Migrate from using the `imp` module to using the `importlib` module. As of Python 3, the `imp` module has been deprecated in favor of the `importlib` module.

Place logic in a new function, `load_module_from_source()` in a new file, `utilities_common/general.py`

Also fix some formatting

config/config_mgmt.py

@@ -2,28 +2,21 @@
 config_mgmt.py provides classes for configuration validation and for Dynamic
 Port Breakout.
 '''
-try:
-    import re
-    import syslog
+import re
+import syslog
+from json import load
+from sys import flags
+from time import sleep as tsleep
 
-    from json import load
-    from time import sleep as tsleep
-    from imp import load_source
-    from jsondiff import diff
-    from sys import flags
+import sonic_yang
+from jsondiff import diff
+from swsssdk import port_util
+from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
+from utilities_common.general import load_module_from_source
 
-    # SONiC specific imports
-    import sonic_yang
-    from swsssdk import port_util
-    from swsscommon.swsscommon import SonicV2Connector, ConfigDBConnector
 
-    # Using load_source to 'import /usr/local/bin/sonic-cfggen as sonic_cfggen'
-    # since /usr/local/bin/sonic-cfggen does not have .py extension.
-    load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
-    from sonic_cfggen import deep_update, FormatConverter
-
-except ImportError as e:
-    raise ImportError("%s - required module not found" % str(e))
+# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension.
+sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
 
 # Globals
 YANG_DIR = "/usr/local/yang-models"
@@ -193,8 +186,8 @@ def readConfigDB(self):
         data = dict()
         configdb = ConfigDBConnector()
         configdb.connect()
-        deep_update(data, FormatConverter.db_to_output(configdb.get_config()))
-        self.configdbJsonIn =  FormatConverter.to_serialized(data)
+        sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.db_to_output(configdb.get_config()))
+        self.configdbJsonIn = sonic_cfggen.FormatConverter.to_serialized(data)
         self.sysLog(syslog.LOG_DEBUG, 'Reading Input from ConfigDB {}'.\
             format(self.configdbJsonIn))
 
@@ -214,9 +207,9 @@ def writeConfigDB(self, jDiff):
         data = dict()
         configdb = ConfigDBConnector()
         configdb.connect(False)
-        deep_update(data, FormatConverter.to_deserialized(jDiff))
+        sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.to_deserialized(jDiff))
         self.sysLog(msg="Write in DB: {}".format(data))
-        configdb.mod_config(FormatConverter.output_to_db(data))
+        configdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data))
 
         return
 

pfcwd/main.py

@@ -1,11 +1,9 @@
+import importlib
 import os
-import imp
 import sys
 
 import click
-
 import utilities_common.cli as clicommon
-
 from natsort import natsorted
 from sonic_py_common.multi_asic import get_external_ports
 from tabulate import tabulate
@@ -27,7 +25,7 @@
         import mock_tables.dbconnector
     if os.environ["UTILITIES_UNIT_TESTING_TOPOLOGY"] == "multi_asic":
         import mock_tables.mock_multi_asic
-        imp.reload(mock_tables.mock_multi_asic)
+        importlib.reload(mock_tables.mock_multi_asic)
         mock_tables.dbconnector.load_namespace_config()
 
 except KeyError:

tests/aclshow_test.py

@@ -1,24 +1,26 @@
-import sys
+import json
 import os
-from imp import load_source
+import sys
 from io import StringIO
 from unittest import mock
 
+from utilities_common.general import load_module_from_source
 
 test_path = os.path.dirname(os.path.abspath(__file__))
 modules_path = os.path.dirname(test_path)
 scripts_path = os.path.join(modules_path, "scripts")
 sys.path.insert(0, modules_path)
 
-load_source('aclshow', scripts_path+'/aclshow')
-from aclshow import *
+# Load the file under test
+aclshow_path = os.path.join(scripts_path, 'aclshow')
+aclshow = load_module_from_source('aclshow', aclshow_path)
 
 from .mock_tables import dbconnector
 
 
 # Expected output for aclshow
-default_output = ''+ \
-"""RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+default_output = """\
+RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 ------------  ------------  ------  ---------------  -------------
 RULE_1        DATAACL         9999              101            100
 RULE_2        DATAACL         9998              201            200
@@ -32,8 +34,8 @@
 """
 
 # Expected output for aclshow -a
-all_output = '' + \
-"""RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+all_output = """\
+RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 ------------  ------------  ------  ---------------  -------------
 RULE_1        DATAACL         9999              101            100
 RULE_2        DATAACL         9998              201            200
@@ -49,35 +51,35 @@
 """
 
 # Expected output for aclshow -r RULE_1 -t DATAACL
-rule1_dataacl_output = '' + \
-"""RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+rule1_dataacl_output = """\
+RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 -----------  ------------  ------  ---------------  -------------
 RULE_1       DATAACL         9999              101            100
 """
 
 # Expected output for aclshow -r RULE_1 -t DATAACL
-rule10_dataacl_output = '' + \
-"""RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+rule10_dataacl_output = """\
+RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 -----------  ------------  ------  ---------------  -------------
 RULE_10      DATAACL         9989             1001           1000
 """
 
 # Expected output for aclshow -a -r RULE_05
-rule05_all_output = ''+ \
-"""RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+rule05_all_output = """\
+RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 -----------  ------------  ------  ---------------  -------------
 RULE_05      DATAACL         9995                0              0
 """
 
 # Expected output for aclshow -r RULE_0
-rule0_output = '' + \
-"""RULE NAME    TABLE NAME    PRIO    PACKETS COUNT    BYTES COUNT
+rule0_output = """\
+RULE NAME    TABLE NAME    PRIO    PACKETS COUNT    BYTES COUNT
 -----------  ------------  ------  ---------------  -------------
 """
 
 # Expected output for aclshow -r RULE_4,RULE_6 -vv
-rule4_rule6_verbose_output = '' + \
-"""Reading ACL info...
+rule4_rule6_verbose_output = """\
+Reading ACL info...
 Total number of ACL Tables: 8
 Total number of ACL Rules: 11
 
@@ -88,15 +90,15 @@
 """
 
 # Expected output for aclshow -t EVERFLOW
-everflow_output = '' + \
-"""RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+everflow_output = """\
+RULE NAME    TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 -----------  ------------  ------  ---------------  -------------
 RULE_6       EVERFLOW        9994              601            600
 """
 
 # Expected output for aclshow -t DATAACL
-dataacl_output = '' + \
-"""RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+dataacl_output = """\
+RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 ------------  ------------  ------  ---------------  -------------
 RULE_1        DATAACL         9999              101            100
 RULE_2        DATAACL         9998              201            200
@@ -113,8 +115,8 @@
 
 # Expected output for
 # aclshow -a -c ; aclshow -a
-all_after_clear_output = '' + \
-"""RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
+all_after_clear_output = """\
+RULE NAME     TABLE NAME      PRIO    PACKETS COUNT    BYTES COUNT
 ------------  ------------  ------  ---------------  -------------
 RULE_1        DATAACL         9999                0              0
 RULE_2        DATAACL         9998                0              0
@@ -129,6 +131,7 @@
 RULE_08       EVERFLOW        9992                0              0
 """
 
+
 class Aclshow():
     def __init__(self, *args, **kwargs):
         """
@@ -146,19 +149,19 @@ def nullify_counters(self):
         This method is used to empty dumped counters
         if exist in /tmp/.counters_acl.p (by default).
         """
-        if os.path.isfile(COUNTER_POSITION):
-            with open(COUNTER_POSITION, 'w') as fp:
+        if os.path.isfile(aclshow.COUNTER_POSITION):
+            with open(aclshow.COUNTER_POSITION, 'w') as fp:
                 json.dump([], fp)
 
     def runTest(self):
         """
         This method invokes main() from aclshow utility (parametrized by argparse)
         parametrized by mock argparse.
         """
-        @mock.patch('argparse.ArgumentParser.parse_args', return_value = argparse.Namespace(**self.kwargs))
-        def run(mock_args):
-            main()
-        run()
+        with mock.patch.object(aclshow.argparse.ArgumentParser,
+                               'parse_args',
+                               return_value=aclshow.argparse.Namespace(**self.kwargs)):
+            aclshow.main()
 
     def setUp(self):
         if self.nullify_on_start:
@@ -173,56 +176,78 @@ def tearDown(self):
         sys.stdout = self.old_stdout
 
 # aclshow
+
+
 def test_default():
-    test = Aclshow(all = None, clear = None, rules = None, tables = None, verbose = None)
+    test = Aclshow(all=None, clear=None, rules=None, tables=None, verbose=None)
     assert test.result.getvalue() == default_output
 
 # aclshow -a
+
+
 def test_all():
-    test = Aclshow(all = True, clear = None, rules = None, tables = None, verbose = None)
+    test = Aclshow(all=True, clear=None, rules=None, tables=None, verbose=None)
     assert test.result.getvalue() == all_output
 
 # aclshow -r RULE_1 -t DATAACL
+
+
 def test_rule1_dataacl():
-    test = Aclshow(all = None, clear = None, rules = 'RULE_1', tables = 'DATAACL', verbose = None)
+    test = Aclshow(all=None, clear=None, rules='RULE_1', tables='DATAACL', verbose=None)
     assert test.result.getvalue() == rule1_dataacl_output
 
 # aclshow -a -r RULE_05
+
+
 def test_rule05_all():
-    test = Aclshow(all = True, clear = None, rules = 'RULE_05', tables = None, verbose = None)
+    test = Aclshow(all=True, clear=None, rules='RULE_05', tables=None, verbose=None)
     assert test.result.getvalue() == rule05_all_output
 
 # aclshow -r RULE_0
+
+
 def test_rule0():
-    test = Aclshow(all = None, clear = None, rules = 'RULE_0', tables = None, verbose = None)
+    test = Aclshow(all=None, clear=None, rules='RULE_0', tables=None, verbose=None)
     assert test.result.getvalue() == rule0_output
 
 # aclshow -r RULE_10 -t DATAACL
+
+
 def test_rule10_lowercase_priority():
-    test = Aclshow(all = None, clear = None, rules = 'RULE_10', tables = 'DATAACL', verbose = None)
+    test = Aclshow(all=None, clear=None, rules='RULE_10', tables='DATAACL', verbose=None)
     assert test.result.getvalue() == rule10_dataacl_output
 
 # aclshow -r RULE_4,RULE_6 -vv
+
+
 def test_rule4_rule6_verbose():
-    test = Aclshow(all = None, clear = None, rules = 'RULE_4,RULE_6', tables = None, verbose = True)
+    test = Aclshow(all=None, clear=None, rules='RULE_4,RULE_6', tables=None, verbose=True)
     assert test.result.getvalue() == rule4_rule6_verbose_output
 
 # aclshow -t EVERFLOW
+
+
 def test_everflow():
     test = Aclshow(all=None, clear=None, rules=None, tables='EVERFLOW', verbose=None)
     assert test.result.getvalue() == everflow_output
 
 # aclshow -t DATAACL
+
+
 def test_dataacl():
     test = Aclshow(all=None, clear=None, rules=None, tables='DATAACL', verbose=None)
     assert test.result.getvalue() == dataacl_output
 
 # aclshow -c
+
+
 def test_clear():
     test = Aclshow(all=None, clear=True, rules=None, tables=None, verbose=None)
     assert test.result.getvalue() == clear_output
 
 # aclshow -a -c ; aclshow -a
+
+
 def test_all_after_clear():
     nullify_on_start, nullify_on_exit = True, False
     test = Aclshow(nullify_on_start, nullify_on_exit, all=True, clear=True, rules=None, tables=None, verbose=None)

tests/buffer_test.py

@@ -1,4 +1,3 @@
-import imp
 import os
 import sys
 from click.testing import CliRunner

tests/config_dpb_test.py

@@ -1,21 +1,22 @@
 import json
 import os
 import re
-from imp import load_source
 from unittest import mock
 
 import pytest
 from click.testing import CliRunner
 from utilities_common.db import Db
+from utilities_common.general import load_module_from_source
 
 import config.main as config
 
-load_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
-from sonic_cfggen import deep_update, FormatConverter
+# Load sonic-cfggen from source since /usr/local/bin/sonic-cfggen does not have .py extension.
+sonic_cfggen = load_module_from_source('sonic_cfggen', '/usr/local/bin/sonic-cfggen')
+
+# Import config_mgmt.py
+config_mgmt_py_path = os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py')
+config_mgmt = load_module_from_source('config_mgmt', config_mgmt_py_path)
 
-load_source('config_mgmt', \
-    os.path.join(os.path.dirname(__file__), '..', 'config', 'config_mgmt.py'))
-import config_mgmt
 
 # Sample platform.json for Test
 BRKOUT_CFG_FILE_JSON = {
@@ -137,14 +138,14 @@ def mock_func(breakout_cfg_file, sonic_db):
 
 def write_config_db(cfgdb, config):
     data = dict()
-    deep_update(data, FormatConverter.to_deserialized(config))
-    cfgdb.mod_config(FormatConverter.output_to_db(data))
+    sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.to_deserialized(config))
+    cfgdb.mod_config(sonic_cfggen.FormatConverter.output_to_db(data))
     return
 
 def read_config_db(cfgdb):
     data = dict()
-    deep_update(data, FormatConverter.db_to_output(cfgdb.get_config()))
-    return FormatConverter.to_serialized(data)
+    sonic_cfggen.deep_update(data, sonic_cfggen.FormatConverter.db_to_output(cfgdb.get_config()))
+    return sonic_cfggen.FormatConverter.to_serialized(data)
 
 def writeJson(d, file):
     with open(file, 'w') as f: