Merge pull request #24 from napalm-automation/fix-script

More improvements, also for the CLI script

napalm_logs/base.py

@@ -65,7 +65,6 @@ def __init__(self,
         self._setup_transport()
         self._build_config()
         self._verify_config()
-        self._precompile_regex()
         # Private vars
         self.__os_proc_map = {}
 
@@ -178,7 +177,6 @@ def _verify_config(self):
             self._verify_config_dict(VALID_CONFIG, dev_config, dev_os)
         log.debug('Read the config without error \o/')
 
-
     def _build_config(self):
         '''
         Build the config of the napalm syslog parser.
@@ -209,11 +207,18 @@ def _build_config(self):
                 continue
             self.config_dict[nos].update(nos_config)
 
-    def _precompile_regex(self):
+    def _respawn_when_dead(self, pid, start_fun, shut_fun=None):
         '''
-        Go through the configuration and precompile all regular expressions,
-        so the parsing should be faster.
+        Restart a process when dead.
+        Requires a thread checking the status using the PID:
+        if not alive anymore, restart.
+
+        :param pid: The process ID.
+        :param start_fun: The process start function.
+        :param shut_fun: the process shutdown function. Not mandatory.
         '''
+        # TODO
+        # TODO requires a fun per process type: server, listener, device
         pass
 
     def start_engine(self):
@@ -261,6 +266,10 @@ def start_engine(self):
             self.__os_proc_map[device_os] = os_proc
         log.debug('Setting up the syslog pipe')
         serve_pipe, listen_pipe = Pipe(duplex=False)
+        log.debug('Serve handle is {shandle} ({shash})'.format(shandle=str(serve_pipe),
+                                                               shash=hash(serve_pipe)))
+        log.debug('Listen handle is {lhandle} ({lhash})'.format(lhandle=str(listen_pipe),
+                                                                lhash=hash(listen_pipe)))
         log.debug('Starting the server process')
         server = NapalmLogsServerProc(serve_pipe,
                                       os_pipe_map,

napalm_logs/config/__init__.py

@@ -17,6 +17,7 @@
 LOG_LEVEL = 'warning'
 LOG_FORMAT = '%(asctime)s,%(msecs)03.0f [%(name)-17s][%(levelname)-8s] %(message)s'
 LOG_FILE = os.path.join(ROOT_DIR, 'var', 'log', 'napalm', 'logs')
+LOG_FILE_CLI_OPTIONS = ('cli', 'screen')
 
 LOGGING_LEVEL = {
     'debug': logging.DEBUG,
@@ -26,6 +27,8 @@
     'critical': logging.CRITICAL
 }
 
+BUFFER_SIZE = 1024
+
 DEFAULT_DELIM = '/'
 
 VALID_CONFIG = {

napalm_logs/device.py

@@ -10,9 +10,11 @@
 import re
 import logging
 import threading
-import napalm_yang
 from datetime import datetime
 
+# Import thrid party libs
+import napalm_yang
+
 # Import napalm-logs pkgs
 from napalm_logs.proc import NapalmLogsProc
 from napalm_logs.config import DEFAULT_DELIM
@@ -185,7 +187,8 @@ def _emit(self, **kwargs):
         try:
             oc_obj.add_model(getattr(napalm_yang.models, kwargs['oc_model']))
         except AttributeError:
-            error_string = 'Unable to load openconfig module {}, please make sure the config is correct'.format(kwargs['oc_model'])
+            error_string = 'Unable to load openconfig module {0},'
+                           ' please make sure the config is correct'.format(kwargs['oc_model'])
             log.error(error_string, exc_info=True)
             raise UnknownOpenConfigModel(error_string)
 
@@ -196,7 +199,8 @@ def _emit(self, **kwargs):
         try:
             oc_obj.load_dict(oc_dict)
         except AttributeError:
-            error_string = 'Error whilst mapping to open config, please check that the mappings are correct for {}'.format(self._name)
+            error_string = 'Error whilst mapping to open config, '
+                           'please check that the mappings are correct for {0}'.format(self._name)
             log.error(error_string, exc_info=True)
             raise OpenConfigPathException(error_string)
 
@@ -244,7 +248,7 @@ def start(self):
                 'timestamp': timestamp,
                 'open_config': oc_obj,
                 'message_details': msg_dict
-                }
+            }
             self._publish(to_publish)
 
     def stop(self):

napalm_logs/exceptions.py

@@ -42,3 +42,10 @@ class OpenConfigPathException(NapalmLogsException):
     Unable to set the open config path specified.
     '''
     pass
+
+
+class NapalmLogsExit(NapalmLogsException):
+    '''
+    Raised on unexpected exit.
+    '''
+    pass

napalm_logs/listener.py

@@ -11,12 +11,11 @@
 import threading
 
 # Import napalm-logs pkgs
+from napalm_logs.config import BUFFER_SIZE
 from napalm_logs.proc import NapalmLogsProc
 
 log = logging.getLogger(__name__)
 
-BUFFER_SIZE = 1024
-
 
 class NapalmLogsListenerProc(NapalmLogsProc):
     '''

napalm_logs/scripts/main.py → napalm_logs/scripts/cli.py

@@ -75,13 +75,13 @@ def prepare(self):
             dest='config_file',
             help=('Config file absolute path. Default: {0}'.format(defaults.CONFIG_FILE))
         )
-        self.add_option(
-            '-d', '--daemon',
-            default=True,
-            dest='daemon',
-            action='store_true',
-            help='Run the {0} as a daemon. Default: %default'.format(self.get_prog_name())
-        )
+        # self.add_option(
+        #     '-d', '--daemon',
+        #     default=True,
+        #     dest='daemon',
+        #     action='store_true',
+        #     help='Run the {0} as a daemon. Default: %default'.format(self.get_prog_name())
+        # )
         self.add_option(
             '-a', '--address',
             dest='address',
@@ -156,16 +156,17 @@ def parse(self, screen_handler):
         config_file_path = self.options.config_file or defaults.CONFIG_FILE
         file_cfg = self.read_config_file(config_file_path)
         log_file = self.options.log_file or file_cfg.get('log_file') or defaults.LOG_FILE
-        log_file_dir = os.path.dirname(log_file)
-        if not os.path.isdir(log_file_dir):
-            log.warning('{0} does not exist, trying to create'.format(log_file_dir))
-            try:
-                os.mkdir(log_file_dir)
-            except OSError:
-                log.error('Unable to create {0}'.format(log_file_dir), exc_info=True)
-                sys.exit(0)
-        log.removeHandler(screen_handler)  # remove printing to the screen
-        logging.basicConfig(filename=log_file)  # log to file
+        if log_file.lower() not in defaults.LOG_FILE_CLI_OPTIONS:
+            log_file_dir = os.path.dirname(log_file)
+            if not os.path.isdir(log_file_dir):
+                log.warning('{0} does not exist, trying to create'.format(log_file_dir))
+                try:
+                    os.mkdir(log_file_dir)
+                except OSError:
+                    log.error('Unable to create {0}'.format(log_file_dir), exc_info=True)
+                    sys.exit(0)
+            log.removeHandler(screen_handler)  # remove printing to the screen
+            logging.basicConfig(filename=log_file)  # log to file
         cfg = {
             'address': self.options.address or file_cfg.get('address') or defaults.ADDRESS,
             'port': self.options.port or file_cfg.get('port') or defaults.PORT,
@@ -182,7 +183,7 @@ def parse(self, screen_handler):
         return cfg
 
 
-def main():
+def napalm_logs_engine():
     if '' in sys.path:
         sys.path.remove('')
     # Temporarily will forward the log entries to the screen
@@ -194,8 +195,12 @@ def main():
     nlop = NLOptionParser()
     config = nlop.parse(screen)
     nl = napalm_logs.NapalmLogs(**config)
-    nl.start_engine()
+    try:
+        nl.start_engine()
+    except KeyboardInterrupt:
+        log.warning('Exiting on Ctrl-c')
+        nl.stop_engine()
 
-if __name__ == '__main__':
-    main()
 
+if __name__ == '__main__':
+    napalm_logs_engine()

napalm_logs/transport/zeromq.py

@@ -7,13 +7,17 @@
 
 # Import stdlib
 import json
+import logging
 
 # Import third party libs
 import zmq
 
 # Import napalm-logs pkgs
+from napalm_logs.exceptions import BindException
 from napalm_logs.transport.base import TransportBase
 
+log = logging.getLogger(__name__)
+
 
 class ZMQTransport(TransportBase):
     '''
@@ -26,10 +30,14 @@ def __init__(self, addr, port):
     def start(self):
         self.context = zmq.Context()
         self.socket = self.context.socket(zmq.PUB)
-        self.socket.bind('tcp://{addr}:{port}'.format(
-            addr=self.addr,
-            port=self.port)
-        )
+        try:
+            self.socket.bind('tcp://{addr}:{port}'.format(
+                addr=self.addr,
+                port=self.port)
+            )
+        except zmq.error.ZMQError as err:
+            log.error(err, exc_info=True)
+            raise BindException(err)
 
     def serialise(self, obj):
         return json.dumps(obj)

setup.py

@@ -36,7 +36,7 @@
     install_requires=reqs,
     entry_points={
         'console_scripts': [
-            'napalm-logs=napalm_logs.scripts.main:main'
+            'napalm-logs=napalm_logs.scripts.cli:napalm_logs_engine'
         ],
     }
 )