Add Db store

requirements/base.txt

@@ -5,6 +5,7 @@ mytb>=0.0.4
 minibelt
 pyyaml
 requests
+peewee
 pyopenssl
 trio
 quart-trio

setup.py

@@ -42,6 +42,7 @@
         'httpx',
         'mytb',
         'minibelt',
+        'peewee',
         'pyyaml',
         'requests',
         "trio",

timon/commands.py

@@ -139,6 +139,10 @@ def mk_parser():
             help=('specific location for a state file'
                   'only used when creating initial conf')
             ),
+    sub_prs.add_argument(
+            '--dbsqlitefname', default="timon_sqlite.db",
+            help=('location of the sqlite db (relative path to workdir)')
+            ),
     sub_prs.add_argument(
             'probe', nargs="*",
             help="probe_id(s) to execute")

timon/conf/config.py

@@ -13,10 +13,12 @@
 import json
 import logging
 import os
+from pathlib import Path
 
 import minibelt
 
 from timon import plugins
+from timon.db.store import get_store
 
 logger = logging.getLogger()
 
@@ -47,6 +49,7 @@ def __init__(self, int_conf_file):
         plugins_cfg = cfg.get('plugins', {})
         if plugins_cfg:
             self._init_plugins(plugins_cfg)
+        self.dbstore = None
 
     def _init_plugins(self, plugins_cfg):
         for pluginname, pluginparams in plugins_cfg.items():
@@ -66,6 +69,24 @@ async def start_plugins(self, nursery):
     async def stop_plugins(self, nursery):
         await plugins.stop_plugins(nursery=nursery)
 
+    def init_dbstore(self, db_cfg):
+        """
+        Init DbStore object that permits store rslts in a db
+
+        Args:
+            db_cfg (dict): The db config params to pass to the DbStore
+                            constructor
+        """
+        probenames = self.probes.keys()
+        self.dbstore = get_store(**db_cfg)
+        self.dbstore.start(probenames=probenames)
+
+    def stop_dbstore(self):
+        """
+        Properly stop the DbStore
+        """
+        self.dbstore.stop()
+
     def get_state(self):
         """ gets current state of timon
             currently a json file
@@ -145,6 +166,7 @@ def __repr__(self):
 def get_config(fname=None, options=None, reload=False):
     """ gets config from fname or options
         uses a cached version per filename except reload = True
+        + Init and start the db store
         :param fname: path of timon config
         :param reload: if true reloading / recompiling config will be forced
     """
@@ -166,7 +188,6 @@ def get_config(fname=None, options=None, reload=False):
         norm_fname = os.path.join(workdir, options.compiled_config)
 
     cfgname = os.path.join(workdir, options.fname)
-
     # get timestamps of compiled cfg and src cfg
     t_src = os.path.getmtime(cfgname)
     if os.path.isfile(norm_fname):
@@ -179,6 +200,9 @@ def get_config(fname=None, options=None, reload=False):
         options.check = False
         apply_config(options)
     config = TMonConfig(norm_fname)
+    sqlitedbfpath = str(Path(workdir) / options.dbsqlitefname)
+    db_cfg = {"db_fpath": sqlitedbfpath}
+    config.init_dbstore(db_cfg=db_cfg)
     configs[fname] = config
     if norm_fname is None:
         configs[None] = config

timon/db/backends.py

@@ -0,0 +1,135 @@
+#!/usr/bin/env python
+
+# #############################################################################
+# Copyright : (C) 2023 by MHComm. All rights reserved
+#
+# __author__ = "Quentin Laymajoux"
+# __email__ = "info@mhcomm.fr"
+#
+# Name       : timon.db.backends
+"""
+Summary: Backend Classes to use in dbstore
+"""
+# #############################################################################
+from datetime import datetime
+import logging
+from queue import Queue
+from threading import Event
+from threading import Lock
+
+from peewee import SqliteDatabase
+
+logger = logging.getLogger(__name__)
+
+
+class BaseBackend():
+    """
+    This class is unuseful, but helps to know which funcs must be implemented
+    in backends
+    """
+
+    def __init__(self, **db_cfg):
+        raise NotImplementedError("BaseBackend must be inherited")
+
+    def stop(self):
+        """
+        Cleanly stop the backend
+        """
+        raise NotImplementedError("Backend stop func must be implemented")
+
+    def start(self, probenames):
+        """
+        Setup and start the backend
+        """
+        raise NotImplementedError("Backend setup func must be implemented")
+
+    def get_probe_results(self, probename):
+        """
+        Get probe results for a given probename
+        Rslts is a list of dict ordered by datetime
+        """
+        raise NotImplementedError(
+            "Backend get_probe_results func must be implemented")
+
+    def store_probe_result(self, probename, timestamp, msg, status):
+        """Store a probe result
+
+        Args:
+            probename (str): name of the probe
+            timestamp (int|float): timestamp if the probe run
+            msg (str): probe rslt message
+            status (str): status of the probe result
+        """
+        raise NotImplementedError(
+            "Backend store_probe_result func must be implemented")
+
+
+class PeeweeBaseBackend(BaseBackend):
+    """
+    Store probe results in a db via peewee ORM
+    CAUTION: must be inherited and _get_db func must be
+    implemented.
+    """
+    def __init__(self, **db_cfg):
+        self.rsltqueue = Queue(maxsize=10000)
+        self.stopevent = Event()
+        self.queuelock = Lock()
+        self.thread_store = None
+        self.db = None
+
+    def start(self, probenames):
+        self.db = db = self._get_db()
+        from timon.db import peewee_utils
+        self.thread_store = peewee_utils.PeeweeDbStoreThread(self, probenames)
+        peewee_utils.init_db(db)
+        self.thread_store.start()
+
+    def stop(self):
+        logger.info("Stopping Peewee backend")
+        self.stopevent.set()
+        self.thread_store.join()
+        self.db.close()
+
+    def get_probe_results(self, probename):
+        from timon.db import peewee_utils
+        probe_results = []
+        with self.queuelock:
+            rslts = []
+            while not self.rsltqueue.empty():
+                # Search in queue
+                rslt = self.rsltqueue.get()
+                rslts.append(rslt)
+                prbname = rslt["name"]
+                if prbname == probename:
+                    probe_results.append(rslt)
+            for rslt in rslts:
+                # Re-put in queue
+                self.rsltqueue.put(rslt)
+        rslts_in_db = peewee_utils.get_probe_results(probename)
+        return probe_results + rslts_in_db
+
+    def store_probe_result(self, probename, timestamp, msg, status):
+        prb_rslt = {
+            "name": probename,
+            "msg": msg,
+            "status": status,
+            "dt": datetime.fromtimestamp(timestamp),
+        }
+        self.rsltqueue.put(prb_rslt)
+
+    def _get_db(self):
+        raise NotImplementedError(
+            "PeeweeBackend _get_db func must be implemented")
+
+
+class PeeweeSqliteBackend(PeeweeBaseBackend):
+    """
+    Store results in an sqlite db
+    """
+    def __init__(self, **db_cfg):
+        self.db_fpath = db_cfg["db_fpath"]
+        super().__init__(**db_cfg)
+
+    def _get_db(self):
+        db = SqliteDatabase(self.db_fpath)
+        return db

timon/db/models.py

@@ -0,0 +1,42 @@
+#!/usr/bin/env python
+
+# #############################################################################
+# Copyright : (C) 2023 by UNKNOWN. All rights reserved
+#
+# __author__ = "Quentin Laymajoux"
+#
+# Name       : timon.db.models
+"""
+Summary: Peewee models for probe results
+"""
+# #############################################################################
+import datetime
+import logging
+
+from peewee import CharField
+from peewee import DateTimeField
+from peewee import Model
+from peewee import TextField
+
+from timon.conf.flags import FLAG_MAP
+from timon.db import store
+
+logger = logging.getLogger(__name__)
+
+
+db = store.store.backend.db
+
+STATUS_CHOICES = {key: key for key in FLAG_MAP.keys()}
+
+
+class ProbeRslt(Model):
+    """
+    Model of a Probe result object
+    """
+    name = CharField(index=True)
+    dt = DateTimeField(default=datetime.datetime.now)
+    msg = TextField()
+    status = CharField(choices=STATUS_CHOICES)
+
+    class Meta:
+        database = db