From 9bb71b9e90ca9b3cbb17c0e123da1a58f16cc3e8 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 22 Mar 2024 22:47:56 +0100 Subject: [PATCH 01/12] AppSettings, set update interval --- IoTuring/ClassManager/consts.py | 5 +- IoTuring/Configurator/Configuration.py | 8 ++-- IoTuring/Configurator/Configurator.py | 47 ++++++++++++++++--- IoTuring/Configurator/ConfiguratorLoader.py | 39 +++++++++++++-- IoTuring/Configurator/ConfiguratorObject.py | 15 ++++-- IoTuring/Entity/Entity.py | 17 +++++-- .../Deployments/AppSettings/AppSettings.py | 25 ++++++++++ IoTuring/Settings/Settings.py | 45 ++++++++++++++++++ IoTuring/Settings/SettingsManager.py | 32 +++++++++++++ IoTuring/Warehouse/Warehouse.py | 18 ++++--- IoTuring/__init__.py | 13 ++++- 11 files changed, 231 insertions(+), 33 deletions(-) create mode 100644 IoTuring/Settings/Deployments/AppSettings/AppSettings.py create mode 100644 IoTuring/Settings/Settings.py create mode 100644 IoTuring/Settings/SettingsManager.py diff --git a/IoTuring/ClassManager/consts.py b/IoTuring/ClassManager/consts.py index 9e073220c..c2ebdcb43 100644 --- a/IoTuring/ClassManager/consts.py +++ b/IoTuring/ClassManager/consts.py @@ -1,7 +1,10 @@ +# Class keys: KEY_ENTITY = "entity" KEY_WAREHOUSE = "warehouse" +KEY_SETTINGS = "settings" CLASS_PATH = { KEY_ENTITY: "Entity/Deployments", - KEY_WAREHOUSE: "Warehouse/Deployments" + KEY_WAREHOUSE: "Warehouse/Deployments", + KEY_SETTINGS: "Settings/Deployments" } diff --git a/IoTuring/Configurator/Configuration.py b/IoTuring/Configurator/Configuration.py index 69fbb6a93..e6a354e02 100644 --- a/IoTuring/Configurator/Configuration.py +++ b/IoTuring/Configurator/Configuration.py @@ -1,15 +1,15 @@ from __future__ import annotations -from IoTuring.ClassManager.consts import KEY_ENTITY, KEY_WAREHOUSE +from IoTuring.ClassManager.consts import KEY_ENTITY, KEY_WAREHOUSE, KEY_SETTINGS CONFIG_CLASS = { KEY_ENTITY: "active_entities", - KEY_WAREHOUSE: "active_warehouses" + KEY_WAREHOUSE: "active_warehouses", + KEY_SETTINGS: "settings" } BLANK_CONFIGURATION = { - CONFIG_CLASS[KEY_ENTITY]: [{"type": "AppInfo"}], - CONFIG_CLASS[KEY_WAREHOUSE]: [] + CONFIG_CLASS[KEY_ENTITY]: [{"type": "AppInfo"}] } diff --git a/IoTuring/Configurator/Configurator.py b/IoTuring/Configurator/Configurator.py index c1fd1877f..41c8bf72e 100644 --- a/IoTuring/Configurator/Configurator.py +++ b/IoTuring/Configurator/Configurator.py @@ -5,15 +5,13 @@ from IoTuring.Configurator.MenuPreset import QuestionPreset from IoTuring.Configurator.Configuration import FullConfiguration, SingleConfiguration - -from IoTuring.Logger.LogObject import LogObject -from IoTuring.Exceptions.Exceptions import UserCancelledException - -from IoTuring.ClassManager.ClassManager import ClassManager, KEY_ENTITY, KEY_WAREHOUSE - from IoTuring.Configurator import ConfiguratorIO from IoTuring.Configurator import messages +from IoTuring.ClassManager.ClassManager import ClassManager, KEY_ENTITY, KEY_WAREHOUSE, KEY_SETTINGS + +from IoTuring.Logger.LogObject import LogObject +from IoTuring.Exceptions.Exceptions import UserCancelledException from IoTuring.MyApp.SystemConsts import OperatingSystemDetection as OsD from InquirerPy import inquirer @@ -88,6 +86,7 @@ def Menu(self) -> None: mainMenuChoices = [ {"name": "Manage entities", "value": self.ManageEntities}, {"name": "Manage warehouses", "value": self.ManageWarehouses}, + {"name": "Settings", "value": self.ManageSettings}, {"name": "Start IoTuring", "value": self.WriteConfigurations}, {"name": "Help", "value": self.DisplayHelp}, {"name": "Quit", "value": self.Quit}, @@ -167,6 +166,39 @@ def ManageWarehouses(self) -> None: else: self.AddNewWarehouse(choice) + def ManageSettings(self) -> None: + """ UI for App and other Settings """ + + scm = ClassManager(KEY_SETTINGS) + + choices = [] + + availableSettings = scm.ListAvailableClasses() + for sClass in availableSettings: + + choices.append( + {"name": sClass.NAME + " Settings", + "value": sClass}) + + choice = self.DisplayMenu( + choices=choices, + message=f"Select settings to edit" + ) + + if choice == CHOICE_GO_BACK: + self.Menu() + + else: + if not self.IsClassActive(choice): + self.config.configs.append(choice.GetDefaultConfigurations()) + + settings_config = self.config.GetConfigsOfType(choice.NAME)[0] + + self.EditActiveConfiguration( + choice, settings_config) + + self.ManageSettings() + def IsClassActive(self, typeClass) -> bool: """Check if class has an active configuration """ return bool(self.config.GetConfigsOfType(typeClass.NAME)) @@ -206,7 +238,8 @@ def AddNewWarehouse(self, whClass) -> None: def ManageActiveConfiguration(self, typeClass, single_config: SingleConfiguration) -> None: choices = [ - {"name": f"Edit the {typeClass.GetClassKey()} settings", "value": "Edit"}, + {"name": f"Edit the {typeClass.GetClassKey()} settings", + "value": "Edit"}, {"name": f"Remove the {typeClass.GetClassKey()}", "value": "Remove"} ] diff --git a/IoTuring/Configurator/ConfiguratorLoader.py b/IoTuring/Configurator/ConfiguratorLoader.py index 54f07d029..dbb9094b6 100644 --- a/IoTuring/Configurator/ConfiguratorLoader.py +++ b/IoTuring/Configurator/ConfiguratorLoader.py @@ -1,15 +1,18 @@ from __future__ import annotations +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from IoTuring.Entity.Entity import Entity + from IoTuring.Warehouse.Warehouse import Warehouse + from IoTuring.Settings.Settings import Settings + import sys -from IoTuring.Entity.Entity import Entity from IoTuring.Logger.LogObject import LogObject from IoTuring.Configurator.Configurator import Configurator -from IoTuring.ClassManager.ClassManager import ClassManager, KEY_ENTITY, KEY_WAREHOUSE -from IoTuring.Warehouse.Warehouse import Warehouse +from IoTuring.ClassManager.ClassManager import ClassManager, KEY_ENTITY, KEY_WAREHOUSE, KEY_SETTINGS class ConfiguratorLoader(LogObject): - configurator = None def __init__(self, configurator: Configurator) -> None: self.configurations = configurator.config @@ -64,3 +67,31 @@ def LoadEntities(self) -> list[Entity]: # - for each one: # - pass the configuration to the warehouse function that uses the configuration to init the Warehouse # - append the Warehouse to the list + + def LoadSettings(self) -> list[Settings]: + settings = [] + scm = ClassManager(KEY_SETTINGS) + settingsClasses = scm.ListAvailableClasses() + + for sClass in settingsClasses: + + # Check if config was saved: + savedConfigs = self.configurations\ + .GetConfigsOfType(sClass.NAME) + + if savedConfigs: + sc = sClass(savedConfigs[0]) + + # Fallback to default: + else: + sc = sClass(sClass.GetDefaultConfigurations()) + + settings.append(sc) + return settings + + # How Settings configurations work: + # - Settings' configs are added to SettingsManager singleton on main __init__ + # - For advanced usage custom loading could be set up in the class' __init()__ + # - Setting classes has classmethods to get their configs from SettingsManager Singleton, + # SettingsManager shouldn't be called directly, e.g: + # AppSettings.GetFromSettingsConfigurations(CONFIG_KEY_UPDATE_INTERVAL) diff --git a/IoTuring/Configurator/ConfiguratorObject.py b/IoTuring/Configurator/ConfiguratorObject.py index a308cfa25..cccf81549 100644 --- a/IoTuring/Configurator/ConfiguratorObject.py +++ b/IoTuring/Configurator/ConfiguratorObject.py @@ -1,5 +1,5 @@ from IoTuring.Configurator.MenuPreset import BooleanAnswers, MenuPreset -from IoTuring.Configurator.Configuration import SingleConfiguration, CONFIG_CLASS +from IoTuring.Configurator.Configuration import SingleConfiguration, CONFIG_CLASS, CONFIG_KEY_TYPE class ConfiguratorObject: @@ -11,8 +11,7 @@ def __init__(self, single_configuration: SingleConfiguration) -> None: self.configurations = single_configuration # Add missing default values: - preset = self.ConfigurationPreset() - defaults = preset.GetDefaults() + defaults = self.ConfigurationPreset().GetDefaults() if defaults: for default_key, default_value in defaults.items(): @@ -59,3 +58,13 @@ def GetClassKey(cls) -> str: if class_key not in CONFIG_CLASS: raise Exception(f"Invalid class {class_key}") return class_key + + @classmethod + def GetDefaultConfigurations(cls) -> SingleConfiguration: + """Get the default configuration of this class""" + + # Get default configs as dict: + config_dict = cls.ConfigurationPreset().GetDefaults() + config_dict[CONFIG_KEY_TYPE] = cls.NAME + + return SingleConfiguration(cls.GetClassKey(), config_dict=config_dict) diff --git a/IoTuring/Entity/Entity.py b/IoTuring/Entity/Entity.py index 1ee44d46d..3c420517c 100644 --- a/IoTuring/Entity/Entity.py +++ b/IoTuring/Entity/Entity.py @@ -1,15 +1,20 @@ from __future__ import annotations +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from IoTuring.Configurator.Configuration import SingleConfiguration + from IoTuring.Entity.EntityData import EntityData, EntitySensor, EntityCommand, ExtraAttribute + + import time import subprocess from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject -from IoTuring.Configurator.Configuration import SingleConfiguration -from IoTuring.Exceptions.Exceptions import UnknownEntityKeyException from IoTuring.Logger.LogObject import LogObject -from IoTuring.Entity.EntityData import EntityData, EntitySensor, EntityCommand, ExtraAttribute +from IoTuring.Exceptions.Exceptions import UnknownEntityKeyException + from IoTuring.MyApp.SystemConsts import OperatingSystemDetection as OsD -DEFAULT_UPDATE_TIMEOUT = 10 +from IoTuring.Settings.Deployments.AppSettings.AppSettings import AppSettings, CONFIG_KEY_UPDATE_INTERVAL class Entity(ConfiguratorObject, LogObject): @@ -25,7 +30,9 @@ def __init__(self, single_configuration: SingleConfiguration) -> None: # When I update the values this number changes (randomly) so each warehouse knows I have updated self.valuesID = 0 - self.updateTimeout = DEFAULT_UPDATE_TIMEOUT + + self.updateTimeout = int( + AppSettings.GetFromSettingsConfigurations(CONFIG_KEY_UPDATE_INTERVAL)) def Initialize(self): """ Must be implemented in sub-classes, may be useful here to use the configuration """ diff --git a/IoTuring/Settings/Deployments/AppSettings/AppSettings.py b/IoTuring/Settings/Deployments/AppSettings/AppSettings.py new file mode 100644 index 000000000..98fb244c5 --- /dev/null +++ b/IoTuring/Settings/Deployments/AppSettings/AppSettings.py @@ -0,0 +1,25 @@ +from IoTuring.Configurator.MenuPreset import MenuPreset +from IoTuring.Settings.Settings import Settings + + +CONFIG_KEY_UPDATE_INTERVAL = "update_interval" +# CONFIG_KEY_SLOW_INTERVAL = "slow_interval" + + +class AppSettings(Settings): + """Singleton for storing AppSettings, not related to a specifuc Entity or Warehouse """ + NAME = "App" + + @classmethod + def ConfigurationPreset(cls): + preset = MenuPreset() + + preset.AddEntry(name="Main update interval in seconds", + key=CONFIG_KEY_UPDATE_INTERVAL, mandatory=True, + question_type="integer", default=10) + + # preset.AddEntry(name="Secondary update interval in minutes", + # key=CONFIG_KEY_SLOW_INTERVAL, mandatory=True, + # question_type="integer", default=10) + + return preset diff --git a/IoTuring/Settings/Settings.py b/IoTuring/Settings/Settings.py new file mode 100644 index 000000000..ffe851967 --- /dev/null +++ b/IoTuring/Settings/Settings.py @@ -0,0 +1,45 @@ +from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject +from IoTuring.Settings.SettingsManager import SettingsManager +from IoTuring.Configurator.MenuPreset import BooleanAnswers + + +class Settings(ConfiguratorObject): + """Base class for settings""" + NAME = "Settings" + + @classmethod + def GetFromSettingsConfigurations(cls, key: str): + """Get value from settings' saved configurations from SettingsManager + + Args: + key (str): The CONFIG_KEY of the configuration + + Raises: + Exception: If the key not found + + Returns: + Any: The config value + """ + + sM = SettingsManager() + saved_config = sM.GetConfigOfType(cls) + + if saved_config.HasConfigKey(key): + return saved_config.GetConfigValue(key) + else: + raise Exception( + f"Can't find key {key} in SettingsManager configurations") + + @classmethod + def GetTrueOrFalseFromSettingsConfigurations(cls, key: str) -> bool: + """Get boolean value from settings' saved configurations from SettingsManager + + Args: + key (str): The CONFIG_KEY of the configuration + + Returns: + bool: The config value + """ + + value = cls.GetFromSettingsConfigurations(key).lower() + return bool(value in BooleanAnswers.TRUE_ANSWERS) diff --git a/IoTuring/Settings/SettingsManager.py b/IoTuring/Settings/SettingsManager.py new file mode 100644 index 000000000..688be3504 --- /dev/null +++ b/IoTuring/Settings/SettingsManager.py @@ -0,0 +1,32 @@ +from __future__ import annotations +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from IoTuring.Settings.Settings import Settings + from IoTuring.Configurator.Configuration import SingleConfiguration + +from IoTuring.Logger.Logger import Singleton +from IoTuring.Logger.LogObject import LogObject + + +class SettingsManager(LogObject, metaclass=Singleton): + """Singleton for storing configurations of Settings""" + + def __init__(self) -> None: + self.setting_configs = {} + + def AddSettings(self, setting_entities: list[Settings]) -> None: + """Add settings configuration + + Args: + setting_entities (list[Settings]): The loaded settings classes + """ + for setting_entity in setting_entities: + self.setting_configs[setting_entity.NAME] = setting_entity.configurations + + def GetConfigOfType(self, setting_class) -> SingleConfiguration: + """Get the configuration of a saved class. Raises exception if not found""" + + if setting_class.NAME in self.setting_configs: + return self.setting_configs[setting_class.NAME] + else: + raise Exception(f"No settings config for {setting_class.NAME}") diff --git a/IoTuring/Warehouse/Warehouse.py b/IoTuring/Warehouse/Warehouse.py index 1acd5bb19..8b3ccb151 100644 --- a/IoTuring/Warehouse/Warehouse.py +++ b/IoTuring/Warehouse/Warehouse.py @@ -1,14 +1,17 @@ from __future__ import annotations -from IoTuring.Entity.Entity import Entity -from IoTuring.Logger.LogObject import LogObject -from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject -from IoTuring.Configurator.Configuration import SingleConfiguration -from IoTuring.Entity.EntityManager import EntityManager +from typing import TYPE_CHECKING +if TYPE_CHECKING: + from IoTuring.Entity.Entity import Entity + from IoTuring.Configurator.Configuration import SingleConfiguration from threading import Thread import time -DEFAULT_LOOP_TIMEOUT = 10 +from IoTuring.Logger.LogObject import LogObject +from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject +from IoTuring.Entity.EntityManager import EntityManager + +from IoTuring.Settings.Deployments.AppSettings.AppSettings import AppSettings, CONFIG_KEY_UPDATE_INTERVAL class Warehouse(ConfiguratorObject, LogObject): @@ -16,7 +19,8 @@ class Warehouse(ConfiguratorObject, LogObject): def __init__(self, single_configuration: SingleConfiguration) -> None: super().__init__(single_configuration) - self.loopTimeout = DEFAULT_LOOP_TIMEOUT + self.loopTimeout = int( + AppSettings.GetFromSettingsConfigurations(CONFIG_KEY_UPDATE_INTERVAL)) def Start(self) -> None: """ Initial configuration and start the thread that will loop the Warehouse.Loop() function""" diff --git a/IoTuring/__init__.py b/IoTuring/__init__.py index 6dfa3682c..d90d75246 100644 --- a/IoTuring/__init__.py +++ b/IoTuring/__init__.py @@ -9,6 +9,7 @@ from IoTuring.Configurator.Configurator import Configurator from IoTuring.Configurator.ConfiguratorLoader import ConfiguratorLoader from IoTuring.Entity.EntityManager import EntityManager +from IoTuring.Settings.SettingsManager import SettingsManager from IoTuring.Logger.Logger import Logger from IoTuring.Logger.Colors import Colors @@ -72,9 +73,17 @@ def loop(): # This have to start after configurator.Menu(), otherwise won't work starting from the menu signal.signal(signal.SIGINT, Exit_SIGINT_handler) + # Load Settings: + settings = ConfiguratorLoader(configurator).LoadSettings() + sM = SettingsManager() + sM.AddSettings(settings) + logger.Log(Logger.LOG_INFO, "App", App()) # Print App info - logger.Log(Logger.LOG_INFO, "Configurator", - "Run the script with -c to enter configuration mode") + + # Add help if not started from Configurator + if not args.configurator: + logger.Log(Logger.LOG_INFO, "Configurator", + "Run the script with -c to enter configuration mode") eM = EntityManager() From 8afc49e072fee1b0ad3dfba8669e62c27b8ef72f Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 23 Mar 2024 05:20:43 +0100 Subject: [PATCH 02/12] Add retry interval setting --- .../Deployments/AppSettings/AppSettings.py | 7 +++++++ .../HomeAssistantWarehouse.py | 15 ++++++++++----- .../Deployments/MQTTWarehouse/MQTTWarehouse.py | 3 +-- IoTuring/Warehouse/Warehouse.py | 4 +++- 4 files changed, 21 insertions(+), 8 deletions(-) diff --git a/IoTuring/Settings/Deployments/AppSettings/AppSettings.py b/IoTuring/Settings/Deployments/AppSettings/AppSettings.py index 98fb244c5..b0eabe5dd 100644 --- a/IoTuring/Settings/Deployments/AppSettings/AppSettings.py +++ b/IoTuring/Settings/Deployments/AppSettings/AppSettings.py @@ -3,6 +3,7 @@ CONFIG_KEY_UPDATE_INTERVAL = "update_interval" +CONFIG_KEY_RETRY_INTERVAL = "retry_interval" # CONFIG_KEY_SLOW_INTERVAL = "slow_interval" @@ -18,6 +19,12 @@ def ConfigurationPreset(cls): key=CONFIG_KEY_UPDATE_INTERVAL, mandatory=True, question_type="integer", default=10) + preset.AddEntry(name="Connection retry interval in seconds", + instruction="If broker is not available retry after this amount of time passed", + key=CONFIG_KEY_RETRY_INTERVAL, mandatory=True, + question_type="integer", default=1) + + # preset.AddEntry(name="Secondary update interval in minutes", # key=CONFIG_KEY_SLOW_INTERVAL, mandatory=True, # question_type="integer", default=10) diff --git a/IoTuring/Warehouse/Deployments/HomeAssistantWarehouse/HomeAssistantWarehouse.py b/IoTuring/Warehouse/Deployments/HomeAssistantWarehouse/HomeAssistantWarehouse.py index 5e34bbaa6..dc011e6ed 100644 --- a/IoTuring/Warehouse/Deployments/HomeAssistantWarehouse/HomeAssistantWarehouse.py +++ b/IoTuring/Warehouse/Deployments/HomeAssistantWarehouse/HomeAssistantWarehouse.py @@ -15,11 +15,12 @@ from IoTuring.Logger import consts from IoTuring.Entity.ValueFormat import ValueFormatter +from IoTuring.Settings.Deployments.AppSettings.AppSettings import AppSettings, CONFIG_KEY_UPDATE_INTERVAL + INCLUDE_UNITS_IN_SENSORS = False INCLUDE_UNITS_IN_EXTRA_ATTRIBUTES = True -SLEEP_TIME_NOT_CONNECTED_WHILE = 1 # That stands for: App name, Client name, EntityData Id TOPIC_DATA_FORMAT = "{}/{}HomeAssistant/{}" @@ -240,8 +241,12 @@ def __init__(self, entityData: EntitySensor, wh: "HomeAssistantWarehouse") -> N if self.supports_extra_attributes: self.AddTopic("json_attributes_topic") - # Extra payload for sensors: - self.discovery_payload['expire_after'] = 600 # TODO Improve + # Make sure expire_after is greater than the loop timeout: + loop_timeout = int(AppSettings.GetFromSettingsConfigurations( + CONFIG_KEY_UPDATE_INTERVAL)) + sensor_expire_seconds = 600 if loop_timeout < 600 else int( + loop_timeout * 1.5) + self.discovery_payload['expire_after'] = sensor_expire_seconds def SendValues(self): """ Send values of the sensor to the state topic """ @@ -404,7 +409,7 @@ def RegisterEntityCommands(self): def Loop(self): while (not self.client.IsConnected()): - time.sleep(SLEEP_TIME_NOT_CONNECTED_WHILE) + time.sleep(self.retry_interval) # Mechanism to call the function to send discovery data every CONFIGURATION_SEND_LOOP_SKIP_NUMBER loop if self.loopCounter == 0: @@ -434,7 +439,7 @@ def MakeValuesTopic(self, topic_suffix: str) -> str: def NormalizeTopic(topicstring: str) -> str: """ Home assistant requires stricter topic names """ # Remove non ascii characters: - topicstring=topicstring.encode("ascii", "ignore").decode() + topicstring = topicstring.encode("ascii", "ignore").decode() return MQTTClient.NormalizeTopic(topicstring).replace(" ", "_") @classmethod diff --git a/IoTuring/Warehouse/Deployments/MQTTWarehouse/MQTTWarehouse.py b/IoTuring/Warehouse/Deployments/MQTTWarehouse/MQTTWarehouse.py index 09eb2727f..8530124fb 100644 --- a/IoTuring/Warehouse/Deployments/MQTTWarehouse/MQTTWarehouse.py +++ b/IoTuring/Warehouse/Deployments/MQTTWarehouse/MQTTWarehouse.py @@ -9,7 +9,6 @@ import time -SLEEP_TIME_NOT_CONNECTED_WHILE = 1 TOPIC_FORMAT = "{}/{}/{}" # That stands for: App name, Client name, EntityData Id @@ -53,7 +52,7 @@ def RegisterEntityCommands(self): def Loop(self): while(not self.client.IsConnected()): - time.sleep(SLEEP_TIME_NOT_CONNECTED_WHILE) + time.sleep(self.retry_interval) # Here in Loop I send sensor's data (command callbacks are not managed here) for entity in self.GetEntities(): diff --git a/IoTuring/Warehouse/Warehouse.py b/IoTuring/Warehouse/Warehouse.py index 8b3ccb151..15e586dd7 100644 --- a/IoTuring/Warehouse/Warehouse.py +++ b/IoTuring/Warehouse/Warehouse.py @@ -11,7 +11,7 @@ from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject from IoTuring.Entity.EntityManager import EntityManager -from IoTuring.Settings.Deployments.AppSettings.AppSettings import AppSettings, CONFIG_KEY_UPDATE_INTERVAL +from IoTuring.Settings.Deployments.AppSettings.AppSettings import AppSettings, CONFIG_KEY_UPDATE_INTERVAL, CONFIG_KEY_RETRY_INTERVAL class Warehouse(ConfiguratorObject, LogObject): @@ -21,6 +21,8 @@ def __init__(self, single_configuration: SingleConfiguration) -> None: self.loopTimeout = int( AppSettings.GetFromSettingsConfigurations(CONFIG_KEY_UPDATE_INTERVAL)) + self.retry_interval = int(AppSettings + .GetFromSettingsConfigurations(CONFIG_KEY_RETRY_INTERVAL)) def Start(self) -> None: """ Initial configuration and start the thread that will loop the Warehouse.Loop() function""" From 8843b74aab4bc9321a20a1482df2d61d782341ab Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 30 Mar 2024 17:22:39 +0100 Subject: [PATCH 03/12] wip --- IoTuring/Logger/Logger.py | 286 +++++++++++++++++++++----------------- IoTuring/Logger/consts.py | 12 +- IoTuring/__init__.py | 4 +- 3 files changed, 166 insertions(+), 136 deletions(-) diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 4cc098b45..219ebc579 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -5,6 +5,7 @@ import json import threading from io import TextIOWrapper +import logging from IoTuring.Logger import consts from IoTuring.Logger.LogLevel import LogLevelObject, LogLevel @@ -18,7 +19,7 @@ class Singleton(type): _instances = {} - def __call__(cls): + def __call__(cls, *args, **kwargs): if cls not in cls._instances: cls._instances[cls] = super(Singleton, cls).__call__() return cls._instances[cls] @@ -26,143 +27,172 @@ def __call__(cls): class Logger(LogLevelObject, metaclass=Singleton): - lock = threading.Lock() + # lock = threading.Lock() - log_filename = "" - log_file_descriptor = None + # log_filename = "" + # log_file_descriptor = None - # Default log levels: - console_log_level = LogLevel(consts.CONSOLE_LOG_LEVEL) - file_log_level = LogLevel(consts.FILE_LOG_LEVEL) + # # Default log levels: + # console_log_level = LogLevel(consts.CONSOLE_LOG_LEVEL) + # file_log_level = LogLevel(consts.FILE_LOG_LEVEL) def __init__(self) -> None: - self.terminalSupportsColors = Logger.checkTerminalSupportsColors() - - # Prepare the log - self.SetLogFilename() - # Open the file descriptor - self.GetLogFileDescriptor() - - # Override log level from envvar: - try: - if os.getenv("IOTURING_LOG_LEVEL"): - level_override = LogLevel(str(os.getenv("IOTURING_LOG_LEVEL"))) - self.console_log_level = level_override - except UnknownLoglevelException as e: - self.Log(self.LOG_ERROR, "Logger", - f"Unknown Loglevel: {e.loglevel}") - - def SetLogFilename(self) -> str: - """ Set filename with timestamp and also call setup folder """ - dateTimeObj = datetime.now() - self.log_filename = os.path.join( - self.SetupFolder(), dateTimeObj.strftime(consts.LOG_FILENAME_FORMAT).replace(":", "_")) - return self.log_filename - - def SetupFolder(self) -> str: - """ Check if exists (or create) the folder of logs inside this file's folder """ - thisFolder = os.path.dirname(inspect.getfile(Logger)) - newFolder = os.path.join(thisFolder, consts.LOGS_FOLDER) - if not os.path.exists(newFolder): - os.mkdir(newFolder) - - return newFolder - - def GetMessageDatetimeString(self) -> str: - now = datetime.now() - return now.strftime(consts.MESSAGE_DATETIME_FORMAT) + self.logger = logging.getLogger() + self.logger.setLevel(logging.NOTSET) + + # formatter = logging.Formatter('[ %(asctime)s - %(name)s - %(levelname)s - %(message)s') + formatter = logging.Formatter( + fmt='[ {asctime:s} | {levelname:^10s} | {name:^30s} | {message:s}{multilines:s}', + datefmt='%Y-%m-%d %H:%M:%S', + style='{' + ) + + ch = logging.StreamHandler() + ch.setLevel(logging.DEBUG) + ch.setFormatter(formatter) + self.logger.addHandler(ch) + + # self.terminalSupportsColors = Logger.checkTerminalSupportsColors() + + # # Prepare the log + # self.SetLogFilename() + # # Open the file descriptor + # self.GetLogFileDescriptor() + + # # Override log level from envvar: + # try: + # if os.getenv("IOTURING_LOG_LEVEL"): + # level_override = LogLevel(str(os.getenv("IOTURING_LOG_LEVEL"))) + # self.console_log_level = level_override + # except UnknownLoglevelException as e: + # self.Log(self.LOG_ERROR, "Logger", + # f"Unknown Loglevel: {e.loglevel}") + + # def SetLogFilename(self) -> str: + # """ Set filename with timestamp and also call setup folder """ + # dateTimeObj = datetime.now() + # self.log_filename = os.path.join( + # self.SetupFolder(), dateTimeObj.strftime(consts.LOG_FILENAME_FORMAT).replace(":", "_")) + # return self.log_filename + + # def SetupFolder(self) -> str: + # """ Check if exists (or create) the folder of logs inside this file's folder """ + # thisFolder = os.path.dirname(inspect.getfile(Logger)) + # newFolder = os.path.join(thisFolder, consts.LOGS_FOLDER) + # if not os.path.exists(newFolder): + # os.mkdir(newFolder) + + # return newFolder + + # def GetMessageDatetimeString(self) -> str: + # now = datetime.now() + # return now.strftime(consts.MESSAGE_DATETIME_FORMAT) # LOG def Log(self, loglevel: LogLevel, source: str, message, printToConsole=True, writeToFile=True) -> None: - if type(message) == dict: - self.LogDict(loglevel, source, message, - printToConsole, writeToFile) - return # Log dict will call this function so I don't need to go down at the moment - elif type(message) == list: - self.LogList(loglevel, source, message, - printToConsole, writeToFile) - return # Log list will call this function so I don't need to go down at the moment - - message = str(message) - # Call this function for each line of the message if there are more than one line. - messageLines = message.split("\n") - if len(messageLines) > 1: - for line in messageLines: - self.Log(loglevel, source, line, - printToConsole, writeToFile) - return # Stop the function then because I've already called this function from each line so I don't have to go down here - - prestring = f"[ {self.GetMessageDatetimeString()} | {str(loglevel).center(consts.STRINGS_LENGTH[0])} | " \ - + f"{source.center(consts.STRINGS_LENGTH[1])}]{consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' '}" # justify - - # Manage string to print in more lines if it's too long - while len(message) > 0: - string = prestring+message[:consts.MESSAGE_WIDTH] - # Cut for next iteration if message is longer than a line - message = message[consts.MESSAGE_WIDTH:] - # then I add the dash to the row - if (len(message) > 0 and string[-1] != " " and string[-1] != "." and string[-1] != ","): - string = string + '-' # Print new line indicator if I will go down in the next iteration - self.PrintAndSave(string, loglevel, printToConsole, writeToFile) - # -1 + space cause if the char in the prestring isn't a space, it will be directly attached to my message without a space - - prestring = (len(prestring)-consts.PRESTRING_MESSAGE_SEPARATOR_LEN) * \ - consts.LONG_MESSAGE_PRESTRING_CHAR+consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' ' - - def LogDict(self, loglevel, source, message_dict: dict, *args): - try: - string = json.dumps(message_dict, indent=4, sort_keys=False, - default=lambda o: '') - lines = string.splitlines() - for line in lines: - self.Log(loglevel, source, "> "+line, *args) - except Exception as e: - self.Log(self.LOG_ERROR, source, "Can't print dictionary content") - - def LogList(self, loglevel, source, message_list: list, *args): - try: - for index, item in enumerate(message_list): - if type(item) == dict or type(item) == list: - self.Log(loglevel, source, "Item #"+str(index), *args) - self.Log(loglevel, source, item, *args) - else: - self.Log(loglevel, source, - f"{str(index)}: {str(item)}", *args) - - except: - self.Log(self.LOG_ERROR, source, "Can't print dictionary content") - - # Both print and save to file - def PrintAndSave(self, string, loglevel: LogLevel, printToConsole=True, writeToFile=True) -> None: - - if printToConsole and int(loglevel) <= int(self.console_log_level): - self.ColoredPrint(string, loglevel) - - if writeToFile and int(loglevel) <= int(self.file_log_level): - # acquire the lock - with self.lock: - self.GetLogFileDescriptor().write(string+' \n') - # so I can see the log in real time from a reader - self.GetLogFileDescriptor().flush() - - def ColoredPrint(self, string, loglevel: LogLevel) -> None: - if not self.terminalSupportsColors: - print(string) - else: - print(loglevel.get_colored_string(string)) - - def GetLogFileDescriptor(self) -> TextIOWrapper: - if self.log_file_descriptor is None: - self.log_file_descriptor = open(self.log_filename, "a", encoding="utf-8") - return self.log_file_descriptor - - def CloseFile(self) -> None: - if self.log_file_descriptor is not None: - self.log_file_descriptor.close() - self.log_file_descriptor = None + if isinstance(message, str) and len(message) > 10: + lines = [message[i:i+10] for i in range(0, len(message), 10)] + ml_string = "\n ".join(lines) + extra = {"multilines": ml_string} + message = "" + else: + extra = {"multilines": ""} + + + + l = logging.getLogger(source) + l.log(int(loglevel), message, extra=extra) + + # if type(message) == dict: + # self.LogDict(loglevel, source, message, + # printToConsole, writeToFile) + # return # Log dict will call this function so I don't need to go down at the moment + # elif type(message) == list: + # self.LogList(loglevel, source, message, + # printToConsole, writeToFile) + # return # Log list will call this function so I don't need to go down at the moment + + # message = str(message) + # # Call this function for each line of the message if there are more than one line. + # messageLines = message.split("\n") + # if len(messageLines) > 1: + # for line in messageLines: + # self.Log(loglevel, source, line, + # printToConsole, writeToFile) + # return # Stop the function then because I've already called this function from each line so I don't have to go down here + + # prestring = f"[ {self.GetMessageDatetimeString()} | {str(loglevel).center(consts.STRINGS_LENGTH[0])} | " \ + # + f"{source.center(consts.STRINGS_LENGTH[1])}]{consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' '}" # justify + + # # Manage string to print in more lines if it's too long + # while len(message) > 0: + # string = prestring+message[:consts.MESSAGE_WIDTH] + # # Cut for next iteration if message is longer than a line + # message = message[consts.MESSAGE_WIDTH:] + # # then I add the dash to the row + # if (len(message) > 0 and string[-1] != " " and string[-1] != "." and string[-1] != ","): + # string = string + '-' # Print new line indicator if I will go down in the next iteration + # self.PrintAndSave(string, loglevel, printToConsole, writeToFile) + # # -1 + space cause if the char in the prestring isn't a space, it will be directly attached to my message without a space + + # prestring = (len(prestring)-consts.PRESTRING_MESSAGE_SEPARATOR_LEN) * \ + # consts.LONG_MESSAGE_PRESTRING_CHAR+consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' ' + + # def LogDict(self, loglevel, source, message_dict: dict, *args): + # try: + # string = json.dumps(message_dict, indent=4, sort_keys=False, + # default=lambda o: '') + # lines = string.splitlines() + # for line in lines: + # self.Log(loglevel, source, "> "+line, *args) + # except Exception as e: + # self.Log(self.LOG_ERROR, source, "Can't print dictionary content") + + # def LogList(self, loglevel, source, message_list: list, *args): + # try: + # for index, item in enumerate(message_list): + # if type(item) == dict or type(item) == list: + # self.Log(loglevel, source, "Item #"+str(index), *args) + # self.Log(loglevel, source, item, *args) + # else: + # self.Log(loglevel, source, + # f"{str(index)}: {str(item)}", *args) + + # except: + # self.Log(self.LOG_ERROR, source, "Can't print dictionary content") + + # # Both print and save to file + # def PrintAndSave(self, string, loglevel: LogLevel, printToConsole=True, writeToFile=True) -> None: + + # if printToConsole and int(loglevel) <= int(self.console_log_level): + # self.ColoredPrint(string, loglevel) + + # if writeToFile and int(loglevel) <= int(self.file_log_level): + # # acquire the lock + # with self.lock: + # self.GetLogFileDescriptor().write(string+' \n') + # # so I can see the log in real time from a reader + # self.GetLogFileDescriptor().flush() + + # def ColoredPrint(self, string, loglevel: LogLevel) -> None: + # if not self.terminalSupportsColors: + # print(string) + # else: + # print(loglevel.get_colored_string(string)) + + # def GetLogFileDescriptor(self) -> TextIOWrapper: + # if self.log_file_descriptor is None: + # self.log_file_descriptor = open(self.log_filename, "a", encoding="utf-8") + + # return self.log_file_descriptor + + # def CloseFile(self) -> None: + # if self.log_file_descriptor is not None: + # self.log_file_descriptor.close() + # self.log_file_descriptor = None @staticmethod def checkTerminalSupportsColors(): diff --git a/IoTuring/Logger/consts.py b/IoTuring/Logger/consts.py index 8678c2e5a..23b03e655 100644 --- a/IoTuring/Logger/consts.py +++ b/IoTuring/Logger/consts.py @@ -7,37 +7,37 @@ { "const": "LOG_MESSAGE", "string": "Message", - "number": 0, + "number": 50, "color": "green" }, { "const": "LOG_ERROR", "string": "Error", - "number": 1, + "number": 40, "color": "red" }, { "const": "LOG_WARNING", "string": "Warning", - "number": 2, + "number": 30, "color": "yellow" }, { "const": "LOG_INFO", "string": "Info", - "number": 3, + "number": 20, }, { "const": "LOG_DEBUG", "string": "Debug", - "number": 4, + "number": 10, }, { "const": "LOG_DEVELOPMENT", "string": "Dev", - "number": 5, + "number": 0, } ] diff --git a/IoTuring/__init__.py b/IoTuring/__init__.py index d90d75246..38e5768af 100644 --- a/IoTuring/__init__.py +++ b/IoTuring/__init__.py @@ -47,7 +47,7 @@ def loop(): sys.exit("Error: Invalid arguments!") # Clear the terminal - Configurator.ClearTerminal() + # Configurator.ClearTerminal() # Start logger: logger = Logger() @@ -130,5 +130,5 @@ def Exit_SIGINT_handler(sig=None, frame=None): logger.Log(Logger.LOG_INFO, "Main", text, writeToFile=False) # to terminal - logger.CloseFile() + # logger.CloseFile() sys.exit(0) From ff0df66169a1bc33d97fe3e6c21f2cc8e50715cc Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 4 May 2024 12:47:47 +0200 Subject: [PATCH 04/12] Mostly finished log rewrite --- IoTuring/ClassManager/ClassManager.py | 4 +- IoTuring/Configurator/Configurator.py | 13 +- IoTuring/Configurator/ConfiguratorIO.py | 8 +- IoTuring/Logger/LogLevel.py | 51 +-- IoTuring/Logger/LogObject.py | 8 +- IoTuring/Logger/Logger.py | 354 +++++++++--------- IoTuring/Logger/consts.py | 57 +-- .../MyApp/SystemConsts/TerminalDetection.py | 44 +++ IoTuring/MyApp/SystemConsts/__init__.py | 3 +- .../Deployments/LogSettings/LogSettings.py | 101 +++++ .../ConsoleWarehouse/ConsoleWarehouse.py | 2 +- IoTuring/__init__.py | 22 +- 12 files changed, 375 insertions(+), 292 deletions(-) create mode 100644 IoTuring/MyApp/SystemConsts/TerminalDetection.py create mode 100644 IoTuring/Settings/Deployments/LogSettings/LogSettings.py diff --git a/IoTuring/ClassManager/ClassManager.py b/IoTuring/ClassManager/ClassManager.py index a522fc0de..751acc27c 100644 --- a/IoTuring/ClassManager/ClassManager.py +++ b/IoTuring/ClassManager/ClassManager.py @@ -51,7 +51,7 @@ def GetModuleFilePaths(self) -> list[Path]: if not classesRootPath.exists: raise Exception(f"Path does not exist: {classesRootPath}") - self.Log(self.LOG_DEVELOPMENT, + self.Log(self.LOG_DEBUG, f'Looking for python files in "{classesRootPath}"...') python_files = classesRootPath.rglob("*.py") @@ -63,7 +63,7 @@ def GetModuleFilePaths(self) -> list[Path]: raise FileNotFoundError( f"No module files found in {classesRootPath}") - self.Log(self.LOG_DEVELOPMENT, + self.Log(self.LOG_DEBUG, f"Found {str(len(filepaths))} modules files") return filepaths diff --git a/IoTuring/Configurator/Configurator.py b/IoTuring/Configurator/Configurator.py index 41c8bf72e..532b10f64 100644 --- a/IoTuring/Configurator/Configurator.py +++ b/IoTuring/Configurator/Configurator.py @@ -1,6 +1,5 @@ import os import subprocess -import shutil import sys from IoTuring.Configurator.MenuPreset import QuestionPreset @@ -13,6 +12,7 @@ from IoTuring.Logger.LogObject import LogObject from IoTuring.Exceptions.Exceptions import UserCancelledException from IoTuring.MyApp.SystemConsts import OperatingSystemDetection as OsD +from IoTuring.MyApp.SystemConsts import TerminalDetection from InquirerPy import inquirer from InquirerPy.separator import Separator @@ -457,16 +457,13 @@ def DisplayMenu(self, choices: list, message: str = "", add_back_choice=True, ** # Default max_height: kwargs["max_height"] = "100%" - # Actual lines in the terminal. fallback to 0 on error: - terminal_lines = shutil.get_terminal_size(fallback=(0, 0)).lines - # Check for pinned messages: - if terminal_lines > 0 and self.pinned_lines > 0: + if TerminalDetection.CheckTerminalSupportsSize() and self.pinned_lines > 0: # Lines of message and instruction if too long: if "instruction" in kwargs: - message_lines = ((len(kwargs["instruction"]) + len(message) + 3) - / shutil.get_terminal_size().columns) // 1 + message_lines = TerminalDetection.CalculateNumberOfLines( + len(kwargs["instruction"]) + len(message) + 3) # Add only the line of the message: else: message_lines = 1 @@ -475,6 +472,8 @@ def DisplayMenu(self, choices: list, message: str = "", add_back_choice=True, ** required_lines = len(choices) + \ self.pinned_lines + message_lines + terminal_lines = TerminalDetection.GetTerminalLines() + # Set the calculated height: if required_lines > terminal_lines: kwargs["max_height"] = terminal_lines \ diff --git a/IoTuring/Configurator/ConfiguratorIO.py b/IoTuring/Configurator/ConfiguratorIO.py index a06688135..c2becf3d0 100644 --- a/IoTuring/Configurator/ConfiguratorIO.py +++ b/IoTuring/Configurator/ConfiguratorIO.py @@ -33,7 +33,7 @@ def readConfigurations(self): try: with open(self.getFilePath(), "r", encoding="utf-8") as f: config = json.loads(f.read()) - self.Log(self.LOG_MESSAGE, f"Loaded \"{self.getFilePath()}\"") + self.Log(self.LOG_INFO, f"Loaded \"{self.getFilePath()}\"") except FileNotFoundError: self.Log(self.LOG_WARNING, f"It seems you don't have a configuration yet. Use configuration mode (-c) to enable your favourite entities and warehouses.\ \nConfigurations will be saved in \"{str(self.getFolderPath())}\"") @@ -48,7 +48,7 @@ def writeConfigurations(self, data): self.createFolderPathIfDoesNotExist() with open(self.getFilePath(), "w", encoding="utf-8") as f: f.write(json.dumps(data, indent=4, ensure_ascii=False)) - self.Log(self.LOG_MESSAGE, f"Saved \"{str(self.getFilePath())}\"") + self.Log(self.LOG_INFO, f"Saved \"{str(self.getFilePath())}\"") except Exception as e: self.Log(self.LOG_ERROR, f"Error saving configuration file: {str(e)}") sys.exit(str(e)) @@ -152,7 +152,7 @@ def manageOldConfig(self, moveFile: bool) -> None: self.createFolderPathIfDoesNotExist() # copy file from old to new location self.oldFolderPath().joinpath(CONFIGURATION_FILE_NAME).rename(self.getFilePath()) - self.Log(self.LOG_MESSAGE, + self.Log(self.LOG_INFO, f"Copied to \"{str(self.getFilePath())}\"") else: # create dont move file @@ -161,7 +161,7 @@ def manageOldConfig(self, moveFile: bool) -> None: "This file is here to remember you that you don't want to move the configuration file into the new location.", "If you want to move it, delete this file and run the script in -c mode." ])) - self.Log(self.LOG_MESSAGE, " ".join([ + self.Log(self.LOG_INFO, " ".join([ "You won't be asked again. A new blank configuration will be used;", f"if you want to move the existing configuration file, delete \"{self.oldFolderPath().joinpath(DONT_MOVE_FILE_FILENAME)}", "and run the script in -c mode." diff --git a/IoTuring/Logger/LogLevel.py b/IoTuring/Logger/LogLevel.py index 01575055e..4b4285692 100644 --- a/IoTuring/Logger/LogLevel.py +++ b/IoTuring/Logger/LogLevel.py @@ -1,4 +1,4 @@ -from IoTuring.Logger import consts +import logging from IoTuring.Logger.Colors import Colors from IoTuring.Exceptions.Exceptions import UnknownLoglevelException @@ -7,19 +7,22 @@ class LogLevel: """ A loglevel with numeric and string values""" def __init__(self, level_const: str) -> None: - level_dict = next( - (l for l in consts.LOG_LEVELS if l["const"] == level_const), None) - if not level_dict: + self.string = level_const.upper() + if self.string.startswith("LOG_"): + self.string = self.string[4:] + + try: + self.number = getattr(logging, self.string) + except AttributeError: raise UnknownLoglevelException(level_const) - self.const = level_const - self.string = level_dict["string"] - self.number = int(level_dict["number"]) - if "color" in level_dict.keys(): - self.color = getattr(Colors, level_dict["color"]) + if self.number == 30: + self.color = Colors.yellow + elif self.number > 30: + self.color = Colors.red else: - self.color = None + self.color = "" def __str__(self) -> str: return self.string @@ -27,22 +30,20 @@ def __str__(self) -> str: def __int__(self) -> int: return self.number - def get_colored_string(self, string: str) -> str: - """ Get colored text according to LogLevel """ - if self.color: - out_string = self.color + string + Colors.reset - else: - out_string = string - return out_string - - class LogLevelObject: """ Base class for loglevel properties """ - LOG_MESSAGE = LogLevel("LOG_MESSAGE") - LOG_ERROR = LogLevel("LOG_ERROR") - LOG_WARNING = LogLevel("LOG_WARNING") - LOG_INFO = LogLevel("LOG_INFO") - LOG_DEBUG = LogLevel("LOG_DEBUG") - LOG_DEVELOPMENT = LogLevel("LOG_DEVELOPMENT") + LOG_DEBUG = LogLevel("DEBUG") + LOG_INFO = LogLevel("INFO") + LOG_WARNING = LogLevel("WARNING") + LOG_ERROR = LogLevel("ERROR") + LOG_CRITICAL = LogLevel("CRITICAL") + + LOG_FILE_ONLY = "file" + LOG_CONSOLE_ONLY = "console" + LOG_BOTH = LOG_FILE_ONLY + " " + LOG_CONSOLE_ONLY + + @classmethod + def GetLoglevels(cls) -> list[LogLevel]: + return [getattr(cls, l) for l in dir(cls) if isinstance(getattr(cls, l), LogLevel)] diff --git a/IoTuring/Logger/LogObject.py b/IoTuring/Logger/LogObject.py index bc452dc64..09536f8ef 100644 --- a/IoTuring/Logger/LogObject.py +++ b/IoTuring/Logger/LogObject.py @@ -3,11 +3,13 @@ class LogObject(LogLevelObject): - def Log(self, loglevel: LogLevel, message): - Logger().Log( + def Log(self, loglevel: LogLevel, message, **kwargs): + logger = Logger() + logger.Log( source=self.LogSource(), message=message, - loglevel=loglevel + loglevel=loglevel, + **kwargs ) # to override in classes where I want a source different from class name diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 219ebc579..212e1ddab 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -1,15 +1,14 @@ -import sys -import os -import inspect -from datetime import datetime -import json -import threading -from io import TextIOWrapper import logging +import logging.handlers +from pathlib import Path +from IoTuring.Logger.Colors import Colors from IoTuring.Logger import consts from IoTuring.Logger.LogLevel import LogLevelObject, LogLevel from IoTuring.Exceptions.Exceptions import UnknownLoglevelException +from IoTuring.MyApp.SystemConsts import TerminalDetection +from IoTuring.MyApp.SystemConsts import OperatingSystemDetection as OsD +from IoTuring.MyApp.App import App class Singleton(type): @@ -25,184 +24,179 @@ def __call__(cls, *args, **kwargs): return cls._instances[cls] +class LogTargetFilter(logging.Filter): + def __init__(self, target: str) -> None: + self.target = target + + def filter(self, record): + if self.target in record.getMessage(): + return True + else: + return False + + +class LogLevelFilter(logging.Filter): + def __init__(self, loglevel: LogLevel) -> None: + self.loglevel = loglevel + + def filter(self, record): + if int(self.loglevel) > int(record.levelno): + return False + else: + return True + + class Logger(LogLevelObject, metaclass=Singleton): - # lock = threading.Lock() + prefix_length = 70 + + console_formatter = logging.Formatter( + fmt="{color_prefix}[ {asctime:s} | {levelname:^10s} | {source:^30s} | {console_message:s}{color_suffix}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{", + defaults={"color_prefix": "", + "color_suffix": "" + }) - # log_filename = "" - # log_file_descriptor = None + file_formatter = logging.Formatter( + fmt="[ {asctime:s} | {levelname:^10s} | {source:^30s} | {file_message:s}", + datefmt="%Y-%m-%d %H:%M:%S", + style="{" + ) - # # Default log levels: - # console_log_level = LogLevel(consts.CONSOLE_LOG_LEVEL) - # file_log_level = LogLevel(consts.FILE_LOG_LEVEL) + final_settings = False + log_dir_path = "" + file_handler = None def __init__(self) -> None: - self.logger = logging.getLogger() - self.logger.setLevel(logging.NOTSET) - - # formatter = logging.Formatter('[ %(asctime)s - %(name)s - %(levelname)s - %(message)s') - formatter = logging.Formatter( - fmt='[ {asctime:s} | {levelname:^10s} | {name:^30s} | {message:s}{multilines:s}', - datefmt='%Y-%m-%d %H:%M:%S', - style='{' - ) - - ch = logging.StreamHandler() - ch.setLevel(logging.DEBUG) - ch.setFormatter(formatter) - self.logger.addHandler(ch) - - # self.terminalSupportsColors = Logger.checkTerminalSupportsColors() - - # # Prepare the log - # self.SetLogFilename() - # # Open the file descriptor - # self.GetLogFileDescriptor() - - # # Override log level from envvar: - # try: - # if os.getenv("IOTURING_LOG_LEVEL"): - # level_override = LogLevel(str(os.getenv("IOTURING_LOG_LEVEL"))) - # self.console_log_level = level_override - # except UnknownLoglevelException as e: - # self.Log(self.LOG_ERROR, "Logger", - # f"Unknown Loglevel: {e.loglevel}") - - # def SetLogFilename(self) -> str: - # """ Set filename with timestamp and also call setup folder """ - # dateTimeObj = datetime.now() - # self.log_filename = os.path.join( - # self.SetupFolder(), dateTimeObj.strftime(consts.LOG_FILENAME_FORMAT).replace(":", "_")) - # return self.log_filename - - # def SetupFolder(self) -> str: - # """ Check if exists (or create) the folder of logs inside this file's folder """ - # thisFolder = os.path.dirname(inspect.getfile(Logger)) - # newFolder = os.path.join(thisFolder, consts.LOGS_FOLDER) - # if not os.path.exists(newFolder): - # os.mkdir(newFolder) - - # return newFolder - - # def GetMessageDatetimeString(self) -> str: - # now = datetime.now() - # return now.strftime(consts.MESSAGE_DATETIME_FORMAT) - - # LOG - - def Log(self, loglevel: LogLevel, source: str, message, printToConsole=True, writeToFile=True) -> None: - - if isinstance(message, str) and len(message) > 10: - lines = [message[i:i+10] for i in range(0, len(message), 10)] - ml_string = "\n ".join(lines) - extra = {"multilines": ml_string} - message = "" + # Start root logger: + self.logger = logging.getLogger(__name__) + self.logger.setLevel(10) + + # Init console logger handler: + self.console_handler = logging.StreamHandler() + self.SetConsoleLogLevel() + self.console_handler.setFormatter(self.console_formatter) + self.console_handler.addFilter(LogTargetFilter(self.LOG_CONSOLE_ONLY)) + self.logger.addHandler(self.console_handler) + + # Init file logger buffer handler: + self.memory_handler = logging.handlers.MemoryHandler(capacity=100) + self.logger.addHandler(self.memory_handler) + + def SetConsoleLogLevel(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_LEVEL)) -> None: + if OsD.GetEnv("IOTURING_LOG_LEVEL"): + try: + env_level = LogLevel(OsD.GetEnv("IOTURING_LOG_LEVEL")) + self.console_handler.setLevel(int(env_level)) + return + except UnknownLoglevelException: + pass + self.console_handler.setLevel(int(loglevel)) + + def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path) -> None: + + if enabled: + self.StartFileLogging(loglevel, log_dir_path) + + elif self.final_settings: + self.DisableFileLogging() + + self.final_settings = True + + def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: + + self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {log_dir_path.absolute()}", + logtarget=self.LOG_CONSOLE_ONLY) + + if self.file_handler: + if log_dir_path.samefile(self.log_dir_path): + self.file_handler.setLevel(int(loglevel)) + return + else: + self.logger.removeHandler(self.file_handler) + + filepath = log_dir_path.joinpath(App.getName() + ".log") + self.log_dir_path = log_dir_path + + self.file_handler = logging.handlers.RotatingFileHandler( + filepath, backupCount=5) + + if filepath.exists(): + self.file_handler.doRollover() + + self.file_handler.setFormatter(self.file_formatter) + self.file_handler.addFilter(LogLevelFilter(loglevel)) + self.file_handler.addFilter(LogTargetFilter(self.LOG_FILE_ONLY)) + self.file_handler.setLevel(int(loglevel)) + + self.logger.addHandler(self.file_handler) + + self.memory_handler.setTarget(self.file_handler) + self.memory_handler.close() + self.logger.removeHandler(self.memory_handler) + + def DisableFileLogging(self) -> None: + + if self.file_handler: + self.logger.removeHandler(self.file_handler) + self.file_handler.close() + + if self.memory_handler: + self.logger.removeHandler(self.memory_handler) + self.memory_handler.close() + + def GetConsoleMessage(self, message, line_length) -> str: + + if isinstance(message, str) and len(message.splitlines()) == 1 and len(message) < line_length: + return message.strip() + + final_lines = [] + + for l in self.GetMessageAsList(message): + short_lines = [l[i:i+line_length] + for i in range(0, len(l), line_length)] + + final_lines.extend(short_lines) + + line_prefix = "\n" + " " * self.prefix_length + return line_prefix.join(final_lines) + + def GetFileMessage(self, message) -> str: + return " ".join(self.GetMessageAsList(message)) + + def GetMessageAsList(self, message) -> list[str]: + if isinstance(message, list): + messagelines = [str(i) for i in message] + elif isinstance(message, dict): + messagelines = [f"{k}: {v}" for k, v in message.items()] else: - extra = {"multilines": ""} - - - - l = logging.getLogger(source) - l.log(int(loglevel), message, extra=extra) - - # if type(message) == dict: - # self.LogDict(loglevel, source, message, - # printToConsole, writeToFile) - # return # Log dict will call this function so I don't need to go down at the moment - # elif type(message) == list: - # self.LogList(loglevel, source, message, - # printToConsole, writeToFile) - # return # Log list will call this function so I don't need to go down at the moment - - # message = str(message) - # # Call this function for each line of the message if there are more than one line. - # messageLines = message.split("\n") - # if len(messageLines) > 1: - # for line in messageLines: - # self.Log(loglevel, source, line, - # printToConsole, writeToFile) - # return # Stop the function then because I've already called this function from each line so I don't have to go down here - - # prestring = f"[ {self.GetMessageDatetimeString()} | {str(loglevel).center(consts.STRINGS_LENGTH[0])} | " \ - # + f"{source.center(consts.STRINGS_LENGTH[1])}]{consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' '}" # justify - - # # Manage string to print in more lines if it's too long - # while len(message) > 0: - # string = prestring+message[:consts.MESSAGE_WIDTH] - # # Cut for next iteration if message is longer than a line - # message = message[consts.MESSAGE_WIDTH:] - # # then I add the dash to the row - # if (len(message) > 0 and string[-1] != " " and string[-1] != "." and string[-1] != ","): - # string = string + '-' # Print new line indicator if I will go down in the next iteration - # self.PrintAndSave(string, loglevel, printToConsole, writeToFile) - # # -1 + space cause if the char in the prestring isn't a space, it will be directly attached to my message without a space - - # prestring = (len(prestring)-consts.PRESTRING_MESSAGE_SEPARATOR_LEN) * \ - # consts.LONG_MESSAGE_PRESTRING_CHAR+consts.PRESTRING_MESSAGE_SEPARATOR_LEN*' ' - - # def LogDict(self, loglevel, source, message_dict: dict, *args): - # try: - # string = json.dumps(message_dict, indent=4, sort_keys=False, - # default=lambda o: '') - # lines = string.splitlines() - # for line in lines: - # self.Log(loglevel, source, "> "+line, *args) - # except Exception as e: - # self.Log(self.LOG_ERROR, source, "Can't print dictionary content") - - # def LogList(self, loglevel, source, message_list: list, *args): - # try: - # for index, item in enumerate(message_list): - # if type(item) == dict or type(item) == list: - # self.Log(loglevel, source, "Item #"+str(index), *args) - # self.Log(loglevel, source, item, *args) - # else: - # self.Log(loglevel, source, - # f"{str(index)}: {str(item)}", *args) - - # except: - # self.Log(self.LOG_ERROR, source, "Can't print dictionary content") - - # # Both print and save to file - # def PrintAndSave(self, string, loglevel: LogLevel, printToConsole=True, writeToFile=True) -> None: - - # if printToConsole and int(loglevel) <= int(self.console_log_level): - # self.ColoredPrint(string, loglevel) - - # if writeToFile and int(loglevel) <= int(self.file_log_level): - # # acquire the lock - # with self.lock: - # self.GetLogFileDescriptor().write(string+' \n') - # # so I can see the log in real time from a reader - # self.GetLogFileDescriptor().flush() - - # def ColoredPrint(self, string, loglevel: LogLevel) -> None: - # if not self.terminalSupportsColors: - # print(string) - # else: - # print(loglevel.get_colored_string(string)) - - # def GetLogFileDescriptor(self) -> TextIOWrapper: - # if self.log_file_descriptor is None: - # self.log_file_descriptor = open(self.log_filename, "a", encoding="utf-8") - - # return self.log_file_descriptor - - # def CloseFile(self) -> None: - # if self.log_file_descriptor is not None: - # self.log_file_descriptor.close() - # self.log_file_descriptor = None - - @staticmethod - def checkTerminalSupportsColors(): - """ - Returns True if the running system's terminal supports color, and False - otherwise. - """ - plat = sys.platform - supported_platform = plat != 'Pocket PC' and (plat != 'win32' or - 'ANSICON' in os.environ) - # isatty is not always implemented, #6223. - is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() - return supported_platform and is_a_tty + messagelines = [str(message)] + + lines = [] + + # replace and split by newlines + for m in messagelines: + lines.extend(m.splitlines()) + + return lines + + def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = LogLevelObject.LOG_BOTH) -> None: + + available_length = TerminalDetection.GetTerminalColumns() - self.prefix_length + + extra = {"source": source, + "file_message": self.GetFileMessage(message), + "console_message": self.GetConsoleMessage(message, available_length) + } + + if TerminalDetection.CheckTerminalSupportsColors(): + + if color or loglevel.color: + extra["color_prefix"] = color or loglevel.color + extra["color_suffix"] = Colors.reset + + l = logging.getLogger(__name__).getChild(source) + + l.log(int(loglevel), msg=logtarget, extra=extra) diff --git a/IoTuring/Logger/consts.py b/IoTuring/Logger/consts.py index 23b03e655..92f147efc 100644 --- a/IoTuring/Logger/consts.py +++ b/IoTuring/Logger/consts.py @@ -1,62 +1,7 @@ LOGS_FOLDER = "Logs" -LOG_FILENAME_FORMAT = "Log_%Y-%m-%d_%H:%M:%S.log" -MESSAGE_DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S' - - -LOG_LEVELS = [ - { - "const": "LOG_MESSAGE", - "string": "Message", - "number": 50, - "color": "green" - }, - { - "const": "LOG_ERROR", - "string": "Error", - "number": 40, - "color": "red" - }, - { - "const": "LOG_WARNING", - "string": "Warning", - "number": 30, - "color": "yellow" - }, - { - "const": "LOG_INFO", - "string": "Info", - "number": 20, - - }, - { - "const": "LOG_DEBUG", - "string": "Debug", - "number": 10, - - }, - { - "const": "LOG_DEVELOPMENT", - "string": "Dev", - "number": 0, - - } -] +DEFAULT_LOG_LEVEL = "INFO" # On/off states as strings: STATE_ON = "ON" STATE_OFF = "OFF" - -# Fill start of string with spaces to jusitfy the message (0: no padding) -# First for type, second for source -STRINGS_LENGTH = [8, 30] - -# number of spaces to separe the message from the previuos part of the row -PRESTRING_MESSAGE_SEPARATOR_LEN = 2 -# before those spaces I add this string -LONG_MESSAGE_PRESTRING_CHAR = ' ' - -CONSOLE_LOG_LEVEL = "LOG_INFO" -FILE_LOG_LEVEL = "LOG_INFO" - -MESSAGE_WIDTH = 95 diff --git a/IoTuring/MyApp/SystemConsts/TerminalDetection.py b/IoTuring/MyApp/SystemConsts/TerminalDetection.py new file mode 100644 index 000000000..8f0f73d0c --- /dev/null +++ b/IoTuring/MyApp/SystemConsts/TerminalDetection.py @@ -0,0 +1,44 @@ +import sys +import os +import shutil + + +class TerminalDetection: + @staticmethod + def CheckTerminalSupportsColors() -> bool: + """ + Returns True if the running system's terminal supports color, and False + otherwise. + """ + plat = sys.platform + supported_platform = plat != 'Pocket PC' and (plat != 'win32' or + 'ANSICON' in os.environ) + # isatty is not always implemented, #6223. + is_a_tty = hasattr(sys.stdout, 'isatty') and sys.stdout.isatty() + return supported_platform and is_a_tty + + @staticmethod + def CheckTerminalSupportsSize() -> bool: + return any(shutil.get_terminal_size(fallback=(0,0))) + + + @staticmethod + def GetTerminalLines() -> int: + return shutil.get_terminal_size().lines + + @staticmethod + def GetTerminalColumns() -> int: + return shutil.get_terminal_size().columns + + + @staticmethod + def CalculateNumberOfLines(string_length: int) -> int: + """Get the number of lines required to display a text with the given length + + Args: + string_length (int): Length of the text + + Returns: + int: Number of lines required + """ + return int((string_length / shutil.get_terminal_size().columns) // 1) diff --git a/IoTuring/MyApp/SystemConsts/__init__.py b/IoTuring/MyApp/SystemConsts/__init__.py index dc709cddb..d30843129 100644 --- a/IoTuring/MyApp/SystemConsts/__init__.py +++ b/IoTuring/MyApp/SystemConsts/__init__.py @@ -1,2 +1,3 @@ from .DesktopEnvironmentDetection import DesktopEnvironmentDetection -from .OperatingSystemDetection import OperatingSystemDetection \ No newline at end of file +from .OperatingSystemDetection import OperatingSystemDetection +from .TerminalDetection import TerminalDetection \ No newline at end of file diff --git a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py new file mode 100644 index 000000000..669c5217f --- /dev/null +++ b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py @@ -0,0 +1,101 @@ +from pathlib import Path + +from IoTuring.Configurator.MenuPreset import MenuPreset +from IoTuring.Settings.Settings import Settings +from IoTuring.Logger.Logger import Logger +from IoTuring.Configurator.Configuration import SingleConfiguration +from IoTuring.MyApp.App import App +from IoTuring.MyApp.SystemConsts import OperatingSystemDetection as OsD +from IoTuring.Logger.LogLevel import LogLevel +from IoTuring.Logger import consts + + +# macOS dep (in PyObjC) +try: + from AppKit import * # type:ignore + from Foundation import * # type:ignore + macos_support = True +except: + macos_support = False + + +CONFIG_KEY_CONSOLE_LOG_LEVEL = "console_log_level" +CONFIG_KEY_FILE_LOG_LEVEL = "file_log_level" +CONFIG_KEY_FILE_LOG_ENABLED = "file_log_enabled" +CONFIG_KEY_FILE_LOG_PATH = "file_log_path" + + +all_loglevels = sorted(Logger.GetLoglevels(), key=lambda l: int(l)) + +loglevel_choices = [{"name": str(l).capitalize(), "value": str(l)} + for l in all_loglevels] + + +class LogSettings(Settings): + NAME = "Log" + + def __init__(self, single_configuration: SingleConfiguration) -> None: + super().__init__(single_configuration) + + # Load settings to logger: + logger = Logger() + + console_level = LogLevel( + self.GetFromConfigurations(CONFIG_KEY_CONSOLE_LOG_LEVEL)) + logger.SetConsoleLogLevel(console_level) + + logger.SetupFileLogging( + enabled=self.GetTrueOrFalseFromConfigurations( + CONFIG_KEY_FILE_LOG_ENABLED), + loglevel=LogLevel(self.GetFromConfigurations( + CONFIG_KEY_FILE_LOG_LEVEL)), + log_dir_path=Path(self.GetFromConfigurations( + CONFIG_KEY_FILE_LOG_PATH)) + ) + + @classmethod + def ConfigurationPreset(cls): + preset = MenuPreset() + + preset.AddEntry(name="Console log level", key=CONFIG_KEY_CONSOLE_LOG_LEVEL, + question_type="select", mandatory=True, default=str(LogLevel(consts.DEFAULT_LOG_LEVEL)), + instruction="IOTURING_LOG_LEVEL envvar overwrites this setting!", + choices=loglevel_choices) + + preset.AddEntry(name="Enable file logging", key=CONFIG_KEY_FILE_LOG_ENABLED, + question_type="yesno", default="N") + + preset.AddEntry(name="File log level", key=CONFIG_KEY_FILE_LOG_LEVEL, + question_type="select", mandatory=True, default=str(LogLevel(consts.DEFAULT_LOG_LEVEL)), + choices=loglevel_choices) + + preset.AddEntry(name="File log path", key=CONFIG_KEY_FILE_LOG_PATH, + question_type="filepath", mandatory=True, default=cls.GetDefaultLogPath(), + instruction="Directory where log files will be saved") + + return preset + + @staticmethod + def GetDefaultLogPath() -> str: + + default_path = App.getRootPath().joinpath( + "Logger").joinpath(consts.LOGS_FOLDER) + base_path = None + + if OsD.IsMacos() and macos_support: + base_path = \ + Path(NSSearchPathForDirectoriesInDomains( # type: ignore + NSLibraryDirectory, # type: ignore + NSUserDomainMask, True)[0]) # type: ignore + elif OsD.IsWindows(): + base_path = Path(OsD.GetEnv("LOCALAPPDATA")) + elif OsD.IsLinux(): + if OsD.GetEnv("XDG_CACHE_HOME"): + base_path = Path(OsD.GetEnv("XDG_CACHE_HOME")) + elif OsD.GetEnv("HOME"): + base_path = Path(OsD.GetEnv("HOME")).joinpath(".cache") + + if base_path: + default_path = base_path.joinpath(App.getName()) + + return str(default_path) diff --git a/IoTuring/Warehouse/Deployments/ConsoleWarehouse/ConsoleWarehouse.py b/IoTuring/Warehouse/Deployments/ConsoleWarehouse/ConsoleWarehouse.py index 1299bd3be..375a231d4 100644 --- a/IoTuring/Warehouse/Deployments/ConsoleWarehouse/ConsoleWarehouse.py +++ b/IoTuring/Warehouse/Deployments/ConsoleWarehouse/ConsoleWarehouse.py @@ -10,7 +10,7 @@ def Loop(self): for entity in self.GetEntities(): for entitySensor in entity.GetEntitySensors(): if(entitySensor.HasValue()): - self.Log(Logger.LOG_MESSAGE, entitySensor.GetId() + + self.Log(self.LOG_INFO, entitySensor.GetId() + ": " + self.FormatValue(entitySensor)) def FormatValue(self, entitySensor: EntitySensor): diff --git a/IoTuring/__init__.py b/IoTuring/__init__.py index 38e5768af..1cb10196f 100644 --- a/IoTuring/__init__.py +++ b/IoTuring/__init__.py @@ -47,12 +47,15 @@ def loop(): sys.exit("Error: Invalid arguments!") # Clear the terminal - # Configurator.ClearTerminal() + Configurator.ClearTerminal() # Start logger: logger = Logger() configurator = Configurator() + # Early log settings: + ConfiguratorLoader(configurator).LoadSettings() + logger.Log(Logger.LOG_DEBUG, "App", f"Selected options: {vars(args)}") if args.configurator: @@ -115,20 +118,13 @@ def loop(): def Exit_SIGINT_handler(sig=None, frame=None): logger = Logger() logger.Log(Logger.LOG_INFO, "Main", "Application closed by SigInt", - printToConsole=False) # to file + logtarget=Logger.LOG_FILE_ONLY) # to file messages = ["Exiting...", "Thanks for using IoTuring !"] + print() # New line - for message in messages: - text = "" - if (Logger.checkTerminalSupportsColors()): - text += Colors.cyan - text += message - if (Logger.checkTerminalSupportsColors()): - text += Colors.reset - logger.Log(Logger.LOG_INFO, "Main", text, - writeToFile=False) # to terminal - - # logger.CloseFile() + logger.Log(Logger.LOG_INFO, "Main", messages, + color=Colors.cyan, logtarget=Logger.LOG_CONSOLE_ONLY) + sys.exit(0) From 4fb2bd80a580181b4db12ca7e87d1ff2dfcee548 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 4 May 2024 12:52:27 +0200 Subject: [PATCH 05/12] fix --- IoTuring/Logger/LogLevel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IoTuring/Logger/LogLevel.py b/IoTuring/Logger/LogLevel.py index 4b4285692..d780b5296 100644 --- a/IoTuring/Logger/LogLevel.py +++ b/IoTuring/Logger/LogLevel.py @@ -1,3 +1,4 @@ +from __future__ import annotations import logging from IoTuring.Logger.Colors import Colors from IoTuring.Exceptions.Exceptions import UnknownLoglevelException From fbdf2903239095a55d88330b2a05033d61f87f62 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 4 May 2024 12:53:06 +0200 Subject: [PATCH 06/12] fix --- IoTuring/Logger/Logger.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 212e1ddab..2c2c50b33 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -1,3 +1,5 @@ +from __future__ import annotations + import logging import logging.handlers from pathlib import Path From a389343d85233ba5415ba28241c12f4af9161bb3 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Sat, 4 May 2024 12:57:02 +0200 Subject: [PATCH 07/12] fix --- IoTuring/Logger/Logger.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 2c2c50b33..41f2eecc2 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -55,10 +55,8 @@ class Logger(LogLevelObject, metaclass=Singleton): console_formatter = logging.Formatter( fmt="{color_prefix}[ {asctime:s} | {levelname:^10s} | {source:^30s} | {console_message:s}{color_suffix}", datefmt="%Y-%m-%d %H:%M:%S", - style="{", - defaults={"color_prefix": "", - "color_suffix": "" - }) + style="{" + ) file_formatter = logging.Formatter( fmt="[ {asctime:s} | {levelname:^10s} | {source:^30s} | {file_message:s}", @@ -190,7 +188,9 @@ def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarg extra = {"source": source, "file_message": self.GetFileMessage(message), - "console_message": self.GetConsoleMessage(message, available_length) + "console_message": self.GetConsoleMessage(message, available_length), + "color_prefix": "", + "color_suffix": "" } if TerminalDetection.CheckTerminalSupportsColors(): From 84b4f59b0f9f8238a2f684592c4fee202d5eab45 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 10 May 2024 06:26:04 +0200 Subject: [PATCH 08/12] Nicer console logging --- IoTuring/Logger/LogLevel.py | 5 ++- IoTuring/Logger/Logger.py | 66 ++++++++++++++++++++++++------------- IoTuring/Logger/consts.py | 20 +++++++++++ IoTuring/__init__.py | 4 +-- 4 files changed, 68 insertions(+), 27 deletions(-) diff --git a/IoTuring/Logger/LogLevel.py b/IoTuring/Logger/LogLevel.py index d780b5296..80c8123cd 100644 --- a/IoTuring/Logger/LogLevel.py +++ b/IoTuring/Logger/LogLevel.py @@ -41,9 +41,8 @@ class LogLevelObject: LOG_ERROR = LogLevel("ERROR") LOG_CRITICAL = LogLevel("CRITICAL") - LOG_FILE_ONLY = "file" - LOG_CONSOLE_ONLY = "console" - LOG_BOTH = LOG_FILE_ONLY + " " + LOG_CONSOLE_ONLY + LOGTARGET_FILE = "file" + LOGTARGET_CONSOLE = "console" @classmethod def GetLoglevels(cls) -> list[LogLevel]: diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 41f2eecc2..eee840d0c 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -27,21 +27,25 @@ def __call__(cls, *args, **kwargs): class LogTargetFilter(logging.Filter): + """ Log filter for log target (console or file) """ + def __init__(self, target: str) -> None: self.target = target - def filter(self, record): - if self.target in record.getMessage(): + def filter(self, record) -> bool: + if not getattr(record, "logtarget") or self.target in getattr(record, "logtarget"): return True else: return False class LogLevelFilter(logging.Filter): + """ Log filter for loglevel, for later file logging""" + def __init__(self, loglevel: LogLevel) -> None: self.loglevel = loglevel - def filter(self, record): + def filter(self, record) -> bool: if int(self.loglevel) > int(record.levelno): return False else: @@ -50,16 +54,15 @@ def filter(self, record): class Logger(LogLevelObject, metaclass=Singleton): - prefix_length = 70 - console_formatter = logging.Formatter( - fmt="{color_prefix}[ {asctime:s} | {levelname:^10s} | {source:^30s} | {console_message:s}{color_suffix}", + fmt="{color_prefix}" + consts.LOG_PREFIX_STRING + + "{console_message}{color_suffix}", datefmt="%Y-%m-%d %H:%M:%S", style="{" ) file_formatter = logging.Formatter( - fmt="[ {asctime:s} | {levelname:^10s} | {source:^30s} | {file_message:s}", + fmt=consts.LOG_PREFIX_STRING + "{file_message}", datefmt="%Y-%m-%d %H:%M:%S", style="{" ) @@ -78,7 +81,7 @@ def __init__(self) -> None: self.console_handler = logging.StreamHandler() self.SetConsoleLogLevel() self.console_handler.setFormatter(self.console_formatter) - self.console_handler.addFilter(LogTargetFilter(self.LOG_CONSOLE_ONLY)) + self.console_handler.addFilter(LogTargetFilter(self.LOGTARGET_CONSOLE)) self.logger.addHandler(self.console_handler) # Init file logger buffer handler: @@ -108,7 +111,7 @@ def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {log_dir_path.absolute()}", - logtarget=self.LOG_CONSOLE_ONLY) + logtarget=self.LOGTARGET_CONSOLE) if self.file_handler: if log_dir_path.samefile(self.log_dir_path): @@ -128,7 +131,7 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.file_handler.setFormatter(self.file_formatter) self.file_handler.addFilter(LogLevelFilter(loglevel)) - self.file_handler.addFilter(LogTargetFilter(self.LOG_FILE_ONLY)) + self.file_handler.addFilter(LogTargetFilter(self.LOGTARGET_FILE)) self.file_handler.setLevel(int(loglevel)) self.logger.addHandler(self.file_handler) @@ -147,20 +150,30 @@ def DisableFileLogging(self) -> None: self.logger.removeHandler(self.memory_handler) self.memory_handler.close() - def GetConsoleMessage(self, message, line_length) -> str: + def GetConsoleMessage(self, source: str, message) -> str: - if isinstance(message, str) and len(message.splitlines()) == 1 and len(message) < line_length: + if isinstance(message, str) \ + and len(message.splitlines()) == 1 \ + and len(message) + self.GetPrefixLength(source) < TerminalDetection.GetTerminalColumns(): return message.strip() + line_length = TerminalDetection.GetTerminalColumns() - \ + self.GetPrefixLength() + final_lines = [] + messagelines = self.GetMessageAsList(message) - for l in self.GetMessageAsList(message): - short_lines = [l[i:i+line_length] - for i in range(0, len(l), line_length)] + if self.GetPrefixLength(source) > self.GetPrefixLength(): + first_line_len = TerminalDetection.GetTerminalColumns() - \ + self.GetPrefixLength(source) + final_lines.append(messagelines[0][:first_line_len]) + messagelines[0] = messagelines[0][first_line_len:] - final_lines.extend(short_lines) + for l in messagelines: + final_lines.extend([l[i:i+line_length] + for i in range(0, len(l), line_length)]) - line_prefix = "\n" + " " * self.prefix_length + line_prefix = "\n" + " " * self.GetPrefixLength() return line_prefix.join(final_lines) def GetFileMessage(self, message) -> str: @@ -182,17 +195,26 @@ def GetMessageAsList(self, message) -> list[str]: return lines - def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = LogLevelObject.LOG_BOTH) -> None: + def GetPrefixLength(self, source: str = "") -> int: + default_source_len = next( + (f["len"] for f in consts.LOG_PREFIX_PARTS if f["attr"].startswith("source"))) + extra_len = max(len(source) - default_source_len, 0) + return len(consts.LOG_PREFIX_FORMAT) + sum([f["len"] - 2 for f in consts.LOG_PREFIX_PARTS]) + extra_len - available_length = TerminalDetection.GetTerminalColumns() - self.prefix_length + def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = "") -> None: extra = {"source": source, "file_message": self.GetFileMessage(message), - "console_message": self.GetConsoleMessage(message, available_length), "color_prefix": "", - "color_suffix": "" + "color_suffix": "", + "logtarget": logtarget } + if TerminalDetection.CheckTerminalSupportsSize() and TerminalDetection.GetTerminalColumns() > consts.MIN_CONSOLE_WIDTH: + extra["console_message"] = self.GetConsoleMessage(source, message) + else: + extra["console_message"] = extra["file_message"] + if TerminalDetection.CheckTerminalSupportsColors(): if color or loglevel.color: @@ -201,4 +223,4 @@ def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarg l = logging.getLogger(__name__).getChild(source) - l.log(int(loglevel), msg=logtarget, extra=extra) + l.log(int(loglevel), msg=message, extra=extra) diff --git a/IoTuring/Logger/consts.py b/IoTuring/Logger/consts.py index 92f147efc..fd717d04d 100644 --- a/IoTuring/Logger/consts.py +++ b/IoTuring/Logger/consts.py @@ -1,5 +1,25 @@ LOGS_FOLDER = "Logs" DEFAULT_LOG_LEVEL = "INFO" +MIN_CONSOLE_WIDTH = 95 + +LOG_PREFIX_PARTS = [ + { + "attr": "asctime", + "len": 19 + }, + { + "attr": "levelname:^8s", + "len": 8 + }, + { + "attr": "source:^30s", + "len": 30 + } +] + +LOG_PREFIX_FORMAT = "[ {} | {} | {} ] " +LOG_PREFIX_STRING = LOG_PREFIX_FORMAT.format( + *["{" + f["attr"] + "}" for f in LOG_PREFIX_PARTS]) # On/off states as strings: diff --git a/IoTuring/__init__.py b/IoTuring/__init__.py index 1cb10196f..f53e1eb46 100644 --- a/IoTuring/__init__.py +++ b/IoTuring/__init__.py @@ -118,13 +118,13 @@ def loop(): def Exit_SIGINT_handler(sig=None, frame=None): logger = Logger() logger.Log(Logger.LOG_INFO, "Main", "Application closed by SigInt", - logtarget=Logger.LOG_FILE_ONLY) # to file + logtarget=Logger.LOGTARGET_FILE) # to file messages = ["Exiting...", "Thanks for using IoTuring !"] print() # New line logger.Log(Logger.LOG_INFO, "Main", messages, - color=Colors.cyan, logtarget=Logger.LOG_CONSOLE_ONLY) + color=Colors.cyan, logtarget=Logger.LOGTARGET_CONSOLE) sys.exit(0) From 3cf66d00d61d795848a087f4ba674f5f75a94b90 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 10 May 2024 08:02:27 +0200 Subject: [PATCH 09/12] Setting to hide time in console log --- IoTuring/Logger/Logger.py | 59 +++++++++++++------ IoTuring/Logger/consts.py | 26 ++++---- .../Deployments/LogSettings/LogSettings.py | 13 +++- 3 files changed, 60 insertions(+), 38 deletions(-) diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index eee840d0c..f4d6c0ee8 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -54,22 +54,10 @@ def filter(self, record) -> bool: class Logger(LogLevelObject, metaclass=Singleton): - console_formatter = logging.Formatter( - fmt="{color_prefix}" + consts.LOG_PREFIX_STRING + - "{console_message}{color_suffix}", - datefmt="%Y-%m-%d %H:%M:%S", - style="{" - ) - - file_formatter = logging.Formatter( - fmt=consts.LOG_PREFIX_STRING + "{file_message}", - datefmt="%Y-%m-%d %H:%M:%S", - style="{" - ) - final_settings = False log_dir_path = "" file_handler = None + console_prefix_length = 0 def __init__(self) -> None: @@ -79,8 +67,7 @@ def __init__(self) -> None: # Init console logger handler: self.console_handler = logging.StreamHandler() - self.SetConsoleLogLevel() - self.console_handler.setFormatter(self.console_formatter) + self.SetupConsoleLogging() self.console_handler.addFilter(LogTargetFilter(self.LOGTARGET_CONSOLE)) self.logger.addHandler(self.console_handler) @@ -88,7 +75,10 @@ def __init__(self) -> None: self.memory_handler = logging.handlers.MemoryHandler(capacity=100) self.logger.addHandler(self.memory_handler) - def SetConsoleLogLevel(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_LEVEL)) -> None: + def SetupConsoleLogging(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_LEVEL), include_time: bool = True) -> None: + self.console_handler.setFormatter( + self.GetFormatter(self.LOGTARGET_CONSOLE, include_time)) + if OsD.GetEnv("IOTURING_LOG_LEVEL"): try: env_level = LogLevel(OsD.GetEnv("IOTURING_LOG_LEVEL")) @@ -129,7 +119,7 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: if filepath.exists(): self.file_handler.doRollover() - self.file_handler.setFormatter(self.file_formatter) + self.file_handler.setFormatter(self.GetFormatter(self.LOGTARGET_FILE)) self.file_handler.addFilter(LogLevelFilter(loglevel)) self.file_handler.addFilter(LogTargetFilter(self.LOGTARGET_FILE)) self.file_handler.setLevel(int(loglevel)) @@ -150,6 +140,37 @@ def DisableFileLogging(self) -> None: self.logger.removeHandler(self.memory_handler) self.memory_handler.close() + def GetFormatter(self, logtarget: str, include_time: bool = True) -> logging.Formatter: + + prefix_lengths = consts.LOG_PREFIX_LENGTHS.copy() + + if not include_time: + prefix_lengths.pop("asctime") + + prefix_strings = [f"{{{s}}}" for s in prefix_lengths] + prefix_string = consts.LOG_PREFIX_ENDS[0] +\ + consts.LOG_PREFIX_SEPARATOR.join(prefix_strings) +\ + consts.LOG_PREFIX_ENDS[1] + + prefix_length = sum([len(s) for s in consts.LOG_PREFIX_ENDS]) + \ + len(consts.LOG_PREFIX_SEPARATOR) * (len(prefix_lengths) - 1) + \ + sum([l for l in prefix_lengths.values()]) + + if logtarget == self.LOGTARGET_CONSOLE: + fmt = "{color_prefix}" + prefix_string + \ + "{console_message}{color_suffix}" + self.console_prefix_length = prefix_length + elif logtarget == self.LOGTARGET_FILE: + fmt = prefix_string + "{file_message}" + else: + raise Exception(f"Unknown logtarget: {logtarget}") + + return logging.Formatter( + fmt=fmt, + datefmt="%Y-%m-%d %H:%M:%S", + style="{" + ) + def GetConsoleMessage(self, source: str, message) -> str: if isinstance(message, str) \ @@ -197,9 +218,9 @@ def GetMessageAsList(self, message) -> list[str]: def GetPrefixLength(self, source: str = "") -> int: default_source_len = next( - (f["len"] for f in consts.LOG_PREFIX_PARTS if f["attr"].startswith("source"))) + (l for s, l in consts.LOG_PREFIX_LENGTHS.items() if s.startswith("source"))) extra_len = max(len(source) - default_source_len, 0) - return len(consts.LOG_PREFIX_FORMAT) + sum([f["len"] - 2 for f in consts.LOG_PREFIX_PARTS]) + extra_len + return self.console_prefix_length + extra_len def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = "") -> None: diff --git a/IoTuring/Logger/consts.py b/IoTuring/Logger/consts.py index fd717d04d..0fd30b3ae 100644 --- a/IoTuring/Logger/consts.py +++ b/IoTuring/Logger/consts.py @@ -2,24 +2,18 @@ DEFAULT_LOG_LEVEL = "INFO" MIN_CONSOLE_WIDTH = 95 -LOG_PREFIX_PARTS = [ - { - "attr": "asctime", - "len": 19 - }, - { - "attr": "levelname:^8s", - "len": 8 - }, - { - "attr": "source:^30s", - "len": 30 - } +LOG_PREFIX_LENGTHS = { + "asctime": 19, + "levelname:^8s": 8, + "source:^30s": 30 +} + +LOG_PREFIX_ENDS = [ + "[ ", " ] " ] -LOG_PREFIX_FORMAT = "[ {} | {} | {} ] " -LOG_PREFIX_STRING = LOG_PREFIX_FORMAT.format( - *["{" + f["attr"] + "}" for f in LOG_PREFIX_PARTS]) + +LOG_PREFIX_SEPARATOR = " | " # On/off states as strings: diff --git a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py index 669c5217f..31a430458 100644 --- a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py +++ b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py @@ -20,6 +20,7 @@ CONFIG_KEY_CONSOLE_LOG_LEVEL = "console_log_level" +CONFIG_KEY_CONSOLE_LOG_TIME = "console_log_time" CONFIG_KEY_FILE_LOG_LEVEL = "file_log_level" CONFIG_KEY_FILE_LOG_ENABLED = "file_log_enabled" CONFIG_KEY_FILE_LOG_PATH = "file_log_path" @@ -40,9 +41,12 @@ def __init__(self, single_configuration: SingleConfiguration) -> None: # Load settings to logger: logger = Logger() - console_level = LogLevel( - self.GetFromConfigurations(CONFIG_KEY_CONSOLE_LOG_LEVEL)) - logger.SetConsoleLogLevel(console_level) + logger.SetupConsoleLogging( + loglevel=LogLevel( + self.GetFromConfigurations(CONFIG_KEY_CONSOLE_LOG_LEVEL)), + include_time=self.GetTrueOrFalseFromConfigurations( + CONFIG_KEY_CONSOLE_LOG_TIME) + ) logger.SetupFileLogging( enabled=self.GetTrueOrFalseFromConfigurations( @@ -62,6 +66,9 @@ def ConfigurationPreset(cls): instruction="IOTURING_LOG_LEVEL envvar overwrites this setting!", choices=loglevel_choices) + preset.AddEntry(name="Display time in console log", key=CONFIG_KEY_CONSOLE_LOG_TIME, + question_type="yesno", default="Y") + preset.AddEntry(name="Enable file logging", key=CONFIG_KEY_FILE_LOG_ENABLED, question_type="yesno", default="N") From 5b452867be3d449435bcf65f60268d454f28e346 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 10 May 2024 09:37:17 +0200 Subject: [PATCH 10/12] File log fixes --- IoTuring/Logger/Logger.py | 31 ++++++++++++------- .../Deployments/LogSettings/LogSettings.py | 5 +-- 2 files changed, 23 insertions(+), 13 deletions(-) diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index f4d6c0ee8..1c08a419b 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -72,7 +72,8 @@ def __init__(self) -> None: self.logger.addHandler(self.console_handler) # Init file logger buffer handler: - self.memory_handler = logging.handlers.MemoryHandler(capacity=100) + self.memory_handler = logging.handlers.MemoryHandler( + capacity=100, flushOnClose=False) self.logger.addHandler(self.memory_handler) def SetupConsoleLogging(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_LEVEL), include_time: bool = True) -> None: @@ -93,23 +94,32 @@ def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path if enabled: self.StartFileLogging(loglevel, log_dir_path) - elif self.final_settings: - self.DisableFileLogging() + if self.final_settings: + self.FinalizeFileLogging(enabled) self.final_settings = True def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: - self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {log_dir_path.absolute()}", - logtarget=self.LOGTARGET_CONSOLE) - if self.file_handler: if log_dir_path.samefile(self.log_dir_path): - self.file_handler.setLevel(int(loglevel)) + if not self.file_handler.level == int(loglevel): + + old_filter = next( + (f for f in self.file_handler.filters if isinstance(f, LogLevelFilter))) + self.file_handler.removeFilter(old_filter) + + self.file_handler.addFilter(LogLevelFilter(loglevel)) + self.file_handler.setLevel(int(loglevel)) + return else: self.logger.removeHandler(self.file_handler) + else: + self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {log_dir_path.absolute()}", + logtarget=self.LOGTARGET_CONSOLE) + filepath = log_dir_path.joinpath(App.getName() + ".log") self.log_dir_path = log_dir_path @@ -127,12 +137,11 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.logger.addHandler(self.file_handler) self.memory_handler.setTarget(self.file_handler) - self.memory_handler.close() - self.logger.removeHandler(self.memory_handler) + self.memory_handler.flush() - def DisableFileLogging(self) -> None: + def FinalizeFileLogging(self, enabled: bool) -> None: - if self.file_handler: + if not enabled and self.file_handler: self.logger.removeHandler(self.file_handler) self.file_handler.close() diff --git a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py index 31a430458..4bc9e2d1a 100644 --- a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py +++ b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py @@ -74,11 +74,12 @@ def ConfigurationPreset(cls): preset.AddEntry(name="File log level", key=CONFIG_KEY_FILE_LOG_LEVEL, question_type="select", mandatory=True, default=str(LogLevel(consts.DEFAULT_LOG_LEVEL)), - choices=loglevel_choices) + choices=loglevel_choices, display_if_key_value={CONFIG_KEY_FILE_LOG_ENABLED: "Y"}) preset.AddEntry(name="File log path", key=CONFIG_KEY_FILE_LOG_PATH, question_type="filepath", mandatory=True, default=cls.GetDefaultLogPath(), - instruction="Directory where log files will be saved") + instruction="Directory where log files will be saved", + display_if_key_value={CONFIG_KEY_FILE_LOG_ENABLED: "Y"}) return preset From 4264b42a032789435208338b58d645f515c3053b Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 10 May 2024 14:09:31 +0200 Subject: [PATCH 11/12] Fixes --- IoTuring/Configurator/ConfiguratorLoader.py | 6 +- IoTuring/Logger/Logger.py | 225 ++++++++++-------- .../MyApp/SystemConsts/TerminalDetection.py | 2 +- .../Deployments/LogSettings/LogSettings.py | 7 +- IoTuring/Settings/Settings.py | 4 + IoTuring/__init__.py | 2 +- 6 files changed, 140 insertions(+), 106 deletions(-) diff --git a/IoTuring/Configurator/ConfiguratorLoader.py b/IoTuring/Configurator/ConfiguratorLoader.py index dbb9094b6..5771727d7 100644 --- a/IoTuring/Configurator/ConfiguratorLoader.py +++ b/IoTuring/Configurator/ConfiguratorLoader.py @@ -68,7 +68,7 @@ def LoadEntities(self) -> list[Entity]: # - pass the configuration to the warehouse function that uses the configuration to init the Warehouse # - append the Warehouse to the list - def LoadSettings(self) -> list[Settings]: + def LoadSettings(self, early_init:bool = False) -> list[Settings]: settings = [] scm = ClassManager(KEY_SETTINGS) settingsClasses = scm.ListAvailableClasses() @@ -80,11 +80,11 @@ def LoadSettings(self) -> list[Settings]: .GetConfigsOfType(sClass.NAME) if savedConfigs: - sc = sClass(savedConfigs[0]) + sc = sClass(savedConfigs[0], early_init) # Fallback to default: else: - sc = sClass(sClass.GetDefaultConfigurations()) + sc = sClass(sClass.GetDefaultConfigurations(), early_init) settings.append(sc) return settings diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index 1c08a419b..f25f00c71 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -52,9 +52,94 @@ def filter(self, record) -> bool: return True +class LogMessage(LogLevelObject): + + def __init__(self, source: str, message, color: str = "", logtarget: str = "") -> None: + + self.source = source + self.color = color + + self.msg = " ".join(self.MessageToList(message)) + + self.extra = {"source": self.source, + "file_message": self.msg, + "console_message": self.GetConsoleMessage(self.MessageToList(message)), + "color_prefix": self.GetColors()["prefix"], + "color_suffix": self.GetColors()["suffix"], + "logtarget": logtarget + } + + def GetColors(self) -> dict: + if TerminalDetection.CheckTerminalSupportsColors(): + return {"prefix": self.color, "suffix": Colors.reset} + else: + return {"prefix": "", "suffix": ""} + + def SetPrefixLength(self) -> None: + default_source_len = next( + (l for s, l in consts.LOG_PREFIX_LENGTHS.items() if s.startswith("source"))) + extra_len = max(len(self.source) - default_source_len, 0) + self.console_prefix_length = Logger().console_prefix_length + self.prefix_length = self.console_prefix_length + extra_len + + def GetConsoleMessage(self, messagelines: list[str]) -> str: + + if not TerminalDetection.CheckTerminalSupportsSize() or TerminalDetection.GetTerminalColumns() < consts.MIN_CONSOLE_WIDTH: + return self.msg + + self.SetPrefixLength() + + if len(messagelines) == 1 \ + and TerminalDetection.CalculateNumberOfLines(len(messagelines[0]) + self.prefix_length) == 1: + return messagelines[0] + + line_length = TerminalDetection.GetTerminalColumns() - \ + self.console_prefix_length + + final_lines = [] + + if self.prefix_length > self.console_prefix_length: + + first_line_len = TerminalDetection.GetTerminalColumns() - \ + self.prefix_length + final_lines.append(messagelines[0][:first_line_len]) + messagelines[0] = messagelines[0][first_line_len:] + + for l in messagelines: + final_lines.extend([l[i:i+line_length] + for i in range(0, len(l), line_length)]) + + line_prefix = "\n" + " " * self.console_prefix_length + return line_prefix.join(final_lines) + + @staticmethod + def MessageToList(message) -> list[str]: + """Convert message to a nice list of strings + + Args: + message (Any): The message + + Returns: + list[str]: Formatted lines of the message as a list + """ + if isinstance(message, list): + messagelines = [str(i) for i in message] + elif isinstance(message, dict): + messagelines = [f"{k}: {v}" for k, v in message.items()] + else: + messagelines = [str(message)] + + lines = [] + + # replace and split by newlines + for m in messagelines: + lines.extend(m.splitlines()) + + return [l.strip() for l in lines] + + class Logger(LogLevelObject, metaclass=Singleton): - final_settings = False log_dir_path = "" file_handler = None console_prefix_length = 0 @@ -89,36 +174,23 @@ def SetupConsoleLogging(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_L pass self.console_handler.setLevel(int(loglevel)) - def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path) -> None: + def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path, early_init: bool) -> None: if enabled: - self.StartFileLogging(loglevel, log_dir_path) - - if self.final_settings: - self.FinalizeFileLogging(enabled) - - self.final_settings = True - - def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: - - if self.file_handler: - if log_dir_path.samefile(self.log_dir_path): - if not self.file_handler.level == int(loglevel): - - old_filter = next( - (f for f in self.file_handler.filters if isinstance(f, LogLevelFilter))) - self.file_handler.removeFilter(old_filter) - - self.file_handler.addFilter(LogLevelFilter(loglevel)) - self.file_handler.setLevel(int(loglevel)) - - return + if self.file_handler: + self.UpdateFileLogging(loglevel, log_dir_path) else: + self.StartFileLogging(loglevel, log_dir_path) + else: + if self.file_handler: self.logger.removeHandler(self.file_handler) + self.file_handler.close() + self.file_handler = None - else: - self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {log_dir_path.absolute()}", - logtarget=self.LOGTARGET_CONSOLE) + if not early_init: + self.DisableFileLogBuffer() + + def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: filepath = log_dir_path.joinpath(App.getName() + ".log") self.log_dir_path = log_dir_path @@ -126,6 +198,9 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.file_handler = logging.handlers.RotatingFileHandler( filepath, backupCount=5) + self.Log(self.LOG_DEBUG, "FileLogger", f"Started file logging: {filepath.absolute()}", + logtarget=self.LOGTARGET_CONSOLE) + if filepath.exists(): self.file_handler.doRollover() @@ -139,11 +214,30 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.memory_handler.setTarget(self.file_handler) self.memory_handler.flush() - def FinalizeFileLogging(self, enabled: bool) -> None: + def UpdateFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: + if not self.file_handler: + raise Exception("File logger not initialized!") + + if log_dir_path.samefile(self.log_dir_path): + if not self.file_handler.level == int(loglevel): + + # Update loglevel: + old_filter = next( + (f for f in self.file_handler.filters if isinstance(f, LogLevelFilter))) + self.file_handler.removeFilter(old_filter) + + self.file_handler.addFilter(LogLevelFilter(loglevel)) + self.file_handler.setLevel(int(loglevel)) - if not enabled and self.file_handler: + else: + # Update path and loglevel self.logger.removeHandler(self.file_handler) - self.file_handler.close() + self.StartFileLogging(loglevel, log_dir_path) + + def DisableFileLogBuffer(self) -> None: + + self.Log(self.LOG_DEBUG, "FileLogger", "File log buffer disabled", + logtarget=self.LOGTARGET_CONSOLE) if self.memory_handler: self.logger.removeHandler(self.memory_handler) @@ -180,77 +274,12 @@ def GetFormatter(self, logtarget: str, include_time: bool = True) -> logging.For style="{" ) - def GetConsoleMessage(self, source: str, message) -> str: - - if isinstance(message, str) \ - and len(message.splitlines()) == 1 \ - and len(message) + self.GetPrefixLength(source) < TerminalDetection.GetTerminalColumns(): - return message.strip() - - line_length = TerminalDetection.GetTerminalColumns() - \ - self.GetPrefixLength() - - final_lines = [] - messagelines = self.GetMessageAsList(message) - - if self.GetPrefixLength(source) > self.GetPrefixLength(): - first_line_len = TerminalDetection.GetTerminalColumns() - \ - self.GetPrefixLength(source) - final_lines.append(messagelines[0][:first_line_len]) - messagelines[0] = messagelines[0][first_line_len:] - - for l in messagelines: - final_lines.extend([l[i:i+line_length] - for i in range(0, len(l), line_length)]) - - line_prefix = "\n" + " " * self.GetPrefixLength() - return line_prefix.join(final_lines) - - def GetFileMessage(self, message) -> str: - return " ".join(self.GetMessageAsList(message)) - - def GetMessageAsList(self, message) -> list[str]: - if isinstance(message, list): - messagelines = [str(i) for i in message] - elif isinstance(message, dict): - messagelines = [f"{k}: {v}" for k, v in message.items()] - else: - messagelines = [str(message)] - - lines = [] - - # replace and split by newlines - for m in messagelines: - lines.extend(m.splitlines()) - - return lines - - def GetPrefixLength(self, source: str = "") -> int: - default_source_len = next( - (l for s, l in consts.LOG_PREFIX_LENGTHS.items() if s.startswith("source"))) - extra_len = max(len(source) - default_source_len, 0) - return self.console_prefix_length + extra_len - def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = "") -> None: - extra = {"source": source, - "file_message": self.GetFileMessage(message), - "color_prefix": "", - "color_suffix": "", - "logtarget": logtarget - } - - if TerminalDetection.CheckTerminalSupportsSize() and TerminalDetection.GetTerminalColumns() > consts.MIN_CONSOLE_WIDTH: - extra["console_message"] = self.GetConsoleMessage(source, message) - else: - extra["console_message"] = extra["file_message"] - - if TerminalDetection.CheckTerminalSupportsColors(): - - if color or loglevel.color: - extra["color_prefix"] = color or loglevel.color - extra["color_suffix"] = Colors.reset + log_message = LogMessage( + source=source, message=message, color=color or loglevel.color, logtarget=logtarget) l = logging.getLogger(__name__).getChild(source) - l.log(int(loglevel), msg=message, extra=extra) + l.log(int(loglevel), + msg=log_message.msg, extra=log_message.extra) diff --git a/IoTuring/MyApp/SystemConsts/TerminalDetection.py b/IoTuring/MyApp/SystemConsts/TerminalDetection.py index 8f0f73d0c..b04083241 100644 --- a/IoTuring/MyApp/SystemConsts/TerminalDetection.py +++ b/IoTuring/MyApp/SystemConsts/TerminalDetection.py @@ -41,4 +41,4 @@ def CalculateNumberOfLines(string_length: int) -> int: Returns: int: Number of lines required """ - return int((string_length / shutil.get_terminal_size().columns) // 1) + return (string_length // shutil.get_terminal_size().columns) + 1 diff --git a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py index 4bc9e2d1a..72ddcaf7a 100644 --- a/IoTuring/Settings/Deployments/LogSettings/LogSettings.py +++ b/IoTuring/Settings/Deployments/LogSettings/LogSettings.py @@ -35,8 +35,8 @@ class LogSettings(Settings): NAME = "Log" - def __init__(self, single_configuration: SingleConfiguration) -> None: - super().__init__(single_configuration) + def __init__(self, single_configuration: SingleConfiguration, early_init: bool) -> None: + super().__init__(single_configuration, early_init) # Load settings to logger: logger = Logger() @@ -54,7 +54,8 @@ def __init__(self, single_configuration: SingleConfiguration) -> None: loglevel=LogLevel(self.GetFromConfigurations( CONFIG_KEY_FILE_LOG_LEVEL)), log_dir_path=Path(self.GetFromConfigurations( - CONFIG_KEY_FILE_LOG_PATH)) + CONFIG_KEY_FILE_LOG_PATH)), + early_init=early_init ) @classmethod diff --git a/IoTuring/Settings/Settings.py b/IoTuring/Settings/Settings.py index ffe851967..24c88e4cf 100644 --- a/IoTuring/Settings/Settings.py +++ b/IoTuring/Settings/Settings.py @@ -1,12 +1,16 @@ from IoTuring.Configurator.ConfiguratorObject import ConfiguratorObject from IoTuring.Settings.SettingsManager import SettingsManager from IoTuring.Configurator.MenuPreset import BooleanAnswers +from IoTuring.Configurator.Configuration import SingleConfiguration class Settings(ConfiguratorObject): """Base class for settings""" NAME = "Settings" + def __init__(self, single_configuration: SingleConfiguration, early_init: bool) -> None: + super().__init__(single_configuration) + @classmethod def GetFromSettingsConfigurations(cls, key: str): """Get value from settings' saved configurations from SettingsManager diff --git a/IoTuring/__init__.py b/IoTuring/__init__.py index f53e1eb46..cc1590b89 100644 --- a/IoTuring/__init__.py +++ b/IoTuring/__init__.py @@ -54,7 +54,7 @@ def loop(): configurator = Configurator() # Early log settings: - ConfiguratorLoader(configurator).LoadSettings() + ConfiguratorLoader(configurator).LoadSettings(early_init=True) logger.Log(Logger.LOG_DEBUG, "App", f"Selected options: {vars(args)}") From 5959f5441a2fe1393c66f27ba41cee87ed9a6572 Mon Sep 17 00:00:00 2001 From: infeeeee Date: Fri, 10 May 2024 14:52:42 +0200 Subject: [PATCH 12/12] Docstrings --- IoTuring/Configurator/ConfiguratorLoader.py | 9 +++ IoTuring/Logger/LogLevel.py | 7 ++ IoTuring/Logger/Logger.py | 79 +++++++++++++++++++-- IoTuring/Settings/Settings.py | 6 ++ 4 files changed, 97 insertions(+), 4 deletions(-) diff --git a/IoTuring/Configurator/ConfiguratorLoader.py b/IoTuring/Configurator/ConfiguratorLoader.py index 5771727d7..bd79502c1 100644 --- a/IoTuring/Configurator/ConfiguratorLoader.py +++ b/IoTuring/Configurator/ConfiguratorLoader.py @@ -69,6 +69,15 @@ def LoadEntities(self) -> list[Entity]: # - append the Warehouse to the list def LoadSettings(self, early_init:bool = False) -> list[Settings]: + """ Load all Settings classes + + Args: + early_init (bool, optional): True when loaded before configurator menu, False when added to SettingsManager. Defaults to False. + + Returns: + list[Settings]: Loaded classes + """ + settings = [] scm = ClassManager(KEY_SETTINGS) settingsClasses = scm.ListAvailableClasses() diff --git a/IoTuring/Logger/LogLevel.py b/IoTuring/Logger/LogLevel.py index 80c8123cd..c4919ec9a 100644 --- a/IoTuring/Logger/LogLevel.py +++ b/IoTuring/Logger/LogLevel.py @@ -18,8 +18,10 @@ def __init__(self, level_const: str) -> None: except AttributeError: raise UnknownLoglevelException(level_const) + # WARNING is yellow: if self.number == 30: self.color = Colors.yellow + # ERROR and CRITICAL red: elif self.number > 30: self.color = Colors.red else: @@ -46,4 +48,9 @@ class LogLevelObject: @classmethod def GetLoglevels(cls) -> list[LogLevel]: + """Get all available log levels + + Returns: + list[LogLevel]: List of LogLevel objects + """ return [getattr(cls, l) for l in dir(cls) if isinstance(getattr(cls, l), LogLevel)] diff --git a/IoTuring/Logger/Logger.py b/IoTuring/Logger/Logger.py index f25f00c71..e8f6b350a 100644 --- a/IoTuring/Logger/Logger.py +++ b/IoTuring/Logger/Logger.py @@ -40,7 +40,7 @@ def filter(self, record) -> bool: class LogLevelFilter(logging.Filter): - """ Log filter for loglevel, for later file logging""" + """ Log filter for loglevel, for file logging from buffer""" def __init__(self, loglevel: LogLevel) -> None: self.loglevel = loglevel @@ -53,11 +53,13 @@ def filter(self, record) -> bool: class LogMessage(LogLevelObject): + """Class for formatting log messages""" def __init__(self, source: str, message, color: str = "", logtarget: str = "") -> None: self.source = source self.color = color + self.logtarget = logtarget self.msg = " ".join(self.MessageToList(message)) @@ -66,16 +68,18 @@ def __init__(self, source: str, message, color: str = "", logtarget: str = "") - "console_message": self.GetConsoleMessage(self.MessageToList(message)), "color_prefix": self.GetColors()["prefix"], "color_suffix": self.GetColors()["suffix"], - "logtarget": logtarget + "logtarget": self.logtarget } def GetColors(self) -> dict: + """Get color prefix and suffix""" if TerminalDetection.CheckTerminalSupportsColors(): return {"prefix": self.color, "suffix": Colors.reset} else: return {"prefix": "", "suffix": ""} def SetPrefixLength(self) -> None: + """Calculate the length of the log prefix""" default_source_len = next( (l for s, l in consts.LOG_PREFIX_LENGTHS.items() if s.startswith("source"))) extra_len = max(len(self.source) - default_source_len, 0) @@ -83,32 +87,48 @@ def SetPrefixLength(self) -> None: self.prefix_length = self.console_prefix_length + extra_len def GetConsoleMessage(self, messagelines: list[str]) -> str: + """Get the formatted message for console logging - if not TerminalDetection.CheckTerminalSupportsSize() or TerminalDetection.GetTerminalColumns() < consts.MIN_CONSOLE_WIDTH: + Args: + messagelines (list[str]): Message as separate lines + + Returns: + str: the formatted message + """ + + # Return single line if unsupported or too small terminal, or console logging disabled: + if not TerminalDetection.CheckTerminalSupportsSize() \ + or TerminalDetection.GetTerminalColumns() < consts.MIN_CONSOLE_WIDTH \ + or self.logtarget == self.LOGTARGET_FILE: return self.msg + # Calculate the length of the prefix: self.SetPrefixLength() + # Single line log, and it can be displayed without linebreaks, next to the prefix: if len(messagelines) == 1 \ and TerminalDetection.CalculateNumberOfLines(len(messagelines[0]) + self.prefix_length) == 1: return messagelines[0] + # Available space for the message line_length = TerminalDetection.GetTerminalColumns() - \ self.console_prefix_length final_lines = [] + # If the prefix longer in this line than de default, make the first line shorter: if self.prefix_length > self.console_prefix_length: - first_line_len = TerminalDetection.GetTerminalColumns() - \ self.prefix_length final_lines.append(messagelines[0][:first_line_len]) messagelines[0] = messagelines[0][first_line_len:] + # Cut the to the correct length: for l in messagelines: final_lines.extend([l[i:i+line_length] for i in range(0, len(l), line_length)]) + # Linebrakes and spaces: line_prefix = "\n" + " " * self.console_prefix_length return line_prefix.join(final_lines) @@ -162,6 +182,12 @@ def __init__(self) -> None: self.logger.addHandler(self.memory_handler) def SetupConsoleLogging(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_LEVEL), include_time: bool = True) -> None: + """Change settings of console logging. This is called from LogSettings init. + + Args: + loglevel (LogLevel, optional): Loglevel to use. ENVVAR owerwrites thi. Defaults to LogLevel(consts.DEFAULT_LOG_LEVEL). + include_time (bool, optional): If the time should be included in the log. Defaults to True. + """ self.console_handler.setFormatter( self.GetFormatter(self.LOGTARGET_CONSOLE, include_time)) @@ -175,6 +201,14 @@ def SetupConsoleLogging(self, loglevel: LogLevel = LogLevel(consts.DEFAULT_LOG_L self.console_handler.setLevel(int(loglevel)) def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path, early_init: bool) -> None: + """Manage file logging. This is called from LogSettings init + + Args: + enabled (bool): If File logging enabled or disabled + loglevel (LogLevel): Loglevel to use + log_dir_path (Path): Path to directory containing log files + early_init (bool): If this is the early or late init. + """ if enabled: if self.file_handler: @@ -182,6 +216,7 @@ def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path else: self.StartFileLogging(loglevel, log_dir_path) else: + # Disable file logging: if self.file_handler: self.logger.removeHandler(self.file_handler) self.file_handler.close() @@ -191,6 +226,12 @@ def SetupFileLogging(self, enabled: bool, loglevel: LogLevel, log_dir_path: Path self.DisableFileLogBuffer() def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: + """Start and setup file logging + + Args: + loglevel (LogLevel): Loglevel to use + log_dir_path (Path): Path to directory containing log files + """ filepath = log_dir_path.joinpath(App.getName() + ".log") self.log_dir_path = log_dir_path @@ -215,6 +256,12 @@ def StartFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.memory_handler.flush() def UpdateFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: + """Change settings of enabled file logging + + Args: + loglevel (LogLevel): Loglevel to use + log_dir_path (Path): Path to directory containing log files + """ if not self.file_handler: raise Exception("File logger not initialized!") @@ -235,6 +282,7 @@ def UpdateFileLogging(self, loglevel: LogLevel, log_dir_path: Path) -> None: self.StartFileLogging(loglevel, log_dir_path) def DisableFileLogBuffer(self) -> None: + """Disable the buffer after file logger was finally disabled or enabled""" self.Log(self.LOG_DEBUG, "FileLogger", "File log buffer disabled", logtarget=self.LOGTARGET_CONSOLE) @@ -244,6 +292,18 @@ def DisableFileLogBuffer(self) -> None: self.memory_handler.close() def GetFormatter(self, logtarget: str, include_time: bool = True) -> logging.Formatter: + """Get the formatter for this logging handle + + Args: + logtarget (str): self.LOGTARGET_CONSOLE or self.LOGTARGET_FILE + include_time (bool, optional): If the time should be included in the log, only affects console logging. Defaults to True. + + Raises: + Exception: invalid logtarget + + Returns: + logging.Formatter: Forrmatter for logging handler + """ prefix_lengths = consts.LOG_PREFIX_LENGTHS.copy() @@ -263,8 +323,10 @@ def GetFormatter(self, logtarget: str, include_time: bool = True) -> logging.For fmt = "{color_prefix}" + prefix_string + \ "{console_message}{color_suffix}" self.console_prefix_length = prefix_length + elif logtarget == self.LOGTARGET_FILE: fmt = prefix_string + "{file_message}" + else: raise Exception(f"Unknown logtarget: {logtarget}") @@ -275,6 +337,15 @@ def GetFormatter(self, logtarget: str, include_time: bool = True) -> logging.For ) def Log(self, loglevel: LogLevel, source: str, message, color: str = "", logtarget: str = "") -> None: + """Log a message + + Args: + loglevel (LogLevel): The loglevel + source (str): Source module name + message (any): The message to log + color (str, optional): Override log color. Defaults to "". + logtarget (str, optional): self.LOGTARGET_CONSOLE or self.LOGTARGET_FILE. Defaults to "". + """ log_message = LogMessage( source=source, message=message, color=color or loglevel.color, logtarget=logtarget) diff --git a/IoTuring/Settings/Settings.py b/IoTuring/Settings/Settings.py index 24c88e4cf..21e1ac008 100644 --- a/IoTuring/Settings/Settings.py +++ b/IoTuring/Settings/Settings.py @@ -9,6 +9,12 @@ class Settings(ConfiguratorObject): NAME = "Settings" def __init__(self, single_configuration: SingleConfiguration, early_init: bool) -> None: + """Initialize a settings class + + Args: + single_configuration (SingleConfiguration): The configuration + early_init (bool): True when loaded before configurator menu, False when added to SettingsManager + """ super().__init__(single_configuration) @classmethod