diff --git a/Starter/TODO.md b/Starter/TODO.md index fc13535..f0b0e0c 100644 --- a/Starter/TODO.md +++ b/Starter/TODO.md @@ -1,5 +1,3 @@ # TODO See the [issues](https://github.com/drkostas/starter/issues) too. -- [X] Create Tests -- [X] Create Readme -- [ ] Stop Global Warming +- [X] Initialize template repository diff --git a/Starter/confs/template_conf.yml b/Starter/confs/template_conf.yml index fbb0910..0c43ac7 100644 --- a/Starter/confs/template_conf.yml +++ b/Starter/confs/template_conf.yml @@ -1,9 +1,9 @@ tag: template -cloudstore: +cloud-filemanager: - config: api_key: yourapikey type: dropbox -datastore: +high-sql: - config: hostname: hostname username: username @@ -11,7 +11,7 @@ datastore: db_name: mydb port: 3306 type: mysql -emailer: +pyemail-sender: - config: email_address: foo@gmail.com api_key: 123 diff --git a/Starter/confs/template_conf_with_env_variables.yml b/Starter/confs/template_conf_with_env_variables.yml index df2c158..a45f0f2 100644 --- a/Starter/confs/template_conf_with_env_variables.yml +++ b/Starter/confs/template_conf_with_env_variables.yml @@ -1,9 +1,9 @@ tag: template -cloudstore: +cloud-filemanager: - config: api_key: !ENV ${DROPBOX_API_KEY} type: dropbox -datastore: +high-sql: - config: hostname: !ENV ${MYSQL_HOST} username: !ENV ${MYSQL_USERNAME} @@ -11,7 +11,7 @@ datastore: db_name: !ENV ${MYSQL_DB_NAME} port: 3306 type: mysql -emailer: +pyemail-sender: - config: email_address: !ENV ${EMAIL_ADDRESS} api_key: !ENV ${GMAIL_API_KEY} diff --git a/Starter/requirements.txt b/Starter/requirements.txt index a3ce04e..d403818 100644 --- a/Starter/requirements.txt +++ b/Starter/requirements.txt @@ -1,9 +1,8 @@ -dropbox~=11.10.0 -gmail~=0.6.3 -jsonschema~=3.2.0 -mysql-connector-python~=8.0.19 -mysql-connector~=2.2.9 -PyYAML~=5.4.1 setuptools~=52.0.0 -termcolor~=1.1.0 typer~=0.3.2 +yaml-config-wrapper>=1.0.4 +termcolor-logger>=1.0.3 +pyemail-sender>=1.0.1 +high-sql>=1.0.2 +cloud-filemanager>=1.0.1 +bench-utils>=1.0.3 \ No newline at end of file diff --git a/Starter/setup.py b/Starter/setup.py index fdf2701..09752c1 100644 --- a/Starter/setup.py +++ b/Starter/setup.py @@ -42,7 +42,9 @@ def run(self): 'starter_main = starter.main:main' ] -data_files = ['starter/configuration/yml_schema.json'] +data_files = [ + # 'starter/a-file' +] setup( author="drkostas", diff --git a/Starter/starter/__init__.py b/Starter/starter/__init__.py index c770516..0bd3b95 100644 --- a/Starter/starter/__init__.py +++ b/Starter/starter/__init__.py @@ -1,12 +1,11 @@ """Top-level package for Starter.""" -from starter.fancy_logger import ColorizedLogger -from starter.timing_tools import timeit -from starter.profiling_funcs import profileit -from starter.configuration import Configuration, validate_json_schema -from starter.cloudstore import DropboxCloudstore -from starter.datastore import MySqlDatastore -from starter.emailer import GmailEmailer +from termcolor_logger import ColorLogger +from bench_utils import timeit, profileit +from yaml_config_wrapper import Configuration, validate_json_schema +from cloud_filemanager import DropboxCloudManager +from high_sql import HighMySQL +from pyemail_sender import GmailPyEmailSender __author__ = "drkostas" __email__ = "georgiou.kostas94@gmail.com" diff --git a/Starter/starter/cloudstore/__init__.py b/Starter/starter/cloudstore/__init__.py deleted file mode 100644 index c5075f9..0000000 --- a/Starter/starter/cloudstore/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cloudstore sub-package of Starter.""" - -from .dropbox_cloudstore import DropboxCloudstore - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" diff --git a/Starter/starter/cloudstore/abstract_cloudstore.py b/Starter/starter/cloudstore/abstract_cloudstore.py deleted file mode 100644 index d626230..0000000 --- a/Starter/starter/cloudstore/abstract_cloudstore.py +++ /dev/null @@ -1,72 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractCloudstore(ABC): - __slots__ = ('_handler',) - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """ - Tha basic constructor. Creates a new instance of Cloudstore using the specified credentials - """ - - pass - - @staticmethod - @abstractmethod - def get_handler(*args, **kwargs): - """ - Returns a Cloudstore handler. - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def upload_file(self, *args, **kwargs): - """ - Uploads a file to the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def download_file(self, *args, **kwargs): - """ - Downloads a file from the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def delete_file(self, *args, **kwargs): - """ - Deletes a file from the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def ls(self, *args, **kwargs): - """ - List the files and folders in the Cloudstore - :param args: - :param kwargs: - :return: - """ - pass diff --git a/Starter/starter/cloudstore/dropbox_cloudstore.py b/Starter/starter/cloudstore/dropbox_cloudstore.py deleted file mode 100644 index e7f982d..0000000 --- a/Starter/starter/cloudstore/dropbox_cloudstore.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import Dict, Union -from dropbox import Dropbox, files, exceptions - -from .abstract_cloudstore import AbstractCloudstore -from starter import ColorizedLogger - -logger = ColorizedLogger('DropboxCloudstore') - - -class DropboxCloudstore(AbstractCloudstore): - __slots__ = '_handler' - - _handler: Dropbox - - def __init__(self, config: Dict) -> None: - """ - The basic constructor. Creates a new instance of Cloudstore using the specified credentials - - :param config: - """ - - self._handler = self.get_handler(api_key=config['api_key']) - super().__init__() - - @staticmethod - def get_handler(api_key: str) -> Dropbox: - """ - Returns a Cloudstore handler. - - :param api_key: - :return: - """ - - dbx = Dropbox(api_key) - return dbx - - def upload_file(self, file_bytes: bytes, upload_path: str, write_mode: str = 'overwrite') -> None: - """ - Uploads a file to the Cloudstore - - :param file_bytes: - :param upload_path: - :param write_mode: - :return: - """ - - # TODO: Add option to support FileStream, StringIO and FilePath - try: - logger.debug("Uploading file to path: %s" % upload_path) - self._handler.files_upload(f=file_bytes, path=upload_path, - mode=files.WriteMode(write_mode)) - except exceptions.ApiError as err: - logger.error('API error: %s' % err) - - def download_file(self, frompath: str, tofile: str = None) -> Union[bytes, None]: - """ - Downloads a file from the Cloudstore - - :param frompath: - :param tofile: - :return: - """ - - try: - if tofile is not None: - logger.debug("Downloading file from path: %s to path %s" % (frompath, tofile)) - self._handler.files_download_to_file(download_path=tofile, path=frompath) - else: - logger.debug("Downloading file from path: %s to variable" % frompath) - md, res = self._handler.files_download(path=frompath) - data = res.content # The bytes of the file - return data - except exceptions.HttpError as err: - logger.error('HTTP error %s' % err) - return None - - def delete_file(self, file_path: str) -> None: - """ - Deletes a file from the Cloudstore - - :param file_path: - :return: - """ - - try: - logger.debug("Deleting file from path: %s" % file_path) - self._handler.files_delete_v2(path=file_path) - except exceptions.ApiError as err: - logger.error('API error %s' % err) - - def ls(self, path: str = '') -> Dict: - """ - List the files and folders in the Cloudstore - - :param path: - :return: - """ - try: - files_list = self._handler.files_list_folder(path=path) - files_dict = {} - for entry in files_list.entries: - files_dict[entry.name] = entry - return files_dict - except exceptions.ApiError as err: - logger.error('Folder listing failed for %s -- assumed empty: %s' % (path, err)) - return {} diff --git a/Starter/starter/configuration/__init__.py b/Starter/starter/configuration/__init__.py deleted file mode 100644 index 4897fe5..0000000 --- a/Starter/starter/configuration/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Configuration sub-package of Starter.""" - -from .configuration import Configuration, validate_json_schema - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" diff --git a/Starter/starter/configuration/configuration.py b/Starter/starter/configuration/configuration.py deleted file mode 100644 index efd76e0..0000000 --- a/Starter/starter/configuration/configuration.py +++ /dev/null @@ -1,177 +0,0 @@ -import os -from typing import Dict, List, Tuple, Union -import json -import _io -from io import StringIO, TextIOWrapper -import re -import yaml -from jsonschema import validate as validate_json_schema - -from starter import ColorizedLogger - -logger = ColorizedLogger('Config', 'white') - - -class Configuration: - __slots__ = ('config', 'config_path', 'config_keys', 'tag') - - config: Dict - config_path: str - tag: str - config_keys: List - env_variable_tag: str = '!ENV' - env_variable_pattern: str = r'.*?\${(\w+)}.*?' # ${var} - - def __init__(self, config_src: Union[TextIOWrapper, StringIO, str], - config_schema_path: str = 'yml_schema.json'): - """ - The basic constructor. Creates a new instance of the Configuration class. - - Args: - config_src: The path, file or StringIO object of the configuration to load - config_schema_path: The path, file or StringIO object of the configuration validation file - """ - - # Load the predefined schema of the configuration - configuration_schema = self.load_configuration_schema(config_schema_path=config_schema_path) - # Load the configuration - self.config, self.config_path = self.load_yml(config_src=config_src, - env_tag=self.env_variable_tag, - env_pattern=self.env_variable_pattern) - # Validate the config - validate_json_schema(self.config, configuration_schema) - logger.debug("Schema Validation was Successful.") - # Set the config properties as instance attributes - self.tag = self.config['tag'] - self.config_keys = [key for key in self.config.keys() if key != 'tag'] - logger.info(f"Configuration file loaded successfully from path: {self.config_path}") - logger.info(f"Configuration Tag: {self.tag}") - - @staticmethod - def load_configuration_schema(config_schema_path: str) -> Dict: - """ - Loads the configuration schema file - - Args: - config_schema_path: The path of the config schema - - Returns: - configuration_schema: The loaded config schema - """ - - if config_schema_path[0] != os.sep: - config_schema_path = '/'.join( - [os.path.dirname(os.path.realpath(__file__)), config_schema_path]) - with open(config_schema_path) as f: - configuration_schema = json.load(f) - return configuration_schema - - @staticmethod - def load_yml(config_src: Union[TextIOWrapper, StringIO, str], env_tag: str, env_pattern: str) -> \ - Tuple[Dict, str]: - """ - Loads the configuration file - Args: - config_src: The path of the configuration - env_tag: The tag that distinguishes the env variables - env_pattern: The regex for finding the env variables - - Returns: - config, config_path - """ - pattern = re.compile(env_pattern) - loader = yaml.SafeLoader - loader.add_implicit_resolver(env_tag, pattern, None) - - def constructor_env_variables(loader, node): - """ - Extracts the environment variable from the node's value - :param yaml.Loader loader: the yaml loader - :param node: the current node in the yaml - :return: the parsed string that contains the value of the environment - variable - """ - value = loader.construct_scalar(node) - match = pattern.findall(value) # to find all env variables in line - if match: - full_value = value - for g in match: - full_value = full_value.replace( - f'${{{g}}}', os.environ.get(g, g) - ) - return full_value - return value - - loader.add_constructor(env_tag, constructor_env_variables) - - if isinstance(config_src, TextIOWrapper): - logger.debug("Loading yaml from TextIOWrapper") - config = yaml.load(config_src, Loader=loader) - config_path = os.path.abspath(config_src.name) - elif isinstance(config_src, StringIO): - logger.debug("Loading yaml from StringIO") - config = yaml.load(config_src, Loader=loader) - config_path = "StringIO" - elif isinstance(config_src, str): - config_path = os.path.abspath(config_src) - logger.debug("Loading yaml from path") - with open(config_path) as f: - config = yaml.load(f, Loader=loader) - else: - raise TypeError('Config file must be TextIOWrapper or path to a file') - return config, config_path - - def get_config(self, config_name) -> List: - """ - Returns the subconfig requested - - Args: - config_name: The name of the subconfig - - Returns: - sub_config: The sub_configs List - """ - - if config_name in self.config.keys(): - return self.config[config_name] - else: - raise ConfigurationError('Config property %s not set!' % config_name) - - def to_yml(self, fn: Union[str, _io.TextIOWrapper]) -> None: - """ - Writes the configuration to a stream. For example a file. - - Args: - fn: - - Returns: - """ - - self.config['tag'] = self.tag - if isinstance(fn, str): - with open(fn, 'w') as f: - yaml.dump(self.config, f, default_flow_style=False) - elif isinstance(fn, _io.TextIOWrapper): - yaml.dump(self.config, fn, default_flow_style=False) - else: - raise TypeError('Expected str or _io.TextIOWrapper not %s' % (type(fn))) - - to_yaml = to_yml - - def to_json(self) -> Dict: - """ - Returns the whole config file - - Returns: - - """ - return self.config - - # def __getitem__(self, item): - # return self.get_config(item) - - -class ConfigurationError(Exception): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super().__init__(message) diff --git a/Starter/starter/configuration/yml_schema.json b/Starter/starter/configuration/yml_schema.json deleted file mode 100644 index b3e6c14..0000000 --- a/Starter/starter/configuration/yml_schema.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Python Configuration", - "description": "A json for python configuration in yml format", - "type": "object", - "properties": { - "tag": { - "type": "string" - } - }, - "required": [ - "tag" - ], - "definitions": { - }, - "additionalProperties": true -} \ No newline at end of file diff --git a/Starter/starter/configuration/yml_schema_strict.json b/Starter/starter/configuration/yml_schema_strict.json deleted file mode 100644 index 0856d43..0000000 --- a/Starter/starter/configuration/yml_schema_strict.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "tag": { - "type": "string" - }, - "example_db": { - "$ref": "#/definitions/example_db" - } - }, - "required": [ - "tag", - "example_db" - ], - "definitions": { - "example_db": { - "type": "array", - "items": { - "type": "object", - "required": [ - "type", - "properties" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "mysql", - "mongodb" - ] - }, - "properties": { - "type": "object", - "additionalProperties": false, - "required": [ - "hostname", - "username", - "password", - "db_name" - ], - "properties": { - "hostname": { - "type": "string" - }, - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "db_name": { - "type": "string" - }, - "port": { - "type": "integer" - } - } - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/Starter/starter/datastore/__init__.py b/Starter/starter/datastore/__init__.py deleted file mode 100644 index dc70c9e..0000000 --- a/Starter/starter/datastore/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cloudstore sub-package of Starter.""" - -from .mysql_datastore import MySqlDatastore - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" diff --git a/Starter/starter/datastore/abstract_datastore.py b/Starter/starter/datastore/abstract_datastore.py deleted file mode 100644 index bde2319..0000000 --- a/Starter/starter/datastore/abstract_datastore.py +++ /dev/null @@ -1,59 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict - - -class AbstractDatastore(ABC): - __slots__ = ('_connection', '_cursor') - - @abstractmethod - def __init__(self, config: Dict) -> None: - """ - Tha basic constructor. Creates a new instance of Datastore using the specified credentials - - :param config: - """ - - self._connection, self._cursor = self.get_connection(username=config['username'], - password=config['password'], - hostname=config['hostname'], - db_name=config['db_name'], - port=config['port']) - - @staticmethod - @abstractmethod - def get_connection(username: str, password: str, hostname: str, db_name: str, port: int): - pass - - @abstractmethod - def create_table(self, table: str, schema: str): - pass - - @abstractmethod - def drop_table(self, table: str) -> None: - pass - - @abstractmethod - def truncate_table(self, table: str) -> None: - pass - - @abstractmethod - def insert_into_table(self, table: str, data: dict) -> None: - pass - - @abstractmethod - def update_table(self, table: str, set_data: dict, where: str) -> None: - pass - - @abstractmethod - def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', - order_by: str = 'NULL', - asc_or_desc: str = 'ASC', limit: int = 1000) -> List: - pass - - @abstractmethod - def delete_from_table(self, table: str, where: str) -> None: - pass - - @abstractmethod - def show_tables(self, *args, **kwargs) -> List: - pass diff --git a/Starter/starter/datastore/mysql_datastore.py b/Starter/starter/datastore/mysql_datastore.py deleted file mode 100644 index 9224242..0000000 --- a/Starter/starter/datastore/mysql_datastore.py +++ /dev/null @@ -1,192 +0,0 @@ -from typing import List, Tuple, Dict - -from mysql import connector as mysql_connector -import mysql.connector.cursor - -from .abstract_datastore import AbstractDatastore -from starter import ColorizedLogger - -logger = ColorizedLogger('MySqlDataStore') - - -class MySqlDatastore(AbstractDatastore): - __slots__ = ('_connection', '_cursor') - - _connection: mysql_connector.MySQLConnection - _cursor: mysql_connector.cursor.MySQLCursor - - def __init__(self, config: Dict) -> None: - """ - The basic constructor. Creates a new instance of Datastore using the specified credentials - - :param config: - """ - - super().__init__(config) - - @staticmethod - def get_connection(username: str, password: str, hostname: str, db_name: str, port: int = 3306) \ - -> Tuple[mysql_connector.MySQLConnection, mysql_connector.cursor.MySQLCursor]: - """ - Creates and returns a connection and a cursor/session to the MySQL DB - - :param username: - :param password: - :param hostname: - :param db_name: - :param port: - :return: - """ - - connection = mysql_connector.connect( - host=hostname, - user=username, - passwd=password, - database=db_name, - use_pure=True - ) - - cursor = connection.cursor() - return connection, cursor - - def create_table(self, table: str, schema: str) -> None: - """ - Creates a table using the specified schema - - :param self: - :param table: - :param schema: - :return: - """ - - query = "CREATE TABLE IF NOT EXISTS {table} ({schema})".format(table=table, schema=schema) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def drop_table(self, table: str) -> None: - """ - Drops the specified table if it exists - - :param self: - :param table: - :return: - """ - - query = "DROP TABLE IF EXISTS {table}".format(table=table) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def truncate_table(self, table: str) -> None: - """ - Truncates the specified table - - :param self: - :param table: - :return: - """ - - query = "TRUNCATE TABLE {table}".format(table=table) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def insert_into_table(self, table: str, data: dict) -> None: - """ - Inserts into the specified table a row based on a column_name: value dictionary - - :param self: - :param table: - :param data: - :return: - """ - - data_str = ", ".join( - list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), data.keys(), data.values()))) - - query = "INSERT INTO {table} SET {data}".format(table=table, data=data_str) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def update_table(self, table: str, set_data: dict, where: str) -> None: - """ - Updates the specified table using a column_name: value dictionary and a where statement - - :param self: - :param table: - :param set_data: - :param where: - :return: - """ - - set_data_str = ", ".join( - list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), set_data.keys(), - set_data.values()))) - - query = "UPDATE {table} SET {data} WHERE {where}".format(table=table, data=set_data_str, where=where) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', order_by: str = 'NULL', - asc_or_desc: str = 'ASC', limit: int = 1000) -> List: - """ - Selects from a specified table based on the given columns, where, ordering and limit - - :param self: - :param table: - :param columns: - :param where: - :param order_by: - :param asc_or_desc: - :param limit: - :return results: - """ - - query = "SELECT {columns} FROM {table} WHERE {where} ORDER BY {order_by} {asc_or_desc} LIMIT {limit}".format( - columns=columns, table=table, where=where, order_by=order_by, asc_or_desc=asc_or_desc, limit=limit) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - results = self._cursor.fetchall() - - return results - - def delete_from_table(self, table: str, where: str) -> None: - """ - Deletes data from the specified table based on a where statement - - :param self: - :param table: - :param where: - :return: - """ - - query = "DELETE FROM {table} WHERE {where}".format(table=table, where=where) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def show_tables(self) -> List: - """ - Show a list of the tables present in the db - :return: - """ - - query = 'SHOW TABLES' - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - results = self._cursor.fetchall() - - return [result[0] for result in results] - - def __exit__(self) -> None: - """ - Flushes and closes the connection - - :return: - """ - - self._connection.commit() - self._cursor.close() diff --git a/Starter/starter/emailer/__init__.py b/Starter/starter/emailer/__init__.py deleted file mode 100644 index e4bc778..0000000 --- a/Starter/starter/emailer/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Emailer sub-package of Starter.""" - -from .gmail_emailer import GmailEmailer - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" diff --git a/Starter/starter/emailer/abstract_emailer.py b/Starter/starter/emailer/abstract_emailer.py deleted file mode 100644 index 17b85d7..0000000 --- a/Starter/starter/emailer/abstract_emailer.py +++ /dev/null @@ -1,39 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractEmailer(ABC): - __slots__ = ('_handler',) - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """ - Tha basic constructor. Creates a new instance of EmailApp using the specified credentials - - """ - - pass - - @staticmethod - @abstractmethod - def get_handler(*args, **kwargs): - """ - Returns an EmailApp handler. - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def send_email(self, *args, **kwargs): - """ - Sends an email with the specified arguments. - - :param args: - :param kwargs: - :return: - """ - - pass diff --git a/Starter/starter/emailer/gmail_emailer.py b/Starter/starter/emailer/gmail_emailer.py deleted file mode 100644 index 7175d4f..0000000 --- a/Starter/starter/emailer/gmail_emailer.py +++ /dev/null @@ -1,87 +0,0 @@ -from typing import List, Dict -import logging -from gmail import GMail, Message - -from .abstract_emailer import AbstractEmailer -from starter import ColorizedLogger - -logger = ColorizedLogger('GmailEmailer') - - -class GmailEmailer(AbstractEmailer): - __slots__ = ('_handler', 'email_address', 'test_mode') - - _handler: GMail - test_mode: bool - - def __init__(self, config: Dict, test_mode: bool = False) -> None: - """ - The basic constructor. Creates a new instance of EmailApp using the specified credentials - - :param config: - :param test_mode: - """ - - self.email_address = config['email_address'] - self._handler = self.get_handler(email_address=self.email_address, - api_key=config['api_key']) - self.test_mode = test_mode - super().__init__() - - @staticmethod - def get_handler(email_address: str, api_key: str) -> GMail: - """ - Returns an EmailApp handler. - - :param email_address: - :param api_key: - :return: - """ - - gmail_handler = GMail(username=email_address, password=api_key) - gmail_handler.connect() - return gmail_handler - - def is_connected(self) -> bool: - return self._handler.is_connected() - - def get_self_email(self): - return self.email_address - - def send_email(self, subject: str, to: List, cc: List = None, bcc: List = None, text: str = None, - html: str = None, - attachments: List = None, sender: str = None, reply_to: str = None) -> None: - """ - Sends an email with the specified arguments. - - :param subject: - :param to: - :param cc: - :param bcc: - :param text: - :param html: - :param attachments: - :param sender: - :param reply_to: - :return: - """ - - if self.test_mode: - to = self.email_address - cc = self.email_address if cc is not None else None - bcc = self.email_address if bcc is not None else None - - msg = Message(subject=subject, - to=",".join(to), - cc=",".join(cc) if cc is not None else None, - bcc=",".join(bcc) if cc is not None else None, - text=text, - html=html, - attachments=attachments, - sender=sender, - reply_to=reply_to) - logger.debug("Sending email with Message: %s" % msg) - self._handler.send(msg) - - def __exit__(self): - self._handler.close() diff --git a/Starter/starter/fancy_logger/__init__.py b/Starter/starter/fancy_logger/__init__.py deleted file mode 100644 index 3167fe0..0000000 --- a/Starter/starter/fancy_logger/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""FancyLog sub-package of Starter.""" - -from .colorized_logger import ColorizedLogger - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" - diff --git a/Starter/starter/fancy_logger/abstract_fancy_logger.py b/Starter/starter/fancy_logger/abstract_fancy_logger.py deleted file mode 100644 index 96eea1a..0000000 --- a/Starter/starter/fancy_logger/abstract_fancy_logger.py +++ /dev/null @@ -1,19 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractFancyLogger(ABC): - """Abstract class of the FancyLog package""" - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """The basic constructor. Creates a new instance of FancyLog using the - specified arguments - - Args: - *args: - **kwargs: - """ - - @abstractmethod - def create_logger(self, *args, **kwargs): - pass diff --git a/Starter/starter/fancy_logger/colorized_logger.py b/Starter/starter/fancy_logger/colorized_logger.py deleted file mode 100644 index b588075..0000000 --- a/Starter/starter/fancy_logger/colorized_logger.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -from typing import List, Union -import types -import logging -from termcolor import colored - -from .abstract_fancy_logger import AbstractFancyLogger - - -class ColorizedLogger(AbstractFancyLogger): - """ColorizedLogger class of the FancyLog package""" - - __slots__ = ('_logger', 'logger_name', '_color', '_on_color', '_attrs', - 'debug', 'info', 'warn', 'warning', 'error', 'exception', 'critical') - - log_fmt: str = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' - log_date_fmt: str = '%Y-%m-%d %H:%M:%S' - log_level: Union[int, str] = logging.INFO - _logger: logging.Logger - log_path: str = None - logger_name: str - _color: str - _on_color: str - _attrs: List - - def __init__(self, logger_name: str, - color: str = 'white', on_color: str = None, - attrs: List = None) -> None: - """ - Args: - logger_name (str): - color (str): - attrs (List): AnyOf('bold', 'dark', 'underline', 'blink', 'reverse', 'concealed') - """ - - self._color = color - self._on_color = on_color - self._attrs = attrs if attrs else ['bold'] - self.logger_name = logger_name - self._logger = self.create_logger(logger_name=logger_name) - super().__init__() - - def __getattr__(self, name: str): - """ - Args: - name (str): - """ - - def log_colored(log_text: str, *args, **kwargs): - color = self._color if 'color' not in kwargs else kwargs['color'] - on_color = self._on_color if 'on_color' not in kwargs else kwargs['on_color'] - attrs = self._attrs if 'attrs' not in kwargs else kwargs['attrs'] - colored_text = colored(log_text, color=color, on_color=on_color, attrs=attrs) - return getattr(self._logger, name)(colored_text, *args) - - if name in ['debug', 'info', 'warn', 'warning', - 'error', 'exception', 'critical']: - self.add_file_handler_if_needed(self._logger) - return log_colored - elif name in ['newline', 'nl']: - self.add_file_handler_if_needed(self._logger) - return getattr(self._logger, name) - else: - return AbstractFancyLogger.__getattribute__(self, name) - - @staticmethod - def log_newline(self, num_lines=1): - # Switch handler, output a blank line - if hasattr(self, 'main_file_handler') and hasattr(self, 'blank_file_handler'): - self.removeHandler(self.main_file_handler) - self.addHandler(self.blank_file_handler) - self.removeHandler(self.main_streaming_handler) - self.addHandler(self.blank_streaming_handler) - # Print the new lines - for i in range(num_lines): - self.info('') - # Switch back - if hasattr(self, 'main_file_handler') and hasattr(self, 'blank_file_handler'): - self.removeHandler(self.blank_file_handler) - self.addHandler(self.main_file_handler) - self.removeHandler(self.blank_streaming_handler) - self.addHandler(self.main_streaming_handler) - - def add_file_handler_if_needed(self, logger): - if not (hasattr(logger, 'main_file_handler') and hasattr(logger, 'blank_file_handler')) \ - and self.log_path: - # Create a file handler - self.create_logs_folder(self.log_path) - main_file_handler = logging.FileHandler(self.log_path) - main_file_handler.setLevel(self.log_level) - main_file_handler.setFormatter(logging.Formatter(fmt=self.log_fmt, - datefmt=self.log_date_fmt)) - # Create a "blank line" file handler - blank_file_handler = logging.FileHandler(self.log_path) - blank_file_handler.setLevel(self.log_level) - blank_file_handler.setFormatter(logging.Formatter(fmt='')) - # Add file handlers - logger.addHandler(main_file_handler) - logger.main_file_handler = main_file_handler - logger.blank_file_handler = blank_file_handler - return logger - - def create_logger(self, logger_name: str): - # Create a logger, with the previously-defined handlers - logger = logging.getLogger(logger_name) - logger.handlers = [] - logger.setLevel(self.log_level) - logger = self.add_file_handler_if_needed(logger) - # Create a streaming handler - main_streaming_handler = logging.StreamHandler() - main_streaming_handler.setLevel(self.log_level) - main_streaming_handler.setFormatter(logging.Formatter(fmt=self.log_fmt, - datefmt=self.log_date_fmt)) - # Create a "blank line" streaming handler - blank_streaming_handler = logging.StreamHandler() - blank_streaming_handler.setLevel(self.log_level) - blank_streaming_handler.setFormatter(logging.Formatter(fmt='')) - # Add streaming handlers - logger.addHandler(main_streaming_handler) - logger.propagate = False - logger.main_streaming_handler = main_streaming_handler - logger.blank_streaming_handler = blank_streaming_handler - # Create the new line method - logger.newline = types.MethodType(self.log_newline, logger) - logger.nl = logger.newline - return logger - - @staticmethod - def create_logs_folder(log_path: str): - log_path = os.path.abspath(log_path).split(os.sep) - log_dir = (os.sep.join(log_path[:-1])) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - - @classmethod - def setup_logger(cls, log_path: str, debug: bool = False, clear_log: bool = False) -> None: - """ Sets-up the basic_logger - - Args: - log_path (str): The path where the log file will be saved - debug (bool): Whether to print debug messages or not - clear_log (bool): Whether to empty the log file or not - """ - cls.log_path = os.path.abspath(log_path) - if clear_log: - open(cls.log_path, 'w').close() - cls.log_level = logging.INFO if debug is not True else logging.DEBUG - fancy_log_logger.info(f"Logger is set. Log file path: {cls.log_path}") - - -fancy_log_logger = ColorizedLogger(logger_name='FancyLogger', color='white') diff --git a/Starter/starter/main.py b/Starter/starter/main.py index c1e8bd4..a0b1d3c 100644 --- a/Starter/starter/main.py +++ b/Starter/starter/main.py @@ -1,14 +1,14 @@ import traceback import argparse -from starter import Configuration, ColorizedLogger, timeit, profileit, \ - DropboxCloudstore, MySqlDatastore, GmailEmailer +from starter import Configuration, ColorLogger, timeit, profileit, \ + DropboxCloudManager, HighMySQL, GmailPyEmailSender -basic_logger = ColorizedLogger(logger_name='Main', color='yellow') -fancy_logger = ColorizedLogger(logger_name='FancyMain', - color='blue', - on_color='on_red', - attrs=['underline', 'reverse', 'bold']) +basic_logger = ColorLogger(logger_name='Main', color='yellow') +fancy_logger = ColorLogger(logger_name='FancyMain', + color='blue', + on_color='on_red', + attrs=['underline', 'reverse', 'bold']) def get_args() -> argparse.Namespace: @@ -18,7 +18,7 @@ def get_args() -> argparse.Namespace: argparse.Namespace: """ parser = argparse.ArgumentParser( - description='A template for python projects.', + description='A starter template for Python packages.', add_help=False) # Required Args required_args = parser.add_argument_group('Required Arguments') @@ -53,7 +53,7 @@ def main(): # Initializing args = get_args() - ColorizedLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=True) + ColorLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=True) # Load the configuration # configuration = Configuration(config_src=args.config_file, # config_schema_path='yml_schema_strict.json') @@ -78,21 +78,21 @@ def main(): basic_logger.info( "Lastly, you can use profileit either as a function Wrapper or a ContextManager:") with profileit(): - # CloudStore - cloud_conf = configuration.get_config('cloudstore')[0] + # DropboxCloudManager + cloud_conf = configuration.get_config('cloud-filemanager')[0] if cloud_conf['type'] == 'dropbox' and cloud_conf['config']['api_key'] != 'DROPBOX_API_KEY': - dropbox_obj = DropboxCloudstore(config=cloud_conf['config']) + dropbox_obj = DropboxCloudManager(config=cloud_conf['config']) basic_logger.info(f"Base folder contents in dropbox:\n{dropbox_obj.ls().keys()}") - # MySqlDatastore - cloud_conf = configuration.get_config('datastore')[0] + # HighMySQL + cloud_conf = configuration.get_config('high-sql')[0] if cloud_conf['type'] == 'mysql' and cloud_conf['config']['username'] != 'MYSQL_USERNAME': - mysql_obj = MySqlDatastore(config=cloud_conf['config']) + mysql_obj = HighMySQL(config=cloud_conf['config']) basic_logger.info(f"List of tables in DB:\n{mysql_obj.show_tables()}") - # GmailEmailer - cloud_conf = configuration.get_config('emailer')[0] - if cloud_conf['type'] == 'gmail' and cloud_conf['config']['api_key'] != 'GMAIL_API_KEY': + # GmailPyEmailSender + mail_conf = configuration.get_config('pyemail-sender')[0] + if cloud_conf['type'] == 'gmail' and mail_conf['config']['api_key'] != 'GMAIL_API_KEY': basic_logger.info(f"Sending Sample Email to the email address set..") - gmail_obj = GmailEmailer(config=cloud_conf['config']) + gmail_obj = GmailEmailer(config=mail_conf['config']) gmail_obj.send_email(subject='starter', to=[gmail_obj.email_address], text='GmailEmailer works!') diff --git a/Starter/starter/profiling_funcs/__init__.py b/Starter/starter/profiling_funcs/__init__.py deleted file mode 100644 index 0ecb228..0000000 --- a/Starter/starter/profiling_funcs/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Profileit sub-package of Starter.""" - -from .profileit import profileit - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" diff --git a/Starter/starter/profiling_funcs/profileit.py b/Starter/starter/profiling_funcs/profileit.py deleted file mode 100644 index f207e3e..0000000 --- a/Starter/starter/profiling_funcs/profileit.py +++ /dev/null @@ -1,114 +0,0 @@ -from contextlib import ContextDecorator -from typing import Callable, IO, List -from io import StringIO -from functools import wraps -import cProfile -import pstats - -from starter import ColorizedLogger - -profile_logger = ColorizedLogger('Profileit', 'white') - - -class profileit(ContextDecorator): - custom_print: str - profiler: cProfile.Profile - stream: StringIO - sort_by: str - keep_only_these: List - fraction: float - skip: bool - profiler_output: str - file: IO - - def __init__(self, **kwargs): - """Decorator/ContextManager for profiling functions and code blocks - - Args: - custom_print: Custom print string. When used as decorator it can also be formatted using - `func_name`, `args`, and {0}, {1}, .. to reference the function's - first, second, ... argument. - sort_by: pstats sorting column - profiler_output: Filepath where to save the profiling results (.o file) - keep_only_these: List of strings - grep on the profiling output and print only lines - containing any of these strings - fraction: pstats.print_stats() fraction argument - skip: If True, don't time this time. Suitable when inside loops - file: Write the timing output to a file too - """ - - self.profiler = cProfile.Profile() - self.stream = StringIO() - self.sort_by = 'stdname' - self.keep_only_these = [] - self.fraction = 1.0 - self.skip = False - self.__dict__.update(kwargs) - - def __call__(self, func: Callable): - """ This is called only when invoked as a decorator - - Args: - func: The method to wrap - """ - - @wraps(func) - def profiled(*args, **kwargs): - with self._recreate_cm(): - self.func_name = func.__name__ - self.args = args - self.kwargs = kwargs - self.all_args = (*args, *kwargs.values()) if kwargs != {} else args - return func(*args, **kwargs) - - return profiled - - def __enter__(self, *args, **kwargs): - if not self.skip: - self.profiler.enable() - return self - - def __exit__(self, type, value, traceback): - if self.skip: - return - - self.profiler.disable() - ps = pstats.Stats(self.profiler, stream=self.stream).sort_stats(self.sort_by) - ps.print_stats(self.fraction) - - # If used as a decorator - if hasattr(self, 'func_name'): - if not hasattr(self, 'custom_print'): - print_string = 'Func: {func_name!r} with args: {args!r} profiled:' - else: - print_string = self.custom_print - print_string = print_string.format(*self.args, func_name=self.func_name, - args=self.all_args, - **self.kwargs) - # If used as contextmanager - else: - if not hasattr(self, 'custom_print'): - print_string = 'Code block profiled:' - else: - print_string = self.custom_print - - # Get Profiling results - prof_res = self.stream.getvalue() - if len(self.keep_only_these) > 0: - # Keep only lines containing the specified words - prof_res_list = [line for line in prof_res.split('\n') - if any(keep_word in line for keep_word in self.keep_only_these)] - prof_res = '\n'.join(prof_res_list) - - # Print to file if requested - if hasattr(self, 'file'): - self.file.write(print_string) - self.file.write("\n%s" % prof_res) - - # Save profiler output to a file if requested - if hasattr(self, 'profiler_output'): - self.profiler.dump_stats(self.profiler_output) - - # Actual Print - profile_logger.info(print_string) - profile_logger.info("%s", prof_res) diff --git a/Starter/starter/timing_tools/__init__.py b/Starter/starter/timing_tools/__init__.py deleted file mode 100644 index 4e9898c..0000000 --- a/Starter/starter/timing_tools/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Timeit sub-package of Starter.""" - -from .timeit import timeit - -__author__ = "drkostas" -__email__ = "georgiou.kostas94@gmail.com" -__version__ = "0.1.0" - diff --git a/Starter/starter/timing_tools/timeit.py b/Starter/starter/timing_tools/timeit.py deleted file mode 100644 index a59e13e..0000000 --- a/Starter/starter/timing_tools/timeit.py +++ /dev/null @@ -1,79 +0,0 @@ -from contextlib import ContextDecorator -from typing import Callable, IO -from functools import wraps -from time import time - -from starter import ColorizedLogger - -time_logger = ColorizedLogger('Timeit', 'white') - - -class timeit(ContextDecorator): - custom_print: str - skip: bool - file: IO - - def __init__(self, **kwargs): - """Decorator/ContextManager for counting the execution times of functions and code blocks - - Args: - custom_print: Custom print string Use {duration} to reference the running time. - When used as decorator it can also be formatted using - `func_name`, `args`, and {0}, {1}, .. to reference the function's - first, second, ... argument. - skip: If True, don't time this time. Suitable when inside loops - file: Write the timing output to a file too - """ - - self.total = None - self.skip = False - self.internal_only = False - self.__dict__.update(kwargs) - - def __call__(self, func: Callable): - """ This is called only when invoked as a decorator - - Args: - func: The method to wrap - """ - - @wraps(func) - def timed(*args, **kwargs): - with self._recreate_cm(): - self.func_name = func.__name__ - self.args = args - self.kwargs = kwargs - self.all_args = (*args, *kwargs.values()) if kwargs != {} else args - return func(*args, **kwargs) - - return timed - - def __enter__(self, *args, **kwargs): - if not self.skip: - self.ts = time() - return self - - def __exit__(self, type, value, traceback): - if self.skip: - return - - self.te = time() - self.total = self.te - self.ts - if hasattr(self, 'func_name'): - if not hasattr(self, 'custom_print'): - print_string = 'Func: {func_name!r} with args: {args!r} took: {duration:2.5f} sec(s)' - else: - print_string = self.custom_print - time_logger.info(print_string.format(*self.args, func_name=self.func_name, - args=self.all_args, - duration=self.total, - **self.kwargs)) - else: - if not hasattr(self, 'custom_print'): - print_string = 'Code block took: {duration:2.5f} sec(s)' - else: - print_string = self.custom_print - if hasattr(self, 'file'): - self.file.write(print_string.format(duration=self.total)) - if not self.internal_only: - time_logger.info(print_string.format(duration=self.total)) diff --git a/Starter/tests/test_configuration.py b/Starter/tests/test_configuration.py deleted file mode 100644 index 5447a74..0000000 --- a/Starter/tests/test_configuration.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -"""Tests for `configuration` sub-package.""" -# pylint: disable=redefined-outer-name - -import unittest -from jsonschema.exceptions import ValidationError -from typing import Dict -import logging -import os - -from starter import Configuration, validate_json_schema - -logger = logging.getLogger('TestConfiguration') - - -class TestConfiguration(unittest.TestCase): - - def test_validation_library(self): - """ Sanity Check unittest""" - configuration_schema = Configuration.load_configuration_schema( - os.path.join(self.test_data_path, 'simplest_yml_schema.json')) - wrong_confs = [ - {"subproperty1": [123, 234], - "subproperty2": 1}, # p1 is string - - {"subproperty1": "10", - "subproperty2": 3}, # p2 is either 1 or 2 - - {"subproperty2": 1}, # p1 is required - - {"subproperty1": "10", - "subproperty2": 1, - "subproperty3": {}}, # p4 is required in p3 - - {"subproperty1": "10", - "subproperty2": 1, - "subproperty3": {"subproperty4": 15}} # p4 is either 1 or 2 - ] - for wrong_conf in wrong_confs: - with self.assertRaises(ValidationError): - # try: - validate_json_schema(wrong_conf, configuration_schema) - # except Exception as e: - # print(e) - logger.info('YMLs failed to validate successfully.') - - def test_schema_validation(self): - try: - logger.info('Loading the correct Configuration..') - Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - except ValidationError as e: - logger.error('Error validating the correct yml: %s', e) - self.fail('Error validating the correct yml') - except Exception as e: - raise e - else: - logger.info('First yml validated successfully.') - - with self.assertRaises(ValidationError): - logger.info('Loading the wrong Configuration..') - Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_wrong.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - logger.info('Second yml failed to validate successfully.') - - def test_to_json(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - - expected_json = {'datastore': 'test', - 'cloudstore': [{ - 'subproperty1': 1, - 'subproperty2': [123, 234] - }], - 'tag': 'test_tag'} - # Compare - logger.info('Comparing the results..') - self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(configuration.to_json())) - - def test_to_yaml(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - # Modify and export yml - logger.info('Changed the host and the api_key..') - configuration.config['cloudstore'][0]['subproperty1'] = 999 - configuration.tag = 'CHANGED VALUE' - logger.info('Exporting to yaml..') - configuration.to_yaml(os.path.join(self.test_data_path, - 'actual_output_to_yaml.yml')) - # Load the modified yml - logger.info('Loading the exported yaml..') - modified_configuration = Configuration( - config_src=os.path.join(self.test_data_path, 'actual_output_to_yaml.yml')) - # Compare - logger.info('Comparing the results..') - expected_json = {'datastore': 'test', - 'cloudstore': [{ - 'subproperty1': 999, - 'subproperty2': [123, 234] - }], - 'tag': 'CHANGED VALUE'} - self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(modified_configuration.to_json())) - - def test_get_config(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - cloudstore_config = configuration.get_config(config_name='cloudstore') - expected_json = [{ - 'subproperty1': 1, - 'subproperty2': [123, 234] - }] - # Compare - logger.info('Comparing the results..') - self.assertListEqual(expected_json, cloudstore_config) - - @classmethod - def _sort_dict(cls, dictionary: Dict) -> Dict: - return {k: cls._sort_dict(v) if isinstance(v, dict) else v - for k, v in sorted(dictionary.items())} - - @staticmethod - def _setup_log() -> None: - # noinspection PyArgumentList - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - handlers=[logging.StreamHandler() - ] - ) - - def setUp(self) -> None: - pass - - def tearDown(self) -> None: - pass - - @classmethod - def setUpClass(cls): - cls._setup_log() - cls.tests_abs_path = os.path.abspath(os.path.dirname(__file__)) - cls.test_data_path: str = os.path.join(cls.tests_abs_path, 'test_data', 'test_configuration') - - @classmethod - def tearDownClass(cls): - pass - - -if __name__ == '__main__': - unittest.main() diff --git a/Starter/tests/test_data/test_configuration/actual_output_to_yaml.yml b/Starter/tests/test_data/test_configuration/actual_output_to_yaml.yml deleted file mode 100644 index 32212e6..0000000 --- a/Starter/tests/test_data/test_configuration/actual_output_to_yaml.yml +++ /dev/null @@ -1,7 +0,0 @@ -cloudstore: -- subproperty1: 999 - subproperty2: - - 123 - - 234 -datastore: test -tag: CHANGED VALUE diff --git a/Starter/tests/test_data/test_configuration/minimal_conf_correct.yml b/Starter/tests/test_data/test_configuration/minimal_conf_correct.yml deleted file mode 100644 index 125c031..0000000 --- a/Starter/tests/test_data/test_configuration/minimal_conf_correct.yml +++ /dev/null @@ -1,7 +0,0 @@ -datastore: test -cloudstore: - - subproperty1: 1 - subproperty2: - - 123 - - 234 -tag: test_tag \ No newline at end of file diff --git a/Starter/tests/test_data/test_configuration/minimal_conf_wrong.yml b/Starter/tests/test_data/test_configuration/minimal_conf_wrong.yml deleted file mode 100644 index 194b5ab..0000000 --- a/Starter/tests/test_data/test_configuration/minimal_conf_wrong.yml +++ /dev/null @@ -1,7 +0,0 @@ -datastore: test -cloudstore: - - subproperty1: 10 - subproperty2: - - 123 - - 234 -tag: test_tag \ No newline at end of file diff --git a/Starter/tests/test_data/test_configuration/minimal_yml_schema.json b/Starter/tests/test_data/test_configuration/minimal_yml_schema.json deleted file mode 100644 index b3bfb0d..0000000 --- a/Starter/tests/test_data/test_configuration/minimal_yml_schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "datastore": { - "type": "string" - }, - "tag": { - "type": "string" - }, - "cloudstore": { - "$ref": "#/definitions/cloudstore" - } - }, - "required": [ - "tag" - ], - "definitions": { - "cloudstore": { - "type": "array", - "items": { - "type": "object", - "required": [ - "subproperty1", - "subproperty2" - ], - "properties": { - "subproperty1": { - "type": "number", - "enum": [ - 1, - 2 - ] - }, - "subproperty2": { - "type": "array" - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/Starter/tests/test_data/test_configuration/simplest_yml_schema.json b/Starter/tests/test_data/test_configuration/simplest_yml_schema.json deleted file mode 100644 index d54bbbd..0000000 --- a/Starter/tests/test_data/test_configuration/simplest_yml_schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "subproperty1": { - "type": "string" - }, - "subproperty2": { - "type": "number", - "enum": [ - 1, - 2 - ] - }, - "subproperty3": { - "$ref": "#/definitions/subproperty3" - } - }, - "required": [ - "subproperty1" - ], - "definitions": { - "subproperty3": { - "type": "object", - "items": { - "type": "object" - }, - "additionalProperties": false, - "required": [ - "subproperty4" - ], - "properties": { - "subproperty4": { - "type": "number", - "enum": [ - 1, - 2 - ] - } - } - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/Starter/tests/test_data/test_configuration/template_conf.yml b/Starter/tests/test_data/test_configuration/template_conf.yml deleted file mode 100644 index 27ef9a9..0000000 --- a/Starter/tests/test_data/test_configuration/template_conf.yml +++ /dev/null @@ -1,13 +0,0 @@ -tag: production -cloudstore: - - config: - api_key: apiqwerty - type: dropbox -datastore: - - config: - hostname: host123 - username: user1 - password: pass2 - db_name: db3 - port: 3306 - type: mysql \ No newline at end of file diff --git a/TODO.md b/TODO.md index f1f3e4f..a47b7ef 100644 --- a/TODO.md +++ b/TODO.md @@ -19,3 +19,14 @@ See the [issues](https://github.com/drkostas/starter/issues) too. - [X] Create `profileit` decorator/contextmanager - [X] Modify timeit to skip more parts of the code when skip=True - [X] Add db, cloudstore packages from the old template project +- [X] Upload packages to pypi and import them +- [ ] Change Makefile to echo the name of env after install +- [ ] Put quote in the env file values. Do the same for the packages +- [ ] Change the Makefile based on the addition you made on other projects +- [ ] On MySQL datastore, add option to set primary key +- [ ] On MySQL datastore, add options for group by and join +- [ ] Add skip option to logger +- [ ] Make top-level inits more abstract (for cookiecutter) +- [ ] Configurable python_requires and classifiers in setup.py +- [ ] Change the readme and makefile to use env=venv or conda +- [ ] Change the acknowledgements diff --git a/{{cookiecutter.package_title_name}}/TODO.md b/{{cookiecutter.package_title_name}}/TODO.md index 7526b1c..40671a3 100644 --- a/{{cookiecutter.package_title_name}}/TODO.md +++ b/{{cookiecutter.package_title_name}}/TODO.md @@ -1,5 +1,3 @@ # TODO See the [issues](https://github.com/{{cookiecutter.author}}/{{cookiecutter.package_name}}/issues) too. -- [X] Create Tests -- [X] Create Readme -- [ ] Stop Global Warming +- [X] Initialize template repository diff --git a/{{cookiecutter.package_title_name}}/confs/template_conf.yml b/{{cookiecutter.package_title_name}}/confs/template_conf.yml index fbb0910..0c43ac7 100644 --- a/{{cookiecutter.package_title_name}}/confs/template_conf.yml +++ b/{{cookiecutter.package_title_name}}/confs/template_conf.yml @@ -1,9 +1,9 @@ tag: template -cloudstore: +cloud-filemanager: - config: api_key: yourapikey type: dropbox -datastore: +high-sql: - config: hostname: hostname username: username @@ -11,7 +11,7 @@ datastore: db_name: mydb port: 3306 type: mysql -emailer: +pyemail-sender: - config: email_address: foo@gmail.com api_key: 123 diff --git a/{{cookiecutter.package_title_name}}/confs/template_conf_with_env_variables.yml b/{{cookiecutter.package_title_name}}/confs/template_conf_with_env_variables.yml index df2c158..a45f0f2 100644 --- a/{{cookiecutter.package_title_name}}/confs/template_conf_with_env_variables.yml +++ b/{{cookiecutter.package_title_name}}/confs/template_conf_with_env_variables.yml @@ -1,9 +1,9 @@ tag: template -cloudstore: +cloud-filemanager: - config: api_key: !ENV ${DROPBOX_API_KEY} type: dropbox -datastore: +high-sql: - config: hostname: !ENV ${MYSQL_HOST} username: !ENV ${MYSQL_USERNAME} @@ -11,7 +11,7 @@ datastore: db_name: !ENV ${MYSQL_DB_NAME} port: 3306 type: mysql -emailer: +pyemail-sender: - config: email_address: !ENV ${EMAIL_ADDRESS} api_key: !ENV ${GMAIL_API_KEY} diff --git a/{{cookiecutter.package_title_name}}/requirements.txt b/{{cookiecutter.package_title_name}}/requirements.txt index a3ce04e..d403818 100644 --- a/{{cookiecutter.package_title_name}}/requirements.txt +++ b/{{cookiecutter.package_title_name}}/requirements.txt @@ -1,9 +1,8 @@ -dropbox~=11.10.0 -gmail~=0.6.3 -jsonschema~=3.2.0 -mysql-connector-python~=8.0.19 -mysql-connector~=2.2.9 -PyYAML~=5.4.1 setuptools~=52.0.0 -termcolor~=1.1.0 typer~=0.3.2 +yaml-config-wrapper>=1.0.4 +termcolor-logger>=1.0.3 +pyemail-sender>=1.0.1 +high-sql>=1.0.2 +cloud-filemanager>=1.0.1 +bench-utils>=1.0.3 \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/setup.py b/{{cookiecutter.package_title_name}}/setup.py index de9f983..49636ca 100644 --- a/{{cookiecutter.package_title_name}}/setup.py +++ b/{{cookiecutter.package_title_name}}/setup.py @@ -42,7 +42,9 @@ def run(self): '{{cookiecutter.package_name}}_main = {{cookiecutter.package_name}}.main:main' ] -data_files = ['{{cookiecutter.package_name}}/configuration/yml_schema.json'] +data_files = [ + # '{{cookiecutter.package_name}}/a-file' +] setup( author="{{cookiecutter.author}}", diff --git a/{{cookiecutter.package_title_name}}/tests/test_configuration.py b/{{cookiecutter.package_title_name}}/tests/test_configuration.py deleted file mode 100644 index f192545..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_configuration.py +++ /dev/null @@ -1,158 +0,0 @@ -#!/usr/bin/env python - -"""Tests for `configuration` sub-package.""" -# pylint: disable=redefined-outer-name - -import unittest -from jsonschema.exceptions import ValidationError -from typing import Dict -import logging -import os - -from {{cookiecutter.package_name}} import Configuration, validate_json_schema - -logger = logging.getLogger('TestConfiguration') - - -class TestConfiguration(unittest.TestCase): - - def test_validation_library(self): - """ Sanity Check unittest""" - configuration_schema = Configuration.load_configuration_schema( - os.path.join(self.test_data_path, 'simplest_yml_schema.json')) - wrong_confs = [ - {"subproperty1": [123, 234], - "subproperty2": 1}, # p1 is string - - {"subproperty1": "10", - "subproperty2": 3}, # p2 is either 1 or 2 - - {"subproperty2": 1}, # p1 is required - - {"subproperty1": "10", - "subproperty2": 1, - "subproperty3": {}}, # p4 is required in p3 - - {"subproperty1": "10", - "subproperty2": 1, - "subproperty3": {"subproperty4": 15}} # p4 is either 1 or 2 - ] - for wrong_conf in wrong_confs: - with self.assertRaises(ValidationError): - # try: - validate_json_schema(wrong_conf, configuration_schema) - # except Exception as e: - # print(e) - logger.info('YMLs failed to validate successfully.') - - def test_schema_validation(self): - try: - logger.info('Loading the correct Configuration..') - Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - except ValidationError as e: - logger.error('Error validating the correct yml: %s', e) - self.fail('Error validating the correct yml') - except Exception as e: - raise e - else: - logger.info('First yml validated successfully.') - - with self.assertRaises(ValidationError): - logger.info('Loading the wrong Configuration..') - Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_wrong.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - logger.info('Second yml failed to validate successfully.') - - def test_to_json(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - - expected_json = {'datastore': 'test', - 'cloudstore': [{ - 'subproperty1': 1, - 'subproperty2': [123, 234] - }], - 'tag': 'test_tag'} - # Compare - logger.info('Comparing the results..') - self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(configuration.to_json())) - - def test_to_yaml(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - # Modify and export yml - logger.info('Changed the host and the api_key..') - configuration.config['cloudstore'][0]['subproperty1'] = 999 - configuration.tag = 'CHANGED VALUE' - logger.info('Exporting to yaml..') - configuration.to_yaml(os.path.join(self.test_data_path, - 'actual_output_to_yaml.yml')) - # Load the modified yml - logger.info('Loading the exported yaml..') - modified_configuration = Configuration( - config_src=os.path.join(self.test_data_path, 'actual_output_to_yaml.yml')) - # Compare - logger.info('Comparing the results..') - expected_json = {'datastore': 'test', - 'cloudstore': [{ - 'subproperty1': 999, - 'subproperty2': [123, 234] - }], - 'tag': 'CHANGED VALUE'} - self.assertDictEqual(self._sort_dict(expected_json), self._sort_dict(modified_configuration.to_json())) - - def test_get_config(self): - logger.info('Loading Configuration..') - configuration = Configuration(config_src=os.path.join(self.test_data_path, 'minimal_conf_correct.yml'), - config_schema_path=os.path.join(self.test_data_path, - 'minimal_yml_schema.json')) - cloudstore_config = configuration.get_config(config_name='cloudstore') - expected_json = [{ - 'subproperty1': 1, - 'subproperty2': [123, 234] - }] - # Compare - logger.info('Comparing the results..') - self.assertListEqual(expected_json, cloudstore_config) - - @classmethod - def _sort_dict(cls, dictionary: Dict) -> Dict: - return {k: cls._sort_dict(v) if isinstance(v, dict) else v - for k, v in sorted(dictionary.items())} - - @staticmethod - def _setup_log() -> None: - # noinspection PyArgumentList - logging.basicConfig(level=logging.DEBUG, - format='%(asctime)s %(name)-12s %(levelname)-8s %(message)s', - datefmt='%Y-%m-%d %H:%M:%S', - handlers=[logging.StreamHandler() - ] - ) - - def setUp(self) -> None: - pass - - def tearDown(self) -> None: - pass - - @classmethod - def setUpClass(cls): - cls._setup_log() - cls.tests_abs_path = os.path.abspath(os.path.dirname(__file__)) - cls.test_data_path: str = os.path.join(cls.tests_abs_path, 'test_data', 'test_configuration') - - @classmethod - def tearDownClass(cls): - pass - - -if __name__ == '__main__': - unittest.main() diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/actual_output_to_yaml.yml b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/actual_output_to_yaml.yml deleted file mode 100644 index 32212e6..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/actual_output_to_yaml.yml +++ /dev/null @@ -1,7 +0,0 @@ -cloudstore: -- subproperty1: 999 - subproperty2: - - 123 - - 234 -datastore: test -tag: CHANGED VALUE diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_correct.yml b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_correct.yml deleted file mode 100644 index 125c031..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_correct.yml +++ /dev/null @@ -1,7 +0,0 @@ -datastore: test -cloudstore: - - subproperty1: 1 - subproperty2: - - 123 - - 234 -tag: test_tag \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_wrong.yml b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_wrong.yml deleted file mode 100644 index 194b5ab..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_conf_wrong.yml +++ /dev/null @@ -1,7 +0,0 @@ -datastore: test -cloudstore: - - subproperty1: 10 - subproperty2: - - 123 - - 234 -tag: test_tag \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_yml_schema.json b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_yml_schema.json deleted file mode 100644 index b3bfb0d..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/minimal_yml_schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "datastore": { - "type": "string" - }, - "tag": { - "type": "string" - }, - "cloudstore": { - "$ref": "#/definitions/cloudstore" - } - }, - "required": [ - "tag" - ], - "definitions": { - "cloudstore": { - "type": "array", - "items": { - "type": "object", - "required": [ - "subproperty1", - "subproperty2" - ], - "properties": { - "subproperty1": { - "type": "number", - "enum": [ - 1, - 2 - ] - }, - "subproperty2": { - "type": "array" - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/simplest_yml_schema.json b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/simplest_yml_schema.json deleted file mode 100644 index d54bbbd..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/simplest_yml_schema.json +++ /dev/null @@ -1,44 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "subproperty1": { - "type": "string" - }, - "subproperty2": { - "type": "number", - "enum": [ - 1, - 2 - ] - }, - "subproperty3": { - "$ref": "#/definitions/subproperty3" - } - }, - "required": [ - "subproperty1" - ], - "definitions": { - "subproperty3": { - "type": "object", - "items": { - "type": "object" - }, - "additionalProperties": false, - "required": [ - "subproperty4" - ], - "properties": { - "subproperty4": { - "type": "number", - "enum": [ - 1, - 2 - ] - } - } - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/template_conf.yml b/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/template_conf.yml deleted file mode 100644 index 27ef9a9..0000000 --- a/{{cookiecutter.package_title_name}}/tests/test_data/test_configuration/template_conf.yml +++ /dev/null @@ -1,13 +0,0 @@ -tag: production -cloudstore: - - config: - api_key: apiqwerty - type: dropbox -datastore: - - config: - hostname: host123 - username: user1 - password: pass2 - db_name: db3 - port: 3306 - type: mysql \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/__init__.py index b8fe201..c537bd8 100644 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/__init__.py +++ b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/__init__.py @@ -1,12 +1,11 @@ """Top-level package for {{cookiecutter.package_title_name}}.""" -from {{cookiecutter.package_name}}.fancy_logger import ColorizedLogger -from {{cookiecutter.package_name}}.timing_tools import timeit -from {{cookiecutter.package_name}}.profiling_funcs import profileit -from {{cookiecutter.package_name}}.configuration import Configuration, validate_json_schema -from {{cookiecutter.package_name}}.cloudstore import DropboxCloudstore -from {{cookiecutter.package_name}}.datastore import MySqlDatastore -from {{cookiecutter.package_name}}.emailer import GmailEmailer +from termcolor_logger import ColorLogger +from bench_utils import timeit, profileit +from yaml_config_wrapper import Configuration, validate_json_schema +from cloud_filemanager import DropboxCloudManager +from high_sql import HighMySQL +from pyemail_sender import GmailPyEmailSender __author__ = "{{cookiecutter.author}}" __email__ = "{{cookiecutter.author_email}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/__init__.py deleted file mode 100644 index 5dbe49b..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cloudstore sub-package of {{cookiecutter.package_title_name}}.""" - -from .dropbox_cloudstore import DropboxCloudstore - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/abstract_cloudstore.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/abstract_cloudstore.py deleted file mode 100644 index d626230..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/abstract_cloudstore.py +++ /dev/null @@ -1,72 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractCloudstore(ABC): - __slots__ = ('_handler',) - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """ - Tha basic constructor. Creates a new instance of Cloudstore using the specified credentials - """ - - pass - - @staticmethod - @abstractmethod - def get_handler(*args, **kwargs): - """ - Returns a Cloudstore handler. - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def upload_file(self, *args, **kwargs): - """ - Uploads a file to the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def download_file(self, *args, **kwargs): - """ - Downloads a file from the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def delete_file(self, *args, **kwargs): - """ - Deletes a file from the Cloudstore - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def ls(self, *args, **kwargs): - """ - List the files and folders in the Cloudstore - :param args: - :param kwargs: - :return: - """ - pass diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/dropbox_cloudstore.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/dropbox_cloudstore.py deleted file mode 100644 index 3b49939..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/cloudstore/dropbox_cloudstore.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import Dict, Union -from dropbox import Dropbox, files, exceptions - -from .abstract_cloudstore import AbstractCloudstore -from {{cookiecutter.package_name}} import ColorizedLogger - -logger = ColorizedLogger('DropboxCloudstore') - - -class DropboxCloudstore(AbstractCloudstore): - __slots__ = '_handler' - - _handler: Dropbox - - def __init__(self, config: Dict) -> None: - """ - The basic constructor. Creates a new instance of Cloudstore using the specified credentials - - :param config: - """ - - self._handler = self.get_handler(api_key=config['api_key']) - super().__init__() - - @staticmethod - def get_handler(api_key: str) -> Dropbox: - """ - Returns a Cloudstore handler. - - :param api_key: - :return: - """ - - dbx = Dropbox(api_key) - return dbx - - def upload_file(self, file_bytes: bytes, upload_path: str, write_mode: str = 'overwrite') -> None: - """ - Uploads a file to the Cloudstore - - :param file_bytes: - :param upload_path: - :param write_mode: - :return: - """ - - # TODO: Add option to support FileStream, StringIO and FilePath - try: - logger.debug("Uploading file to path: %s" % upload_path) - self._handler.files_upload(f=file_bytes, path=upload_path, - mode=files.WriteMode(write_mode)) - except exceptions.ApiError as err: - logger.error('API error: %s' % err) - - def download_file(self, frompath: str, tofile: str = None) -> Union[bytes, None]: - """ - Downloads a file from the Cloudstore - - :param frompath: - :param tofile: - :return: - """ - - try: - if tofile is not None: - logger.debug("Downloading file from path: %s to path %s" % (frompath, tofile)) - self._handler.files_download_to_file(download_path=tofile, path=frompath) - else: - logger.debug("Downloading file from path: %s to variable" % frompath) - md, res = self._handler.files_download(path=frompath) - data = res.content # The bytes of the file - return data - except exceptions.HttpError as err: - logger.error('HTTP error %s' % err) - return None - - def delete_file(self, file_path: str) -> None: - """ - Deletes a file from the Cloudstore - - :param file_path: - :return: - """ - - try: - logger.debug("Deleting file from path: %s" % file_path) - self._handler.files_delete_v2(path=file_path) - except exceptions.ApiError as err: - logger.error('API error %s' % err) - - def ls(self, path: str = '') -> Dict: - """ - List the files and folders in the Cloudstore - - :param path: - :return: - """ - try: - files_list = self._handler.files_list_folder(path=path) - files_dict = {} - for entry in files_list.entries: - files_dict[entry.name] = entry - return files_dict - except exceptions.ApiError as err: - logger.error('Folder listing failed for %s -- assumed empty: %s' % (path, err)) - return {} diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/__init__.py deleted file mode 100644 index e43c897..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Configuration sub-package of {{cookiecutter.package_title_name}}.""" - -from .configuration import Configuration, validate_json_schema - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/configuration.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/configuration.py deleted file mode 100644 index 8b40ee0..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/configuration.py +++ /dev/null @@ -1,177 +0,0 @@ -import os -from typing import Dict, List, Tuple, Union -import json -import _io -from io import StringIO, TextIOWrapper -import re -import yaml -from jsonschema import validate as validate_json_schema - -from {{cookiecutter.package_name}} import ColorizedLogger - -logger = ColorizedLogger('Config', 'white') - - -class Configuration: - __slots__ = ('config', 'config_path', 'config_keys', 'tag') - - config: Dict - config_path: str - tag: str - config_keys: List - env_variable_tag: str = '!ENV' - env_variable_pattern: str = r'.*?\${(\w+)}.*?' # ${var} - - def __init__(self, config_src: Union[TextIOWrapper, StringIO, str], - config_schema_path: str = 'yml_schema.json'): - """ - The basic constructor. Creates a new instance of the Configuration class. - - Args: - config_src: The path, file or StringIO object of the configuration to load - config_schema_path: The path, file or StringIO object of the configuration validation file - """ - - # Load the predefined schema of the configuration - configuration_schema = self.load_configuration_schema(config_schema_path=config_schema_path) - # Load the configuration - self.config, self.config_path = self.load_yml(config_src=config_src, - env_tag=self.env_variable_tag, - env_pattern=self.env_variable_pattern) - # Validate the config - validate_json_schema(self.config, configuration_schema) - logger.debug("Schema Validation was Successful.") - # Set the config properties as instance attributes - self.tag = self.config['tag'] - self.config_keys = [key for key in self.config.keys() if key != 'tag'] - logger.info(f"Configuration file loaded successfully from path: {self.config_path}") - logger.info(f"Configuration Tag: {self.tag}") - - @staticmethod - def load_configuration_schema(config_schema_path: str) -> Dict: - """ - Loads the configuration schema file - - Args: - config_schema_path: The path of the config schema - - Returns: - configuration_schema: The loaded config schema - """ - - if config_schema_path[0] != os.sep: - config_schema_path = '/'.join( - [os.path.dirname(os.path.realpath(__file__)), config_schema_path]) - with open(config_schema_path) as f: - configuration_schema = json.load(f) - return configuration_schema - - @staticmethod - def load_yml(config_src: Union[TextIOWrapper, StringIO, str], env_tag: str, env_pattern: str) -> \ - Tuple[Dict, str]: - """ - Loads the configuration file - Args: - config_src: The path of the configuration - env_tag: The tag that distinguishes the env variables - env_pattern: The regex for finding the env variables - - Returns: - config, config_path - """ - pattern = re.compile(env_pattern) - loader = yaml.SafeLoader - loader.add_implicit_resolver(env_tag, pattern, None) - - def constructor_env_variables(loader, node): - """ - Extracts the environment variable from the node's value - :param yaml.Loader loader: the yaml loader - :param node: the current node in the yaml - :return: the parsed string that contains the value of the environment - variable - """ - value = loader.construct_scalar(node) - match = pattern.findall(value) # to find all env variables in line - if match: - full_value = value - for g in match: - full_value = full_value.replace( - {% raw %}f'${{{g}}}', os.environ.get(g, g){% endraw %} - ) - return full_value - return value - - loader.add_constructor(env_tag, constructor_env_variables) - - if isinstance(config_src, TextIOWrapper): - logger.debug("Loading yaml from TextIOWrapper") - config = yaml.load(config_src, Loader=loader) - config_path = os.path.abspath(config_src.name) - elif isinstance(config_src, StringIO): - logger.debug("Loading yaml from StringIO") - config = yaml.load(config_src, Loader=loader) - config_path = "StringIO" - elif isinstance(config_src, str): - config_path = os.path.abspath(config_src) - logger.debug("Loading yaml from path") - with open(config_path) as f: - config = yaml.load(f, Loader=loader) - else: - raise TypeError('Config file must be TextIOWrapper or path to a file') - return config, config_path - - def get_config(self, config_name) -> List: - """ - Returns the subconfig requested - - Args: - config_name: The name of the subconfig - - Returns: - sub_config: The sub_configs List - """ - - if config_name in self.config.keys(): - return self.config[config_name] - else: - raise ConfigurationError('Config property %s not set!' % config_name) - - def to_yml(self, fn: Union[str, _io.TextIOWrapper]) -> None: - """ - Writes the configuration to a stream. For example a file. - - Args: - fn: - - Returns: - """ - - self.config['tag'] = self.tag - if isinstance(fn, str): - with open(fn, 'w') as f: - yaml.dump(self.config, f, default_flow_style=False) - elif isinstance(fn, _io.TextIOWrapper): - yaml.dump(self.config, fn, default_flow_style=False) - else: - raise TypeError('Expected str or _io.TextIOWrapper not %s' % (type(fn))) - - to_yaml = to_yml - - def to_json(self) -> Dict: - """ - Returns the whole config file - - Returns: - - """ - return self.config - - # def __getitem__(self, item): - # return self.get_config(item) - - -class ConfigurationError(Exception): - def __init__(self, message): - # Call the base class constructor with the parameters it needs - super().__init__(message) diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema.json b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema.json deleted file mode 100644 index b3e6c14..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema.json +++ /dev/null @@ -1,17 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "title": "Python Configuration", - "description": "A json for python configuration in yml format", - "type": "object", - "properties": { - "tag": { - "type": "string" - } - }, - "required": [ - "tag" - ], - "definitions": { - }, - "additionalProperties": true -} \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema_strict.json b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema_strict.json deleted file mode 100644 index 0856d43..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/configuration/yml_schema_strict.json +++ /dev/null @@ -1,66 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-04/schema#", - "type": "object", - "properties": { - "tag": { - "type": "string" - }, - "example_db": { - "$ref": "#/definitions/example_db" - } - }, - "required": [ - "tag", - "example_db" - ], - "definitions": { - "example_db": { - "type": "array", - "items": { - "type": "object", - "required": [ - "type", - "properties" - ], - "properties": { - "type": { - "type": "string", - "enum": [ - "mysql", - "mongodb" - ] - }, - "properties": { - "type": "object", - "additionalProperties": false, - "required": [ - "hostname", - "username", - "password", - "db_name" - ], - "properties": { - "hostname": { - "type": "string" - }, - "username": { - "type": "string" - }, - "password": { - "type": "string" - }, - "db_name": { - "type": "string" - }, - "port": { - "type": "integer" - } - } - } - } - }, - "additionalProperties": false - } - }, - "additionalProperties": false -} \ No newline at end of file diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/__init__.py deleted file mode 100644 index 1fac648..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Cloudstore sub-package of {{cookiecutter.package_title_name}}.""" - -from .mysql_datastore import MySqlDatastore - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/abstract_datastore.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/abstract_datastore.py deleted file mode 100644 index bde2319..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/abstract_datastore.py +++ /dev/null @@ -1,59 +0,0 @@ -from abc import ABC, abstractmethod -from typing import List, Dict - - -class AbstractDatastore(ABC): - __slots__ = ('_connection', '_cursor') - - @abstractmethod - def __init__(self, config: Dict) -> None: - """ - Tha basic constructor. Creates a new instance of Datastore using the specified credentials - - :param config: - """ - - self._connection, self._cursor = self.get_connection(username=config['username'], - password=config['password'], - hostname=config['hostname'], - db_name=config['db_name'], - port=config['port']) - - @staticmethod - @abstractmethod - def get_connection(username: str, password: str, hostname: str, db_name: str, port: int): - pass - - @abstractmethod - def create_table(self, table: str, schema: str): - pass - - @abstractmethod - def drop_table(self, table: str) -> None: - pass - - @abstractmethod - def truncate_table(self, table: str) -> None: - pass - - @abstractmethod - def insert_into_table(self, table: str, data: dict) -> None: - pass - - @abstractmethod - def update_table(self, table: str, set_data: dict, where: str) -> None: - pass - - @abstractmethod - def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', - order_by: str = 'NULL', - asc_or_desc: str = 'ASC', limit: int = 1000) -> List: - pass - - @abstractmethod - def delete_from_table(self, table: str, where: str) -> None: - pass - - @abstractmethod - def show_tables(self, *args, **kwargs) -> List: - pass diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/mysql_datastore.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/mysql_datastore.py deleted file mode 100644 index 466582d..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/datastore/mysql_datastore.py +++ /dev/null @@ -1,192 +0,0 @@ -from typing import List, Tuple, Dict - -from mysql import connector as mysql_connector -import mysql.connector.cursor - -from .abstract_datastore import AbstractDatastore -from {{cookiecutter.package_name}} import ColorizedLogger - -logger = ColorizedLogger('MySqlDataStore') - - -class MySqlDatastore(AbstractDatastore): - __slots__ = ('_connection', '_cursor') - - _connection: mysql_connector.MySQLConnection - _cursor: mysql_connector.cursor.MySQLCursor - - def __init__(self, config: Dict) -> None: - """ - The basic constructor. Creates a new instance of Datastore using the specified credentials - - :param config: - """ - - super().__init__(config) - - @staticmethod - def get_connection(username: str, password: str, hostname: str, db_name: str, port: int = 3306) \ - -> Tuple[mysql_connector.MySQLConnection, mysql_connector.cursor.MySQLCursor]: - """ - Creates and returns a connection and a cursor/session to the MySQL DB - - :param username: - :param password: - :param hostname: - :param db_name: - :param port: - :return: - """ - - connection = mysql_connector.connect( - host=hostname, - user=username, - passwd=password, - database=db_name, - use_pure=True - ) - - cursor = connection.cursor() - return connection, cursor - - def create_table(self, table: str, schema: str) -> None: - """ - Creates a table using the specified schema - - :param self: - :param table: - :param schema: - :return: - """ - - query = "CREATE TABLE IF NOT EXISTS {table} ({schema})".format(table=table, schema=schema) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def drop_table(self, table: str) -> None: - """ - Drops the specified table if it exists - - :param self: - :param table: - :return: - """ - - query = "DROP TABLE IF EXISTS {table}".format(table=table) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def truncate_table(self, table: str) -> None: - """ - Truncates the specified table - - :param self: - :param table: - :return: - """ - - query = "TRUNCATE TABLE {table}".format(table=table) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def insert_into_table(self, table: str, data: dict) -> None: - """ - Inserts into the specified table a row based on a column_name: value dictionary - - :param self: - :param table: - :param data: - :return: - """ - - data_str = ", ".join( - list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), data.keys(), data.values()))) - - query = "INSERT INTO {table} SET {data}".format(table=table, data=data_str) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def update_table(self, table: str, set_data: dict, where: str) -> None: - """ - Updates the specified table using a column_name: value dictionary and a where statement - - :param self: - :param table: - :param set_data: - :param where: - :return: - """ - - set_data_str = ", ".join( - list(map(lambda key, val: "{key}='{val}'".format(key=str(key), val=str(val)), set_data.keys(), - set_data.values()))) - - query = "UPDATE {table} SET {data} WHERE {where}".format(table=table, data=set_data_str, where=where) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def select_from_table(self, table: str, columns: str = '*', where: str = 'TRUE', order_by: str = 'NULL', - asc_or_desc: str = 'ASC', limit: int = 1000) -> List: - """ - Selects from a specified table based on the given columns, where, ordering and limit - - :param self: - :param table: - :param columns: - :param where: - :param order_by: - :param asc_or_desc: - :param limit: - :return results: - """ - - query = "SELECT {columns} FROM {table} WHERE {where} ORDER BY {order_by} {asc_or_desc} LIMIT {limit}".format( - columns=columns, table=table, where=where, order_by=order_by, asc_or_desc=asc_or_desc, limit=limit) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - results = self._cursor.fetchall() - - return results - - def delete_from_table(self, table: str, where: str) -> None: - """ - Deletes data from the specified table based on a where statement - - :param self: - :param table: - :param where: - :return: - """ - - query = "DELETE FROM {table} WHERE {where}".format(table=table, where=where) - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - self._connection.commit() - - def show_tables(self) -> List: - """ - Show a list of the tables present in the db - :return: - """ - - query = 'SHOW TABLES' - logger.debug("Executing: %s" % query) - self._cursor.execute(query) - results = self._cursor.fetchall() - - return [result[0] for result in results] - - def __exit__(self) -> None: - """ - Flushes and closes the connection - - :return: - """ - - self._connection.commit() - self._cursor.close() diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/__init__.py deleted file mode 100644 index ee32174..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Emailer sub-package of {{cookiecutter.package_title_name}}.""" - -from .gmail_emailer import GmailEmailer - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/abstract_emailer.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/abstract_emailer.py deleted file mode 100644 index 17b85d7..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/abstract_emailer.py +++ /dev/null @@ -1,39 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractEmailer(ABC): - __slots__ = ('_handler',) - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """ - Tha basic constructor. Creates a new instance of EmailApp using the specified credentials - - """ - - pass - - @staticmethod - @abstractmethod - def get_handler(*args, **kwargs): - """ - Returns an EmailApp handler. - - :param args: - :param kwargs: - :return: - """ - - pass - - @abstractmethod - def send_email(self, *args, **kwargs): - """ - Sends an email with the specified arguments. - - :param args: - :param kwargs: - :return: - """ - - pass diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/gmail_emailer.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/gmail_emailer.py deleted file mode 100644 index 0c36741..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/emailer/gmail_emailer.py +++ /dev/null @@ -1,87 +0,0 @@ -from typing import List, Dict -import logging -from gmail import GMail, Message - -from .abstract_emailer import AbstractEmailer -from {{cookiecutter.package_name}} import ColorizedLogger - -logger = ColorizedLogger('GmailEmailer') - - -class GmailEmailer(AbstractEmailer): - __slots__ = ('_handler', 'email_address', 'test_mode') - - _handler: GMail - test_mode: bool - - def __init__(self, config: Dict, test_mode: bool = False) -> None: - """ - The basic constructor. Creates a new instance of EmailApp using the specified credentials - - :param config: - :param test_mode: - """ - - self.email_address = config['email_address'] - self._handler = self.get_handler(email_address=self.email_address, - api_key=config['api_key']) - self.test_mode = test_mode - super().__init__() - - @staticmethod - def get_handler(email_address: str, api_key: str) -> GMail: - """ - Returns an EmailApp handler. - - :param email_address: - :param api_key: - :return: - """ - - gmail_handler = GMail(username=email_address, password=api_key) - gmail_handler.connect() - return gmail_handler - - def is_connected(self) -> bool: - return self._handler.is_connected() - - def get_self_email(self): - return self.email_address - - def send_email(self, subject: str, to: List, cc: List = None, bcc: List = None, text: str = None, - html: str = None, - attachments: List = None, sender: str = None, reply_to: str = None) -> None: - """ - Sends an email with the specified arguments. - - :param subject: - :param to: - :param cc: - :param bcc: - :param text: - :param html: - :param attachments: - :param sender: - :param reply_to: - :return: - """ - - if self.test_mode: - to = self.email_address - cc = self.email_address if cc is not None else None - bcc = self.email_address if bcc is not None else None - - msg = Message(subject=subject, - to=",".join(to), - cc=",".join(cc) if cc is not None else None, - bcc=",".join(bcc) if cc is not None else None, - text=text, - html=html, - attachments=attachments, - sender=sender, - reply_to=reply_to) - logger.debug("Sending email with Message: %s" % msg) - self._handler.send(msg) - - def __exit__(self): - self._handler.close() diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/__init__.py deleted file mode 100644 index 8f8c35f..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""FancyLog sub-package of {{cookiecutter.package_title_name}}.""" - -from .colorized_logger import ColorizedLogger - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" - diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/abstract_fancy_logger.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/abstract_fancy_logger.py deleted file mode 100644 index 96eea1a..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/abstract_fancy_logger.py +++ /dev/null @@ -1,19 +0,0 @@ -from abc import ABC, abstractmethod - - -class AbstractFancyLogger(ABC): - """Abstract class of the FancyLog package""" - - @abstractmethod - def __init__(self, *args, **kwargs) -> None: - """The basic constructor. Creates a new instance of FancyLog using the - specified arguments - - Args: - *args: - **kwargs: - """ - - @abstractmethod - def create_logger(self, *args, **kwargs): - pass diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/colorized_logger.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/colorized_logger.py deleted file mode 100644 index b588075..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/fancy_logger/colorized_logger.py +++ /dev/null @@ -1,151 +0,0 @@ -import os -from typing import List, Union -import types -import logging -from termcolor import colored - -from .abstract_fancy_logger import AbstractFancyLogger - - -class ColorizedLogger(AbstractFancyLogger): - """ColorizedLogger class of the FancyLog package""" - - __slots__ = ('_logger', 'logger_name', '_color', '_on_color', '_attrs', - 'debug', 'info', 'warn', 'warning', 'error', 'exception', 'critical') - - log_fmt: str = '%(asctime)s %(name)-12s %(levelname)-8s %(message)s' - log_date_fmt: str = '%Y-%m-%d %H:%M:%S' - log_level: Union[int, str] = logging.INFO - _logger: logging.Logger - log_path: str = None - logger_name: str - _color: str - _on_color: str - _attrs: List - - def __init__(self, logger_name: str, - color: str = 'white', on_color: str = None, - attrs: List = None) -> None: - """ - Args: - logger_name (str): - color (str): - attrs (List): AnyOf('bold', 'dark', 'underline', 'blink', 'reverse', 'concealed') - """ - - self._color = color - self._on_color = on_color - self._attrs = attrs if attrs else ['bold'] - self.logger_name = logger_name - self._logger = self.create_logger(logger_name=logger_name) - super().__init__() - - def __getattr__(self, name: str): - """ - Args: - name (str): - """ - - def log_colored(log_text: str, *args, **kwargs): - color = self._color if 'color' not in kwargs else kwargs['color'] - on_color = self._on_color if 'on_color' not in kwargs else kwargs['on_color'] - attrs = self._attrs if 'attrs' not in kwargs else kwargs['attrs'] - colored_text = colored(log_text, color=color, on_color=on_color, attrs=attrs) - return getattr(self._logger, name)(colored_text, *args) - - if name in ['debug', 'info', 'warn', 'warning', - 'error', 'exception', 'critical']: - self.add_file_handler_if_needed(self._logger) - return log_colored - elif name in ['newline', 'nl']: - self.add_file_handler_if_needed(self._logger) - return getattr(self._logger, name) - else: - return AbstractFancyLogger.__getattribute__(self, name) - - @staticmethod - def log_newline(self, num_lines=1): - # Switch handler, output a blank line - if hasattr(self, 'main_file_handler') and hasattr(self, 'blank_file_handler'): - self.removeHandler(self.main_file_handler) - self.addHandler(self.blank_file_handler) - self.removeHandler(self.main_streaming_handler) - self.addHandler(self.blank_streaming_handler) - # Print the new lines - for i in range(num_lines): - self.info('') - # Switch back - if hasattr(self, 'main_file_handler') and hasattr(self, 'blank_file_handler'): - self.removeHandler(self.blank_file_handler) - self.addHandler(self.main_file_handler) - self.removeHandler(self.blank_streaming_handler) - self.addHandler(self.main_streaming_handler) - - def add_file_handler_if_needed(self, logger): - if not (hasattr(logger, 'main_file_handler') and hasattr(logger, 'blank_file_handler')) \ - and self.log_path: - # Create a file handler - self.create_logs_folder(self.log_path) - main_file_handler = logging.FileHandler(self.log_path) - main_file_handler.setLevel(self.log_level) - main_file_handler.setFormatter(logging.Formatter(fmt=self.log_fmt, - datefmt=self.log_date_fmt)) - # Create a "blank line" file handler - blank_file_handler = logging.FileHandler(self.log_path) - blank_file_handler.setLevel(self.log_level) - blank_file_handler.setFormatter(logging.Formatter(fmt='')) - # Add file handlers - logger.addHandler(main_file_handler) - logger.main_file_handler = main_file_handler - logger.blank_file_handler = blank_file_handler - return logger - - def create_logger(self, logger_name: str): - # Create a logger, with the previously-defined handlers - logger = logging.getLogger(logger_name) - logger.handlers = [] - logger.setLevel(self.log_level) - logger = self.add_file_handler_if_needed(logger) - # Create a streaming handler - main_streaming_handler = logging.StreamHandler() - main_streaming_handler.setLevel(self.log_level) - main_streaming_handler.setFormatter(logging.Formatter(fmt=self.log_fmt, - datefmt=self.log_date_fmt)) - # Create a "blank line" streaming handler - blank_streaming_handler = logging.StreamHandler() - blank_streaming_handler.setLevel(self.log_level) - blank_streaming_handler.setFormatter(logging.Formatter(fmt='')) - # Add streaming handlers - logger.addHandler(main_streaming_handler) - logger.propagate = False - logger.main_streaming_handler = main_streaming_handler - logger.blank_streaming_handler = blank_streaming_handler - # Create the new line method - logger.newline = types.MethodType(self.log_newline, logger) - logger.nl = logger.newline - return logger - - @staticmethod - def create_logs_folder(log_path: str): - log_path = os.path.abspath(log_path).split(os.sep) - log_dir = (os.sep.join(log_path[:-1])) - if not os.path.exists(log_dir): - os.makedirs(log_dir) - - @classmethod - def setup_logger(cls, log_path: str, debug: bool = False, clear_log: bool = False) -> None: - """ Sets-up the basic_logger - - Args: - log_path (str): The path where the log file will be saved - debug (bool): Whether to print debug messages or not - clear_log (bool): Whether to empty the log file or not - """ - cls.log_path = os.path.abspath(log_path) - if clear_log: - open(cls.log_path, 'w').close() - cls.log_level = logging.INFO if debug is not True else logging.DEBUG - fancy_log_logger.info(f"Logger is set. Log file path: {cls.log_path}") - - -fancy_log_logger = ColorizedLogger(logger_name='FancyLogger', color='white') diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/main.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/main.py index 40ed85b..53f27bc 100644 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/main.py +++ b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/main.py @@ -1,14 +1,14 @@ import traceback import argparse -from {{cookiecutter.package_name}} import Configuration, ColorizedLogger, timeit, profileit, \ - DropboxCloudstore, MySqlDatastore, GmailEmailer +from {{cookiecutter.package_name}} import Configuration, ColorLogger, timeit, profileit, \ + DropboxCloudManager, HighMySQL, GmailPyEmailSender -basic_logger = ColorizedLogger(logger_name='Main', color='yellow') -fancy_logger = ColorizedLogger(logger_name='FancyMain', - color='blue', - on_color='on_red', - attrs=['underline', 'reverse', 'bold']) +basic_logger = ColorLogger(logger_name='Main', color='yellow') +fancy_logger = ColorLogger(logger_name='FancyMain', + color='blue', + on_color='on_red', + attrs=['underline', 'reverse', 'bold']) def get_args() -> argparse.Namespace: @@ -18,7 +18,7 @@ def get_args() -> argparse.Namespace: argparse.Namespace: """ parser = argparse.ArgumentParser( - description='A template for python projects.', + description='{{cookiecutter.package_description}}', add_help=False) # Required Args required_args = parser.add_argument_group('Required Arguments') @@ -53,7 +53,7 @@ def main(): # Initializing args = get_args() - ColorizedLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=True) + ColorLogger.setup_logger(log_path=args.log, debug=args.debug, clear_log=True) # Load the configuration # configuration = Configuration(config_src=args.config_file, # config_schema_path='yml_schema_strict.json') @@ -78,21 +78,21 @@ def main(): basic_logger.info( "Lastly, you can use profileit either as a function Wrapper or a ContextManager:") with profileit(): - # CloudStore - cloud_conf = configuration.get_config('cloudstore')[0] + # DropboxCloudManager + cloud_conf = configuration.get_config('cloud-filemanager')[0] if cloud_conf['type'] == 'dropbox' and cloud_conf['config']['api_key'] != 'DROPBOX_API_KEY': - dropbox_obj = DropboxCloudstore(config=cloud_conf['config']) + dropbox_obj = DropboxCloudManager(config=cloud_conf['config']) basic_logger.info(f"Base folder contents in dropbox:\n{dropbox_obj.ls().keys()}") - # MySqlDatastore - cloud_conf = configuration.get_config('datastore')[0] + # HighMySQL + cloud_conf = configuration.get_config('high-sql')[0] if cloud_conf['type'] == 'mysql' and cloud_conf['config']['username'] != 'MYSQL_USERNAME': - mysql_obj = MySqlDatastore(config=cloud_conf['config']) + mysql_obj = HighMySQL(config=cloud_conf['config']) basic_logger.info(f"List of tables in DB:\n{mysql_obj.show_tables()}") - # GmailEmailer - cloud_conf = configuration.get_config('emailer')[0] - if cloud_conf['type'] == 'gmail' and cloud_conf['config']['api_key'] != 'GMAIL_API_KEY': + # GmailPyEmailSender + mail_conf = configuration.get_config('pyemail-sender')[0] + if cloud_conf['type'] == 'gmail' and mail_conf['config']['api_key'] != 'GMAIL_API_KEY': basic_logger.info(f"Sending Sample Email to the email address set..") - gmail_obj = GmailEmailer(config=cloud_conf['config']) + gmail_obj = GmailEmailer(config=mail_conf['config']) gmail_obj.send_email(subject='starter', to=[gmail_obj.email_address], text='GmailEmailer works!') diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/__init__.py deleted file mode 100644 index 69c17d7..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/__init__.py +++ /dev/null @@ -1,7 +0,0 @@ -"""Profileit sub-package of {{cookiecutter.package_title_name}}.""" - -from .profileit import profileit - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/profileit.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/profileit.py deleted file mode 100644 index fefe258..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/profiling_funcs/profileit.py +++ /dev/null @@ -1,114 +0,0 @@ -from contextlib import ContextDecorator -from typing import Callable, IO, List -from io import StringIO -from functools import wraps -import cProfile -import pstats - -from {{cookiecutter.package_name}} import ColorizedLogger - -profile_logger = ColorizedLogger('Profileit', 'white') - - -class profileit(ContextDecorator): - custom_print: str - profiler: cProfile.Profile - stream: StringIO - sort_by: str - keep_only_these: List - fraction: float - skip: bool - profiler_output: str - file: IO - - def __init__(self, **kwargs): - """Decorator/ContextManager for profiling functions and code blocks - - Args: - custom_print: Custom print string. When used as decorator it can also be formatted using - `func_name`, `args`, and {0}, {1}, .. to reference the function's - first, second, ... argument. - sort_by: pstats sorting column - profiler_output: Filepath where to save the profiling results (.o file) - keep_only_these: List of strings - grep on the profiling output and print only lines - containing any of these strings - fraction: pstats.print_stats() fraction argument - skip: If True, don't time this time. Suitable when inside loops - file: Write the timing output to a file too - """ - - self.profiler = cProfile.Profile() - self.stream = StringIO() - self.sort_by = 'stdname' - self.keep_only_these = [] - self.fraction = 1.0 - self.skip = False - self.__dict__.update(kwargs) - - def __call__(self, func: Callable): - """ This is called only when invoked as a decorator - - Args: - func: The method to wrap - """ - - @wraps(func) - def profiled(*args, **kwargs): - with self._recreate_cm(): - self.func_name = func.__name__ - self.args = args - self.kwargs = kwargs - self.all_args = (*args, *kwargs.values()) if kwargs != {} else args - return func(*args, **kwargs) - - return profiled - - def __enter__(self, *args, **kwargs): - if not self.skip: - self.profiler.enable() - return self - - def __exit__(self, type, value, traceback): - if self.skip: - return - - self.profiler.disable() - ps = pstats.Stats(self.profiler, stream=self.stream).sort_stats(self.sort_by) - ps.print_stats(self.fraction) - - # If used as a decorator - if hasattr(self, 'func_name'): - if not hasattr(self, 'custom_print'): - print_string = 'Func: {func_name!r} with args: {args!r} profiled:' - else: - print_string = self.custom_print - print_string = print_string.format(*self.args, func_name=self.func_name, - args=self.all_args, - **self.kwargs) - # If used as contextmanager - else: - if not hasattr(self, 'custom_print'): - print_string = 'Code block profiled:' - else: - print_string = self.custom_print - - # Get Profiling results - prof_res = self.stream.getvalue() - if len(self.keep_only_these) > 0: - # Keep only lines containing the specified words - prof_res_list = [line for line in prof_res.split('\n') - if any(keep_word in line for keep_word in self.keep_only_these)] - prof_res = '\n'.join(prof_res_list) - - # Print to file if requested - if hasattr(self, 'file'): - self.file.write(print_string) - self.file.write("\n%s" % prof_res) - - # Save profiler output to a file if requested - if hasattr(self, 'profiler_output'): - self.profiler.dump_stats(self.profiler_output) - - # Actual Print - profile_logger.info(print_string) - profile_logger.info("%s", prof_res) diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/__init__.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/__init__.py deleted file mode 100644 index 212c3c8..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/__init__.py +++ /dev/null @@ -1,8 +0,0 @@ -"""Timeit sub-package of {{cookiecutter.package_title_name}}.""" - -from .timeit import timeit - -__author__ = "{{cookiecutter.author}}" -__email__ = "{{cookiecutter.author_email}}" -__version__ = "{{cookiecutter.package_version}}" - diff --git a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/timeit.py b/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/timeit.py deleted file mode 100644 index 333e034..0000000 --- a/{{cookiecutter.package_title_name}}/{{cookiecutter.package_name}}/timing_tools/timeit.py +++ /dev/null @@ -1,79 +0,0 @@ -from contextlib import ContextDecorator -from typing import Callable, IO -from functools import wraps -from time import time - -from {{cookiecutter.package_name}} import ColorizedLogger - -time_logger = ColorizedLogger('Timeit', 'white') - - -class timeit(ContextDecorator): - custom_print: str - skip: bool - file: IO - - def __init__(self, **kwargs): - """Decorator/ContextManager for counting the execution times of functions and code blocks - - Args: - custom_print: Custom print string Use {duration} to reference the running time. - When used as decorator it can also be formatted using - `func_name`, `args`, and {0}, {1}, .. to reference the function's - first, second, ... argument. - skip: If True, don't time this time. Suitable when inside loops - file: Write the timing output to a file too - """ - - self.total = None - self.skip = False - self.internal_only = False - self.__dict__.update(kwargs) - - def __call__(self, func: Callable): - """ This is called only when invoked as a decorator - - Args: - func: The method to wrap - """ - - @wraps(func) - def timed(*args, **kwargs): - with self._recreate_cm(): - self.func_name = func.__name__ - self.args = args - self.kwargs = kwargs - self.all_args = (*args, *kwargs.values()) if kwargs != {} else args - return func(*args, **kwargs) - - return timed - - def __enter__(self, *args, **kwargs): - if not self.skip: - self.ts = time() - return self - - def __exit__(self, type, value, traceback): - if self.skip: - return - - self.te = time() - self.total = self.te - self.ts - if hasattr(self, 'func_name'): - if not hasattr(self, 'custom_print'): - print_string = 'Func: {func_name!r} with args: {args!r} took: {duration:2.5f} sec(s)' - else: - print_string = self.custom_print - time_logger.info(print_string.format(*self.args, func_name=self.func_name, - args=self.all_args, - duration=self.total, - **self.kwargs)) - else: - if not hasattr(self, 'custom_print'): - print_string = 'Code block took: {duration:2.5f} sec(s)' - else: - print_string = self.custom_print - if hasattr(self, 'file'): - self.file.write(print_string.format(duration=self.total)) - if not self.internal_only: - time_logger.info(print_string.format(duration=self.total))