From f8207aa82eb2c384721a371519bd39cee0381f59 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 20 Dec 2022 16:08:26 +0100 Subject: [PATCH 1/8] fix(docworker): Fix Excel step chartsheets --- .../dsw/document_worker/templates/steps/excel.py | 1 + 1 file changed, 1 insertion(+) diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py index 63105dbf..a7175f8a 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/steps/excel.py @@ -175,6 +175,7 @@ def _add_chart(self, data: dict): for series in series_list: chart.add_series(series) self._add_chart_axis(chart, data) + self._add_chart_basic(chart, data) self._add_chart_advanced(chart, data) self.charts[name] = chart From 339a2f013c86b22429ba42d582057af1ebcf6bab Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 20 Dec 2022 16:16:36 +0100 Subject: [PATCH 2/8] [DSW-1619] fix(mailer): Fix sign up to registry through settings --- packages/dsw-mailer/dsw/mailer/mailer.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index 09a4f318..656ad94d 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -473,7 +473,7 @@ def __init__(self, email: str, org: CmdOrg, code: str, self.callback_url = callback_url @property - def activation_link(self) -> str: + def registry_link(self) -> str: return f'{self.client_url}/signup/{self.org.id}/{self.code}' @property @@ -482,12 +482,19 @@ def callback_link(self) -> Optional[str]: return None return f'{self.client_url}/registry/signup/{self.org.id}/{self.code}' + @property + def activation_link(self) -> Optional[str]: + if self.callback_url is None: + return self.registry_link + return self.callback_link + def to_context(self) -> dict: return { 'organization': self.org.to_context(), 'hash': self.code, - 'activationLink': self.activation_link, + 'registryLink': self.registry_link, 'callbackLink': self.callback_link, + 'activationLink': self.activation_link, 'clientUrl': self.client_url, } From 276d9c01185b7f552df1f2ad4f4d94fbd2cda10b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 20 Dec 2022 17:39:12 +0100 Subject: [PATCH 3/8] Update dependencies --- .github/dependabot.yml | 9 --------- packages/dsw-command-queue/requirements.txt | 6 +++--- packages/dsw-data-seeder/requirements.txt | 10 +++++----- packages/dsw-database/requirements.txt | 6 +++--- packages/dsw-document-worker/requirements.txt | 14 +++++++------- packages/dsw-mailer/requirements.txt | 12 ++++++------ packages/dsw-storage/requirements.txt | 4 ++-- packages/dsw-tdk/requirements.txt | 12 ++++++------ scripts/prepare-dev-win.sh | 2 +- 9 files changed, 33 insertions(+), 42 deletions(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 160c4c05..a1baffb4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,14 +1,5 @@ version: 2 updates: - - package-ecosystem: "pip" - directory: "/" - schedule: - interval: "weekly" - target-branch: "develop" - assignees: - - "MarekSuchanek" - labels: - - "dependencies" - package-ecosystem: "github-actions" directory: "/" diff --git a/packages/dsw-command-queue/requirements.txt b/packages/dsw-command-queue/requirements.txt index 3d22a4a6..ed91d5dd 100644 --- a/packages/dsw-command-queue/requirements.txt +++ b/packages/dsw-command-queue/requirements.txt @@ -1,6 +1,6 @@ -psycopg==3.1.4 -psycopg-binary==3.1.4 +psycopg==3.1.7 +psycopg-binary==3.1.6 PyYAML==6.0 tenacity==8.1.0 typing_extensions==4.4.0 -tzdata==2022.6 +tzdata==2022.7 diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt index 504dffa4..8513acdd 100644 --- a/packages/dsw-data-seeder/requirements.txt +++ b/packages/dsw-data-seeder/requirements.txt @@ -1,11 +1,11 @@ -certifi==2022.9.24 +certifi==2022.12.7 click==8.1.3 colorama==0.4.6 minio==7.1.12 -psycopg==3.1.4 -psycopg-binary==3.1.4 +psycopg==3.1.7 +psycopg-binary==3.1.6 PyYAML==6.0 tenacity==8.1.0 typing_extensions==4.4.0 -tzdata==2022.6 -urllib3==1.26.12 +tzdata==2022.7 +urllib3==1.26.13 diff --git a/packages/dsw-database/requirements.txt b/packages/dsw-database/requirements.txt index 3d22a4a6..ed91d5dd 100644 --- a/packages/dsw-database/requirements.txt +++ b/packages/dsw-database/requirements.txt @@ -1,6 +1,6 @@ -psycopg==3.1.4 -psycopg-binary==3.1.4 +psycopg==3.1.7 +psycopg-binary==3.1.6 PyYAML==6.0 tenacity==8.1.0 typing_extensions==4.4.0 -tzdata==2022.6 +tzdata==2022.7 diff --git a/packages/dsw-document-worker/requirements.txt b/packages/dsw-document-worker/requirements.txt index 4e9e7964..12208394 100644 --- a/packages/dsw-document-worker/requirements.txt +++ b/packages/dsw-document-worker/requirements.txt @@ -1,4 +1,4 @@ -certifi==2022.9.24 +certifi==2022.12.7 charset-normalizer==2.1.1 click==8.1.3 colorama==0.4.6 @@ -11,20 +11,20 @@ mdx-breakless-lists==1.0.1 minio==7.1.12 pathvalidate==2.5.2 pdfrw==0.4 -psycopg==3.1.4 -psycopg-binary==3.1.4 +psycopg==3.1.7 +psycopg-binary==3.1.6 pyparsing==3.0.9 python-dateutil==2.8.2 -python-slugify==6.1.2 +python-slugify==7.0.0 PyYAML==6.0 rdflib==6.2.0 rdflib-jsonld==0.6.2 requests==2.28.1 -sentry-sdk==1.10.1 +sentry-sdk==1.12.1 six==1.16.0 tenacity==8.1.0 text-unidecode==1.3 typing_extensions==4.4.0 -tzdata==2022.6 -urllib3==1.26.12 +tzdata==2022.7 +urllib3==1.26.13 XlsxWriter==3.0.3 diff --git a/packages/dsw-mailer/requirements.txt b/packages/dsw-mailer/requirements.txt index 5a8126f8..9af9a624 100644 --- a/packages/dsw-mailer/requirements.txt +++ b/packages/dsw-mailer/requirements.txt @@ -1,15 +1,15 @@ -certifi==2022.9.24 +certifi==2022.12.7 click==8.1.3 colorama==0.4.6 Jinja2==3.1.2 MarkupSafe==2.1.1 minio==7.1.12 pathvalidate==2.5.2 -psycopg==3.1.4 -psycopg-binary==3.1.4 +psycopg==3.1.7 +psycopg-binary==3.1.6 PyYAML==6.0 -sentry-sdk==1.10.1 +sentry-sdk==1.12.1 tenacity==8.1.0 typing_extensions==4.4.0 -tzdata==2022.6 -urllib3==1.26.12 +tzdata==2022.7 +urllib3==1.26.13 diff --git a/packages/dsw-storage/requirements.txt b/packages/dsw-storage/requirements.txt index 2736ed81..002f5cc3 100644 --- a/packages/dsw-storage/requirements.txt +++ b/packages/dsw-storage/requirements.txt @@ -1,5 +1,5 @@ -certifi==2022.9.24 +certifi==2022.12.7 minio==7.1.12 PyYAML==6.0 tenacity==8.1.0 -urllib3==1.26.12 +urllib3==1.26.13 diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 4aa602c4..424bfff9 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -1,20 +1,20 @@ aiohttp==3.8.3 -aiosignal==1.2.0 +aiosignal==1.3.1 async-timeout==4.0.2 attrs==22.1.0 charset-normalizer==2.1.1 click==8.1.3 colorama==0.4.6 -frozenlist==1.3.1 +frozenlist==1.3.3 humanize==4.4.0 idna==3.4 Jinja2==3.1.2 MarkupSafe==2.1.1 -multidict==6.0.2 -pathspec==0.10.1 +multidict==6.0.3 +pathspec==0.10.3 python-dotenv==0.21.0 -python-slugify==6.1.2 +python-slugify==7.0.0 text-unidecode==1.3 typing-extensions==4.4.0 watchgod==0.8.2 -yarl==1.8.1 +yarl==1.8.2 diff --git a/scripts/prepare-dev-win.sh b/scripts/prepare-dev-win.sh index 2075ecad..705f972f 100644 --- a/scripts/prepare-dev-win.sh +++ b/scripts/prepare-dev-win.sh @@ -6,7 +6,7 @@ if [ "$1" == "--clean" ]; then rm -rf env echo "Setup new venv" - python3 -m venv env + python -m venv env fi . env/Scripts/activate From 4807f4a696dc30d19f5984a3099c432caf0f015f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 20 Dec 2022 18:43:45 +0100 Subject: [PATCH 4/8] [DSW-1613] feat(mailer): Add mail config to database --- packages/dsw-command-queue/pyproject.toml | 4 +- packages/dsw-config/pyproject.toml | 2 +- packages/dsw-data-seeder/pyproject.toml | 10 +- .../dsw-database/dsw/database/database.py | 29 +++++- packages/dsw-database/dsw/database/model.py | 35 +++++++ packages/dsw-database/pyproject.toml | 4 +- packages/dsw-document-worker/pyproject.toml | 10 +- packages/dsw-mailer/dsw/mailer/cli.py | 6 +- .../dsw-mailer/dsw/mailer/connection/smtp.py | 91 +++++++++++-------- packages/dsw-mailer/dsw/mailer/mailer.py | 39 ++++++-- packages/dsw-mailer/dsw/mailer/templates.py | 12 ++- packages/dsw-mailer/pyproject.toml | 8 +- packages/dsw-storage/pyproject.toml | 4 +- packages/dsw-tdk/pyproject.toml | 2 +- 14 files changed, 178 insertions(+), 78 deletions(-) diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index 1aea0e45..9b6a66a9 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -27,7 +27,7 @@ classifiers = [ requires-python = '>=3.7, <4' dependencies = [ # DSW - 'dsw-database==3.18.0', + 'dsw-database==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index dc3e9ec3..81911ad0 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index 55299b45..d324db57 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = '3.18.0rc1' +version = '3.19.0.dev0' description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -27,10 +27,10 @@ dependencies = [ 'click', 'tenacity', # DSW - 'dsw-command-queue==3.18.0', - 'dsw-config==3.18.0', - 'dsw-database==3.18.0', - 'dsw-storage==3.18.0', + 'dsw-command-queue==3.19.0.dev0', + 'dsw-config==3.19.0.dev0', + 'dsw-database==3.19.0.dev0', + 'dsw-storage==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py index c17e00e3..6ed361a2 100644 --- a/packages/dsw-database/dsw/database/database.py +++ b/packages/dsw-database/dsw/database/database.py @@ -10,7 +10,7 @@ from dsw.config.model import DatabaseConfig from .model import DBTemplate, DBTemplateFile, DBTemplateAsset, DBDocument, \ - DocumentState, DBAppConfig, DBAppLimits, DBSubmission + DocumentState, DBAppConfig, DBAppLimits, DBSubmission, DBInstanceConfigMail LOG = logging.getLogger(__name__) @@ -43,6 +43,10 @@ class Database: SELECT_TEMPLATE = 'SELECT * FROM template WHERE id = %s AND app_uuid = %s LIMIT 1;' SELECT_TEMPLATE_FILES = 'SELECT * FROM template_file WHERE template_id = %s AND app_uuid = %s;' SELECT_TEMPLATE_ASSETS = 'SELECT * FROM template_asset WHERE template_id = %s AND app_uuid = %s;' + SELECT_MAIL_CONFIG = 'SELECT icm.* ' \ + 'FROM app_config ac JOIN instance_config_mail icm ' \ + 'ON ac.mail_config_uuid = icm.uuid ' \ + 'WHERE ac.uuid = %(app_uuid)s;' SUM_FILE_SIZES = 'SELECT (SELECT COALESCE(SUM(file_size)::bigint, 0) ' \ 'FROM document WHERE app_uuid = %(app_uuid)s) ' \ @@ -310,6 +314,29 @@ def get_app_config(self, app_uuid: str) -> Optional[DBAppConfig]: f' "{app_uuid}": {str(e)}') return None + @tenacity.retry( + reraise=True, + wait=tenacity.wait_exponential(multiplier=RETRY_QUERY_MULTIPLIER), + stop=tenacity.stop_after_attempt(RETRY_QUERY_TRIES), + before=tenacity.before_log(LOG, logging.DEBUG), + after=tenacity.after_log(LOG, logging.DEBUG), + ) + def get_mail_config(self, app_uuid: str) -> Optional[DBInstanceConfigMail]: + with self.conn_query.new_cursor(use_dict=True) as cursor: + try: + cursor.execute( + query=self.SELECT_MAIL_CONFIG, + params={'app_uuid': app_uuid}, + ) + result = cursor.fetchone() + if result is None: + return None + return DBInstanceConfigMail.from_dict_row(data=result) + except Exception as e: + LOG.warning(f'Could not retrieve instance_mail_config for app' + f' "{app_uuid}": {str(e)}') + return None + @tenacity.retry( reraise=True, wait=tenacity.wait_exponential(multiplier=RETRY_QUERY_MULTIPLIER), diff --git a/packages/dsw-database/dsw/database/model.py b/packages/dsw-database/dsw/database/model.py index 00b27bef..52d96a4c 100644 --- a/packages/dsw-database/dsw/database/model.py +++ b/packages/dsw-database/dsw/database/model.py @@ -281,3 +281,38 @@ def to_dict(self) -> dict: 'updated_at': self.updated_at.isoformat(timespec='milliseconds'), 'app_uuid': self.app_uuid, } + + +@dataclasses.dataclass +class DBInstanceConfigMail: + TABLE_NAME = 'instance_config_mail' + + uuid: str + enabled: bool + sender_name: Optional[str] + sender_email: Optional[str] + host: str + port: Optional[int] + security: Optional[str] + username: Optional[str] + password: Optional[str] + rate_limit_window: Optional[int] + rate_limit_count: Optional[int] + timeout: Optional[int] + + @staticmethod + def from_dict_row(data: dict): + return DBInstanceConfigMail( + uuid=str(data['uuid']), + enabled=data['enabled'], + sender_name=data['sender_name'], + sender_email=data['sender_email'], + host=data['host'], + port=data['port'], + security=data['security'], + username=data['username'], + password=data['password'], + rate_limit_window=data['rate_limit_window'], + rate_limit_count=data['rate_limit_count'], + timeout=data['timeout'], + ) diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index b512a594..9f4e761d 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -28,7 +28,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - 'dsw-config==3.18.0', + 'dsw-config==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index cbd17a5a..16b4e1a3 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -39,10 +39,10 @@ dependencies = [ 'tenacity', 'XlsxWriter', # DSW - 'dsw-command-queue==3.18.0', - 'dsw-config==3.18.0', - 'dsw-database==3.18.0', - 'dsw-storage==3.18.0', + 'dsw-command-queue==3.19.0.dev0', + 'dsw-config==3.19.0.dev0', + 'dsw-database==3.19.0.dev0', + 'dsw-storage==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-mailer/dsw/mailer/cli.py b/packages/dsw-mailer/dsw/mailer/cli.py index 362e078b..2ef2813e 100644 --- a/packages/dsw-mailer/dsw/mailer/cli.py +++ b/packages/dsw-mailer/dsw/mailer/cli.py @@ -1,7 +1,6 @@ import click # type: ignore import json import pathlib -import sys from typing import IO @@ -54,9 +53,6 @@ def extract_message_request(ctx, param, value: IO): default='wizard') def cli(ctx, config: MailerConfig, workdir: str, mode: str): """Mailer for sending emails from DSW""" - if not config.mail.enabled: - click.echo('Mail is set to disabled, why even running mailer?') - sys.exit(1) path_workdir = pathlib.Path(workdir) prepare_logging(cfg=config) from .mailer import Mailer @@ -71,7 +67,7 @@ def send(ctx, msg_request: MessageRequest): """Send message(s) from given file directly""" from .mailer import Mailer mailer = ctx.obj['mailer'] # type: Mailer - mailer.send(rq=msg_request) + mailer.send(rq=msg_request, cfg=None) @cli.command() diff --git a/packages/dsw-mailer/dsw/mailer/connection/smtp.py b/packages/dsw-mailer/dsw/mailer/connection/smtp.py index 5098f291..6ce4b691 100644 --- a/packages/dsw-mailer/dsw/mailer/connection/smtp.py +++ b/packages/dsw-mailer/dsw/mailer/connection/smtp.py @@ -9,6 +9,7 @@ from email.mime.multipart import MIMEMultipart from email.mime.text import MIMEText from email.utils import formataddr +from typing import Optional from dsw.config.model import MailConfig @@ -24,7 +25,7 @@ class SMTPSender: def __init__(self, cfg: MailConfig): - self.cfg = cfg + self.default_cfg = cfg @tenacity.retry( reraise=True, @@ -33,54 +34,61 @@ def __init__(self, cfg: MailConfig): before=tenacity.before_log(Context.logger, logging.DEBUG), after=tenacity.after_log(Context.logger, logging.DEBUG), ) - def send(self, message: MailMessage): - self._send(message) - - def _send(self, mail: MailMessage): - if self.cfg.is_ssl: - return self._send_smtp_ssl(mail=mail) - return self._send_smtp(mail=mail) - - def _send_smtp_ssl(self, mail: MailMessage): + def send(self, message: MailMessage, cfg: Optional[MailConfig]): + used_cfg = cfg or self.default_cfg + if not used_cfg.enabled: + Context.logger.info('Not actually sending email (enabled=False)') + return + Context.logger.info(f'Sending via SMTP: {used_cfg.host}:{used_cfg.port}') + self._send(message, used_cfg) + + def _send(self, mail: MailMessage, cfg: MailConfig): + if cfg.is_ssl: + return self._send_smtp_ssl(mail=mail, cfg=cfg) + return self._send_smtp(mail=mail, cfg=cfg) + + def _send_smtp_ssl(cls, mail: MailMessage, cfg: MailConfig): context = ssl.create_default_context() with smtplib.SMTP_SSL( - host=self.cfg.host, - port=self.cfg.port, + host=cfg.host, + port=cfg.port, context=context, - timeout=self.cfg.timeout, + timeout=cfg.timeout, ) as server: - if self.cfg.auth: + if cfg.auth: server.login( - user=self.cfg.login_user, - password=self.cfg.login_password, + user=cfg.login_user, + password=cfg.login_password, ) return server.send_message( - msg=self._convert_email(mail), + msg=cls._convert_email(mail), from_addr=formataddr((mail.from_name, mail.from_mail)), to_addrs=mail.recipients, ) - def _send_smtp(self, mail: MailMessage): + @classmethod + def _send_smtp(cls, mail: MailMessage, cfg: MailConfig): context = ssl.create_default_context() with smtplib.SMTP( - host=self.cfg.host, - port=self.cfg.port, - timeout=self.cfg.timeout, + host=cfg.host, + port=cfg.port, + timeout=cfg.timeout, ) as server: - if self.cfg.is_tls: + if cfg.is_tls: server.starttls(context=context) - if self.cfg.auth: + if cfg.auth: server.login( - user=self.cfg.login_user, - password=self.cfg.login_password, + user=cfg.login_user, + password=cfg.login_password, ) return server.send_message( - msg=self._convert_email(mail), + msg=cls._convert_email(mail), from_addr=formataddr((mail.from_name, mail.from_mail)), to_addrs=mail.recipients, ) - def _convert_inline_image(self, image: MailAttachment) -> MIMEBase: + @staticmethod + def _convert_inline_image(image: MailAttachment) -> MIMEBase: mtype, msubtype = image.content_type.split('/', maxsplit=1) part = MIMEBase(mtype, msubtype) part.set_payload(image.data) @@ -90,7 +98,8 @@ def _convert_inline_image(self, image: MailAttachment) -> MIMEBase: part.add_header('Content-Disposition', f'inline; filename={filename}') return part - def _convert_html_part(self, mail: MailMessage) -> MIMEBase: + @classmethod + def _convert_html_part(cls, mail: MailMessage) -> MIMEBase: if mail.html_body is None: raise RuntimeError('Requested HTML body but there is none') txt_part = MIMEText(mail.html_body, 'html', EMAIL_ENCODING) @@ -99,27 +108,30 @@ def _convert_html_part(self, mail: MailMessage) -> MIMEBase: part = MIMEMultipart('related') part.attach(txt_part) for image in mail.html_images: - part.attach(self._convert_inline_image(image)) + part.attach(cls._convert_inline_image(image)) return part return txt_part - def _convert_plain_part(self, mail: MailMessage) -> MIMEText: + @staticmethod + def _convert_plain_part(mail: MailMessage) -> MIMEText: if mail.plain_body is None: raise RuntimeError('Requested plain body but there is none') return MIMEText(mail.plain_body, 'plain', EMAIL_ENCODING) - def _convert_txt_parts(self, mail: MailMessage) -> MIMEBase: + @classmethod + def _convert_txt_parts(cls, mail: MailMessage) -> MIMEBase: if mail.plain_body is None: - return self._convert_html_part(mail) + return cls._convert_html_part(mail) if mail.html_body is None: - return self._convert_plain_part(mail) + return cls._convert_plain_part(mail) part = MIMEMultipart('alternative') part.set_charset(EMAIL_ENCODING) - part.attach(self._convert_plain_part(mail)) - part.attach(self._convert_html_part(mail)) + part.attach(cls._convert_plain_part(mail)) + part.attach(cls._convert_html_part(mail)) return part - def _convert_attachment(self, attachment: MailAttachment) -> MIMEBase: + @staticmethod + def _convert_attachment(attachment: MailAttachment) -> MIMEBase: mtype, msubtype = attachment.content_type.split('/', maxsplit=1) part = MIMEBase(mtype, msubtype) part.set_payload(attachment.data) @@ -128,14 +140,15 @@ def _convert_attachment(self, attachment: MailAttachment) -> MIMEBase: part.add_header('Content-Disposition', f'attachment; filename={filename}') return part - def _convert_email(self, mail: MailMessage) -> MIMEBase: - msg = self._convert_txt_parts(mail) + @classmethod + def _convert_email(cls, mail: MailMessage) -> MIMEBase: + msg = cls._convert_txt_parts(mail) if len(mail.attachments) > 0: txt = msg msg = MIMEMultipart('mixed') msg.attach(txt) for attachment in mail.attachments: - msg.attach(self._convert_attachment(attachment)) + msg.attach(cls._convert_attachment(attachment)) msg['From'] = formataddr((mail.from_name, mail.from_mail)) msg['To'] = ', '.join(mail.recipients) msg['Subject'] = mail.subject diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index 656ad94d..0acce0ec 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -9,9 +9,10 @@ from dsw.command_queue import CommandWorker, CommandQueue from dsw.database.database import Database -from dsw.database.model import DBAppConfig, PersistentCommand +from dsw.database.model import DBAppConfig, PersistentCommand, \ + DBInstanceConfigMail -from .config import MailerConfig +from .config import MailerConfig, MailConfig from .connection import SMTPSender, SentryReporter from .consts import Queries, CMD_COMPONENT from .context import Context @@ -95,9 +96,14 @@ def _process_command(self, cmd: PersistentCommand): msg_id=cmd.uuid, trigger='PersistentComment', ) + # get mailer config from DB + cfg = _transform_mail_config( + cfg=app_ctx.db.get_mail_config(app_uuid=cmd.app_uuid), + ) + Context.logger.debug(f'Config from DB: {cfg}') # update Sentry info SentryReporter.set_context('template', rq.template_name) - self.send(rq) + self.send(rq, cfg) app_ctx.db.execute_query( query=Queries.UPDATE_CMD_DONE, attempts=cmd.attempts + 1, @@ -115,19 +121,40 @@ def run(self): ) queue.run() - def send(self, rq: MessageRequest): + def send(self, rq: MessageRequest, cfg: Optional[MailConfig]): Context.logger.info(f'Sending request: {rq.template_name} ({rq.id})') # get template if not self.ctx.templates.has_template_for(rq): raise RuntimeError(f'Template not found: {rq.template_name}') # render Context.logger.info(f'Rendering message: {rq.template_name}') - msg = self.ctx.templates.render(rq) + msg = self.ctx.templates.render(rq, cfg) # send Context.logger.info(f'Sending message: {rq.template_name}') - self.ctx.app.sender.send(msg) + self.ctx.app.sender.send(msg, cfg) Context.logger.info('Message sent successfully') + +def _transform_mail_config(cfg: Optional[DBInstanceConfigMail]) -> Optional[MailConfig]: + if cfg is None: + return None + return MailConfig( + enabled=cfg.enabled, + name=cfg.sender_name, + email=cfg.sender_email, + host=cfg.host, + port=cfg.port, + security=cfg.security, + username=cfg.username, + password=cfg.password, + rate_limit_window=cfg.rate_limit_window, + rate_limit_count=cfg.rate_limit_count, + timeout=cfg.timeout, + ssl=None, + auth=None, + ) + + ######################################################################################### # Commands and their logic ######################################################################################### diff --git a/packages/dsw-mailer/dsw/mailer/templates.py b/packages/dsw-mailer/dsw/mailer/templates.py index a2bec90f..a189ef91 100644 --- a/packages/dsw-mailer/dsw/mailer/templates.py +++ b/packages/dsw-mailer/dsw/mailer/templates.py @@ -4,7 +4,7 @@ from typing import Optional -from .config import MailerConfig +from .config import MailerConfig, MailConfig from .consts import DEFAULT_ENCODING from .model import MailMessage, MailAttachment, MessageRequest,\ TemplateDescriptor, TemplateDescriptorPart @@ -67,7 +67,8 @@ def _load_jinja2(self, file_path: pathlib.Path) -> Optional[jinja2.Template]: ) return None - def _load_attachment(self, template_path: pathlib.Path, + @staticmethod + def _load_attachment(template_path: pathlib.Path, part: TemplateDescriptorPart) -> Optional[MailAttachment]: file_path = template_path / part.file if file_path.exists() and file_path.is_file(): @@ -139,9 +140,10 @@ def _load_templates(self, mode: str): def has_template_for(self, rq: MessageRequest) -> bool: return rq.template_name in self.templates.keys() - def render(self, rq: MessageRequest) -> MailMessage: + def render(self, rq: MessageRequest, cfg: MailConfig) -> MailMessage: + used_cfg = cfg or self.cfg.mail return self.templates[rq.template_name].render( rq=rq, - mail_name=self.cfg.mail.name, - mail_from=self.cfg.mail.email, + mail_name=used_cfg.name, + mail_from=used_cfg.email, ) diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index 040ec60e..d6e47e2e 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -29,9 +29,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - 'dsw-command-queue==3.18.0', - 'dsw-config==3.18.0', - 'dsw-database==3.18.0', + 'dsw-command-queue==3.19.0.dev0', + 'dsw-config==3.19.0.dev0', + 'dsw-database==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 59ea28fb..da0e810e 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -28,7 +28,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - 'dsw-config==3.18.0', + 'dsw-config==3.19.0.dev0', ] [project.urls] diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index d6c6604f..8418092f 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = '3.18.0' +version = '3.19.0.dev0' description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit'] From 38bd084b59413a8b43fcfdfc964275db8bf408fa Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 23 Dec 2022 17:57:55 +0100 Subject: [PATCH 5/8] [DSW-1623] feat: Support config via ENV variables --- packages/dsw-config/dsw/config/keys.py | 293 ++++++++++++++++++ packages/dsw-config/dsw/config/model.py | 41 +-- packages/dsw-config/dsw/config/parser.py | 178 ++++------- .../dsw-data-seeder/dsw/data_seeder/cli.py | 16 +- .../dsw/document_worker/cli.py | 14 +- .../dsw/document_worker/config.py | 197 +++++++----- packages/dsw-mailer/dsw/mailer/cli.py | 14 +- packages/dsw-mailer/dsw/mailer/config.py | 11 +- 8 files changed, 530 insertions(+), 234 deletions(-) create mode 100644 packages/dsw-config/dsw/config/keys.py diff --git a/packages/dsw-config/dsw/config/keys.py b/packages/dsw-config/dsw/config/keys.py new file mode 100644 index 00000000..710216a0 --- /dev/null +++ b/packages/dsw-config/dsw/config/keys.py @@ -0,0 +1,293 @@ +import collections + +from typing import Any, Optional, Generic, TypeVar, Callable + + +T = TypeVar('T') + + +def cast_bool(value: Any) -> bool: + return bool(value) + + +def cast_int(value: Any) -> int: + return int(value) + + +def cast_optional_int(value: Any) -> Optional[int]: + if value is None: + return None + return int(value) + + +def cast_float(value: Any) -> float: + return float(value) + + +def cast_optional_float(value: Any) -> Optional[float]: + if value is None: + return None + return float(value) + + +def cast_str(value: Any) -> str: + return str(value) + + +def cast_optional_str(value: Any) -> Optional[str]: + if value is None: + return None + return str(value) + + +class ConfigKey(Generic[T]): + + def __init__(self, yaml_path: list[str], cast: Callable[[Any], T], + var_names=None, default=None, required=False): + self.yaml_path = yaml_path + self.var_names = var_names or [] # type: list[str] + self.default = default + self.required = required + self.cast = cast + + def __str__(self): + return 'ConfigKey: ' + '.'.join(self.yaml_path) + + +class ConfigKeysMeta(type): + + @classmethod + def __prepare__(mcs, name, bases, **kwargs): + return collections.OrderedDict() + + def __init__(cls, name, bases, namespace): + cls._config_keys = [] + for attr in namespace: + if attr.startswith('_'): + continue + value = getattr(cls, attr) + if isinstance(value, ConfigKey): + cls._config_keys.append(value) + if hasattr(value, '_config_keys'): + keys = getattr(value, '_config_keys') + if isinstance(keys, list): + cls._config_keys.extend(keys) + super().__init__(name, bases, namespace) + + def __iter__(cls): + return iter(cls._config_keys) + + +class ConfigKeysContainer(metaclass=ConfigKeysMeta): + pass + + +class _GeneralKeys(ConfigKeysContainer): + environment = ConfigKey( + yaml_path=['general', 'environment'], + var_names=['GENERAL_ENVIRONMENT'], + default='Production', + cast=cast_str, + ) + client_url = ConfigKey( + yaml_path=['general', 'clientUrl'], + var_names=['GENERAL_CLIENT_URL'], + default='http://localhost:8080', + cast=cast_str, + ) + secret = ConfigKey( + yaml_path=['general', 'secret'], + var_names=['GENERAL_SECRET'], + default='', + cast=cast_str, + ) + + +class _LoggingKeys(ConfigKeysContainer): + level = ConfigKey( + yaml_path=['logging', 'level'], + var_names=['LOGGING_ENVIRONMENT'], + default='INFO', + cast=cast_str, + ) + global_level = ConfigKey( + yaml_path=['logging', 'globalLevel'], + var_names=['LOGGING_CLIENT_URL'], + default='WARNING', + cast=cast_str, + ) + format = ConfigKey( + yaml_path=['logging', 'format'], + var_names=['LOGGING_FORMAT'], + default='%(asctime)s | %(levelname)8s | %(name)s: [T:%(traceId)s] %(message)s', + cast=cast_str, + ) + + +class _CloudKeys(ConfigKeysContainer): + enabled = ConfigKey( + yaml_path=['cloud', 'enabled'], + var_names=['CLOUD_ENABLED'], + default=False, + cast=cast_bool, + ) + + +class _SentryKeys(ConfigKeysContainer): + enabled = ConfigKey( + yaml_path=['sentry', 'enabled'], + var_names=['SENTRY_ENABLED'], + default=False, + cast=cast_bool, + ) + worker_dsn = ConfigKey( + yaml_path=['sentry', 'workersDsn'], + var_names=['SENTRY_WORKER_DSN'], + default='', + cast=cast_str, + ) + + +class _DatabaseKeys(ConfigKeysContainer): + connection_string = ConfigKey( + yaml_path=['database', 'connectionString'], + var_names=['DATABASE_CONNECTION_STRING'], + default='postgresql://postgres:postgres@postgres:5432/engine-wizard', + cast=cast_str, + ) + connection_timeout = ConfigKey( + yaml_path=['database', 'connectionTimeout'], + var_names=['DATABASE_CONNECTION_TIMEOUT'], + default=30000, + cast=cast_int, + ) + queue_timeout = ConfigKey( + yaml_path=['database', 'queueTimeout'], + var_names=['DATABASE_QUEUE_TIMEOUT'], + default=300, + cast=cast_int, + ) + + +class _S3Keys(ConfigKeysContainer): + url = ConfigKey( + yaml_path=['s3', 'url'], + var_names=['S3_URL'], + default='http://minio:9000', + cast=cast_str, + ) + bucket = ConfigKey( + yaml_path=['s3', 'bucket'], + var_names=['S3_BUCKET'], + default='engine-wizard', + cast=cast_str, + ) + region = ConfigKey( + yaml_path=['s3', 'region'], + var_names=['S3_REGION'], + default='eu-central-1', + cast=cast_str, + ) + username = ConfigKey( + yaml_path=['s3', 'username'], + var_names=['S3_USERNAME'], + default='minio', + cast=cast_str, + ) + password = ConfigKey( + yaml_path=['s3', 'password'], + var_names=['S3_PASSWORD'], + default='minioPassword', + cast=cast_str, + ) + + +class _MailKeys(ConfigKeysContainer): + enabled = ConfigKey( + yaml_path=['mail', 'enabled'], + var_names=['MAIL_ENABLED'], + default=True, + cast=cast_bool, + ) + name = ConfigKey( + yaml_path=['mail', 'name'], + var_names=['MAIL_NAME'], + default='', + cast=cast_str, + ) + email = ConfigKey( + yaml_path=['mail', 'email'], + var_names=['MAIL_EMAIL'], + default='', + cast=cast_str, + ) + host = ConfigKey( + yaml_path=['mail', 'host'], + var_names=['MAIL_HOST'], + default='', + cast=cast_str, + ) + port = ConfigKey( + yaml_path=['mail', 'port'], + var_names=['MAIL_PORT'], + cast=cast_str, + ) + ssl = ConfigKey( + yaml_path=['mail', 'ssl'], + var_names=[], + cast=cast_str, + ) + security = ConfigKey( + yaml_path=['mail', 'security'], + var_names=['MAIL_SECURITY'], + cast=cast_str, + ) + auth_enabled = ConfigKey( + yaml_path=['mail', 'authEnabled'], + var_names=[], + default=False, + cast=cast_bool, + ) + username = ConfigKey( + yaml_path=['mail', 'username'], + var_names=['MAIL_USERNAME'], + cast=cast_str, + ) + password = ConfigKey( + yaml_path=['mail', 'password'], + var_names=['MAIL_PASSWORD'], + cast=cast_str, + ) + rate_limit_window = ConfigKey( + yaml_path=['mail', 'rateLimit', 'window'], + var_names=['MAIL_RATE_LIMIT_WINDOW'], + default=0, + cast=cast_int, + ) + rate_limit_count = ConfigKey( + yaml_path=['mail', 'rateLimit', 'count'], + var_names=['MAIL_RATE_LIMIT_COUNT'], + default=0, + cast=cast_int, + ) + timeout = ConfigKey( + yaml_path=['mail', 'enabled'], + var_names=['MAIL_TIMEOUT'], + default=5, + cast=cast_int, + ) + + +class ConfigKeys(ConfigKeysContainer): + cloud = _CloudKeys + database = _DatabaseKeys + general = _GeneralKeys + logging = _LoggingKeys + mail = _MailKeys + s3 = _S3Keys + sentry = _SentryKeys + + +if __name__ == '__main__': + for key in ConfigKeys: + print(str(key)) diff --git a/packages/dsw-config/dsw/config/model.py b/packages/dsw-config/dsw/config/model.py index c2de6920..6d556028 100644 --- a/packages/dsw-config/dsw/config/model.py +++ b/packages/dsw-config/dsw/config/model.py @@ -11,38 +11,36 @@ def _config_to_string(config: object): return '\n'.join(lines) -class GeneralConfig: - - def __init__(self, environment: str, client_url: str): - self.environment = environment - self.client_url = client_url +class ConfigModel: def __str__(self): return _config_to_string(self) -class SentryConfig: +class GeneralConfig(ConfigModel): + + def __init__(self, environment: str, client_url: str, secret: str): + self.environment = environment + self.client_url = client_url + self.secret = secret + + +class SentryConfig(ConfigModel): def __init__(self, enabled: bool, workers_dsn: Optional[str]): self.enabled = enabled self.workers_dsn = workers_dsn - def __str__(self): - return _config_to_string(self) - -class DatabaseConfig: +class DatabaseConfig(ConfigModel): def __init__(self, connection_string: str, connection_timeout: int, queue_timout: int): self.connection_string = connection_string self.connection_timeout = connection_timeout self.queue_timout = queue_timout - def __str__(self): - return _config_to_string(self) - -class S3Config: +class S3Config(ConfigModel): def __init__(self, url: str, username: str, password: str, bucket: str, region: str): @@ -52,31 +50,22 @@ def __init__(self, url: str, username: str, password: str, self.bucket = bucket self.region = region - def __str__(self): - return _config_to_string(self) - -class LoggingConfig: +class LoggingConfig(ConfigModel): def __init__(self, level: str, global_level: str, message_format: str): self.level = level self.global_level = global_level self.message_format = message_format - def __str__(self): - return _config_to_string(self) - -class CloudConfig: +class CloudConfig(ConfigModel): def __init__(self, multi_tenant: bool): self.multi_tenant = multi_tenant - def __str__(self): - return _config_to_string(self) - -class MailConfig: +class MailConfig(ConfigModel): def __init__(self, enabled: bool, ssl: Optional[bool], name: str, email: str, host: str, port: Optional[int], security: Optional[str], diff --git a/packages/dsw-config/dsw/config/parser.py b/packages/dsw-config/dsw/config/parser.py index aa2f6675..199cffa7 100644 --- a/packages/dsw-config/dsw/config/parser.py +++ b/packages/dsw-config/dsw/config/parser.py @@ -1,7 +1,9 @@ +import os import yaml -from typing import List, Any +from typing import List, Any, IO +from .keys import ConfigKey, ConfigKeys from .model import GeneralConfig, SentryConfig, S3Config, \ DatabaseConfig, LoggingConfig, CloudConfig, MailConfig @@ -14,175 +16,129 @@ def __init__(self, missing: List[str]): class DSWConfigParser: - DB_SECTION = 'database' - S3_SECTION = 's3' - LOGGING_SECTION = 'logging' - CLOUD_SECTION = 'cloud' - SENTRY_SECTION = 'sentry' - GENERAL_SECTION = 'general' - MAIL_SECTION = 'mail' - - DEFAULTS = { - DB_SECTION: { - 'connectionString': 'postgresql://postgres:postgres@postgres:5432/engine-wizard', - 'connectionTimeout': 30000, - 'queueTimeout': 120, - }, - S3_SECTION: { - 'url': 'http://minio:9000', - 'vhost': 'minio', - 'queue': 'minio', - 'bucket': 'engine-wizard', - 'region': 'eu-central-1', - }, - LOGGING_SECTION: { - 'level': 'INFO', - 'globalLevel': 'WARNING', - 'format': '%(asctime)s | %(levelname)8s | %(name)s: [T:%(traceId)s] %(message)s', - }, - CLOUD_SECTION: { - 'enabled': False, - }, - SENTRY_SECTION: { - 'enabled': False, - 'workersDsn': None - }, - GENERAL_SECTION: { - 'environment': 'Production', - 'clientUrl': 'http://localhost:8080', - }, - MAIL_SECTION: { - 'enabled': True, - 'name': 'DS Wizard', - 'email': '', - 'host': '', - 'port': None, - 'ssl': None, - 'security': None, - 'authEnabled': False, - 'username': None, - 'password': None, - 'rateLimit': { - 'window': 0, - 'count': 0, - }, - 'timeout': 5, - }, - } - - REQUIRED = [] # type: List[str] - - def __init__(self): + def __init__(self, keys=ConfigKeys): self.cfg = dict() + self.keys = keys @staticmethod - def can_read(content): + def can_read(content: str): try: yaml.load(content, Loader=yaml.FullLoader) return True except Exception: return False - def read_file(self, fp): - self.cfg = yaml.load(fp, Loader=yaml.FullLoader) + def read_file(self, fp: IO): + self.cfg = yaml.load(fp, Loader=yaml.FullLoader) or self.cfg - def read_string(self, content): - self.cfg = yaml.load(content, Loader=yaml.FullLoader) + def read_string(self, content: str): + self.cfg = yaml.load(content, Loader=yaml.FullLoader) or self.cfg - def has(self, *path): + def has_value_for_path(self, yaml_path: list[str]): x = self.cfg - for p in path: + for p in yaml_path: if not hasattr(x, 'keys') or p not in x.keys(): return False x = x[p] return True - def _get_default(self, *path): - x = self.DEFAULTS # type: Any - for p in path: - x = x[p] - return x + @staticmethod + def _prefix_var(var_name: str) -> str: + return f'DSW_{var_name}' - def get_or_default(self, *path): + def has_value_for_key(self, key: ConfigKey): + if self.has_value_for_path(key.yaml_path): + return True + for var_name in key.var_names: + if var_name in os.environ.keys() or \ + self._prefix_var(var_name) in os.environ.keys(): + return True + + def get_or_default(self, key: ConfigKey): x = self.cfg # type: Any - for p in path: + for p in key.yaml_path: if not hasattr(x, 'keys') or p not in x.keys(): - return self._get_default(*path) + return key.default x = x[p] return x + def get(self, key: ConfigKey): + for var_name in key.var_names: + if var_name in os.environ.keys(): + return key.cast(os.environ[var_name]) + if self._prefix_var(var_name) in os.environ.keys(): + return key.cast(os.environ[self._prefix_var(var_name)]) + return key.cast(self.get_or_default(key)) + def validate(self): missing = [] - for path in self.REQUIRED: - if not self.has(*path): - missing.append('.'.join(path)) + for key in self.keys: + if key.required and not self.has_value_for_key(key): + missing.append('.'.join(key.yaml_path)) if len(missing) > 0: raise MissingConfigurationError(missing) @property def db(self) -> DatabaseConfig: return DatabaseConfig( - connection_string=self.get_or_default(self.DB_SECTION, 'connectionString'), - connection_timeout=self.get_or_default(self.DB_SECTION, 'connectionTimeout'), - queue_timout=self.get_or_default(self.DB_SECTION, 'queueTimeout'), + connection_string=self.get(self.keys.database.connection_string), + connection_timeout=self.get(self.keys.database.connection_timeout), + queue_timout=self.get(self.keys.database.queue_timeout), ) @property def s3(self) -> S3Config: return S3Config( - url=self.get_or_default(self.S3_SECTION, 'url'), - username=self.get_or_default(self.S3_SECTION, 'username'), - password=self.get_or_default(self.S3_SECTION, 'password'), - bucket=self.get_or_default(self.S3_SECTION, 'bucket'), - region=self.get_or_default(self.S3_SECTION, 'region'), + url=self.get(self.keys.s3.url), + username=self.get(self.keys.s3.username), + password=self.get(self.keys.s3.password), + bucket=self.get(self.keys.s3.bucket), + region=self.get(self.keys.s3.region), ) @property def logging(self) -> LoggingConfig: return LoggingConfig( - level=self.get_or_default(self.LOGGING_SECTION, 'level'), - global_level=self.get_or_default(self.LOGGING_SECTION, 'globalLevel'), - message_format=self.get_or_default(self.LOGGING_SECTION, 'format'), + level=self.get(self.keys.logging.level), + global_level=self.get(self.keys.logging.global_level), + message_format=self.get(self.keys.logging.format), ) @property def cloud(self) -> CloudConfig: return CloudConfig( - multi_tenant=self.get_or_default(self.CLOUD_SECTION, 'enabled'), + multi_tenant=self.get(self.keys.cloud.enabled), ) @property def sentry(self) -> SentryConfig: return SentryConfig( - enabled=self.get_or_default(self.SENTRY_SECTION, 'enabled'), - workers_dsn=self.get_or_default(self.SENTRY_SECTION, 'workersDsn'), + enabled=self.get(self.keys.sentry.enabled), + workers_dsn=self.get(self.keys.sentry.worker_dsn), ) @property def general(self) -> GeneralConfig: return GeneralConfig( - environment=self.get_or_default(self.GENERAL_SECTION, 'environment'), - client_url=self.get_or_default(self.GENERAL_SECTION, 'clientUrl'), + environment=self.get(self.keys.general.environment), + client_url=self.get(self.keys.general.client_url), + secret=self.get(self.keys.general.secret), ) @property def mail(self): return MailConfig( - enabled=self.get_or_default(self.MAIL_SECTION, 'enabled'), - name=self.get_or_default(self.MAIL_SECTION, 'name'), - email=self.get_or_default(self.MAIL_SECTION, 'email'), - host=self.get_or_default(self.MAIL_SECTION, 'host'), - ssl=self.get_or_default(self.MAIL_SECTION, 'ssl'), - port=self.get_or_default(self.MAIL_SECTION, 'port'), - security=self.get_or_default(self.MAIL_SECTION, 'security'), - auth=self.get_or_default(self.MAIL_SECTION, 'authEnabled'), - username=self.get_or_default(self.MAIL_SECTION, 'username'), - password=self.get_or_default(self.MAIL_SECTION, 'password'), - rate_limit_window=self.get_or_default( - self.MAIL_SECTION, 'rateLimit', 'window' - ), - rate_limit_count=self.get_or_default( - self.MAIL_SECTION, 'rateLimit', 'count' - ), - timeout=int(self.get_or_default(self.MAIL_SECTION, 'timeout')), + enabled=self.get(self.keys.mail.enabled), + name=self.get(self.keys.mail.name), + email=self.get(self.keys.mail.email), + host=self.get(self.keys.mail.host), + ssl=self.get(self.keys.mail.ssl), + port=self.get(self.keys.mail.port), + security=self.get(self.keys.mail.security), + auth=self.get(self.keys.mail.auth_enabled), + username=self.get(self.keys.mail.username), + password=self.get(self.keys.mail.password), + rate_limit_window=int(self.get(self.keys.mail.rate_limit_window)), + rate_limit_count=int(self.get(self.keys.mail.rate_limit_count)), + timeout=int(self.get(self.keys.mail.timeout)), ) diff --git a/packages/dsw-data-seeder/dsw/data_seeder/cli.py b/packages/dsw-data-seeder/dsw/data_seeder/cli.py index eb376097..a810ca85 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/cli.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/cli.py @@ -1,7 +1,7 @@ +import click # type: ignore import pathlib -import click # type: ignore -from typing import IO +from typing import IO, Optional from dsw.config.parser import MissingConfigurationError @@ -10,8 +10,12 @@ from .logging import prepare_logging -def validate_config(ctx, param, value: IO) -> SeederConfig: - content = value.read() +def validate_config(ctx, param, value: Optional[IO]): + if value is None: + content = '' + else: + content = value.read() + value.close() parser = SeederConfigParser() if not parser.can_read(content): click.echo('Error: Cannot parse config file', err=True) @@ -30,8 +34,8 @@ def validate_config(ctx, param, value: IO) -> SeederConfig: @click.group(name=PROG_NAME) @click.version_option(version=VERSION) @click.option('-c', '--config', envvar='DSW_CONFIG', - type=click.File('r', encoding='utf-8'), - callback=validate_config) + required=False, callback=validate_config, + type=click.File('r', encoding='utf-8')) @click.option('-w', '--workdir', envvar='SEEDER_WORKDIR', type=click.Path(dir_okay=True, exists=True)) @click.pass_context diff --git a/packages/dsw-document-worker/dsw/document_worker/cli.py b/packages/dsw-document-worker/dsw/document_worker/cli.py index ecae6ceb..6772f688 100644 --- a/packages/dsw-document-worker/dsw/document_worker/cli.py +++ b/packages/dsw-document-worker/dsw/document_worker/cli.py @@ -3,7 +3,7 @@ import logging import sys -from typing import IO +from typing import IO, Optional from dsw.config.parser import MissingConfigurationError @@ -12,8 +12,12 @@ from .consts import VERSION -def validate_config(ctx, param, value: IO): - content = value.read() +def validate_config(ctx, param, value: Optional[IO]): + if value is None: + content = '' + else: + content = value.read() + value.close() parser = DocumentWorkerConfigParser() if not parser.can_read(content): click.echo('Error: Cannot parse config file', err=True) @@ -33,8 +37,8 @@ def validate_config(ctx, param, value: IO): @click.command(name='docworker') @click.version_option(version=VERSION) @click.argument('config', envvar='DOCWORKER_CONFIG', - type=click.File('r', encoding='utf-8'), - callback=validate_config) + required=False, callback=validate_config, + type=click.File('r', encoding='utf-8')) @click.argument('workdir', envvar='DOCWORKER_WORKDIR', type=click.Path(dir_okay=True, exists=True)) def main(config: DocumentWorkerConfig, workdir: str): diff --git a/packages/dsw-document-worker/dsw/document_worker/config.py b/packages/dsw-document-worker/dsw/document_worker/config.py index af87bcea..c8f1e380 100644 --- a/packages/dsw-document-worker/dsw/document_worker/config.py +++ b/packages/dsw-document-worker/dsw/document_worker/config.py @@ -1,24 +1,113 @@ import shlex -from typing import List, Optional +from typing import List, Optional, Type from dsw.config import DSWConfigParser +from dsw.config.keys import ConfigKey, ConfigKeys, ConfigKeysContainer,\ + cast_str, cast_optional_int, cast_bool, cast_optional_str from dsw.config.model import GeneralConfig, SentryConfig, DatabaseConfig,\ - S3Config, LoggingConfig, CloudConfig + S3Config, LoggingConfig, CloudConfig, ConfigModel from .consts import DocumentNamingStrategy -class DocumentsConfig: +class _DocumentsKeys(ConfigKeysContainer): + naming_strategy = ConfigKey( + yaml_path=['documents', 'naming', 'strategy'], + var_names=['DOCUMENTS_NAMING_STRATEGY'], + default='sanitize', + cast=cast_str, + ) + + +class _ExperimentalKeys(ConfigKeysContainer): + pdf_only = ConfigKey( + yaml_path=['experimental', 'pdfOnly'], + var_names=['EXPERIMENTAL_PDF_ONLY'], + default=False, + cast=cast_bool, + ) + job_timeout = ConfigKey( + yaml_path=['experimental', 'jobTimeout'], + var_names=['EXPERIMENTAL_JOB_TIMEOUT'], + default=None, + cast=cast_optional_int, + ) + max_doc_size = ConfigKey( + yaml_path=['experimental', 'maxDocumentSize'], + var_names=['EXPERIMENTAL_MAX_DOCUMENT_SIZE'], + default=None, + cast=cast_optional_int, + ) + pdf_watermark = ConfigKey( + yaml_path=['experimental', 'pdfWatermark'], + var_names=['EXPERIMENTAL_PDF_WATERMARK'], + default='/app/data/watermark.pdf', + cast=cast_optional_str, + ) + pdf_watermark_top = ConfigKey( + yaml_path=['experimental', 'pdfWatermarkTop'], + var_names=['EXPERIMENTAL_PDF_WATERMARK_TOP'], + default=True, + cast=cast_bool, + ) + + +class _CommandPandocKeys(ConfigKeysContainer): + executable = ConfigKey( + yaml_path=['externals', 'pandoc', 'executable'], + var_names=['PANDOC_EXECUTABLE'], + default='pandoc', + cast=cast_str, + ) + args = ConfigKey( + yaml_path=['externals', 'pandoc', 'args'], + var_names=['PANDOC_ARGS'], + default='--standalone', + cast=cast_str, + ) + timeout = ConfigKey( + yaml_path=['externals', 'pandoc', 'timeout'], + var_names=['PANDOC_TIMEOUT'], + default=None, + cast=cast_optional_int, + ) + + +class _CommandWkhtmltopdfKeys(ConfigKeysContainer): + executable = ConfigKey( + yaml_path=['externals', 'wkhtmltopdf', 'executable'], + var_names=['WKHTMLTOPDF_EXECUTABLE'], + default='wkhtmltopdf', + cast=cast_str, + ) + args = ConfigKey( + yaml_path=['externals', 'wkhtmltopdf', 'args'], + var_names=['WKHTMLTOPDF_ARGS'], + default='', + cast=cast_str, + ) + timeout = ConfigKey( + yaml_path=['externals', 'wkhtmltopdf', 'timeout'], + var_names=['WKHTMLTOPDF_TIMEOUT'], + default=None, + cast=cast_optional_int, + ) + + +class DocWorkerConfigKeys(ConfigKeys): + documents = _DocumentsKeys + experimental = _ExperimentalKeys + cmd_pandoc = _CommandPandocKeys + cmd_wkhtmltopdf = _CommandWkhtmltopdfKeys + + +class DocumentsConfig(ConfigModel): def __init__(self, naming_strategy: str): self.naming_strategy = DocumentNamingStrategy.get(naming_strategy) - def __str__(self): - return f'DocumentsConfig\n' \ - f'- naming_strategy = {self.naming_strategy}\n' - -class ExperimentalConfig: +class ExperimentalConfig(ConfigModel): def __init__(self, pdf_only: bool, job_timeout: Optional[int], max_doc_size: Optional[float], @@ -29,14 +118,6 @@ def __init__(self, pdf_only: bool, job_timeout: Optional[int], self.pdf_watermark = pdf_watermark self.pdf_watermark_top = pdf_watermark_top - def __str__(self): - return f'ExperimentalConfig\n' \ - f'- pdf_only = {self.pdf_only}\n' \ - f'- job_timeout = {self.job_timeout}\n' \ - f'- max_doc_size = {self.max_doc_size}\n' \ - f'- pdf_watermark = {self.pdf_watermark}\n' \ - f'- pdf_watermark_top = {self.pdf_watermark_top}\n' - class CommandConfig: @@ -49,12 +130,6 @@ def __init__(self, executable: str, args: str, timeout: float): def command(self) -> List[str]: return [self.executable] + shlex.split(self.args) - def __str__(self): - return f'CommandConfig\n' \ - f'- executable = {self.executable} ({type(self.executable)})\n' \ - f'- args = {self.args} ({type(self.args)})\n' \ - f'- timeout = {self.timeout} ({type(self.timeout)})\n' - class TemplateRequestsConfig: @@ -82,7 +157,6 @@ def __init__(self, ids: List[str], requests: TemplateRequestsConfig, @staticmethod def load(data: dict): - print(data) return TemplateConfig( ids=data.get('ids', []), requests=TemplateRequestsConfig.load( @@ -141,80 +215,51 @@ def __str__(self): class DocumentWorkerConfigParser(DSWConfigParser): - DOCS_SECTION = 'documents' - DOCS_NAMING_SUBSECTION = 'naming' - EXTERNAL_SECTION = 'externals' - PANDOC_SUBSECTION = 'pandoc' - WKHTMLTOPDF_SUBSECTION = 'wkhtmltopdf' TEMPLATES_SECTION = 'templates' - EXPERIMENTAL_SECTION = 'experimental' - - DEFAULTS = { - **DSWConfigParser.DEFAULTS, - DOCS_SECTION: { - DOCS_NAMING_SUBSECTION: { - 'strategy': 'sanitize' - } - }, - EXTERNAL_SECTION: { - PANDOC_SUBSECTION: { - 'executable': 'pandoc', - 'args': '--standalone', - 'timeout': None, - }, - WKHTMLTOPDF_SUBSECTION: { - 'executable': 'wkhtmltopdf', - 'args': '', - 'timeout': None, - }, - }, - TEMPLATES_SECTION: [], - EXPERIMENTAL_SECTION: { - 'pdfOnly': False, - 'jobTimeout': None, - 'maxDocumentSize': None, - 'pdfWatermark': '/app/data/watermark.pdf', - 'pdfWatermarkTop': True, - }, - } + + def __init__(self): + super().__init__(keys=DocWorkerConfigKeys) + self.keys = DocWorkerConfigKeys # type: Type[DocWorkerConfigKeys] @property def documents(self) -> DocumentsConfig: return DocumentsConfig( - naming_strategy=self.get_or_default(self.DOCS_SECTION, self.DOCS_NAMING_SUBSECTION, 'strategy') - ) - - def _command_config(self, *path: str) -> CommandConfig: - return CommandConfig( - executable=self.get_or_default(*path, 'executable'), - args=self.get_or_default(*path, 'args'), - timeout=self.get_or_default(*path, 'timeout'), + naming_strategy=self.get(self.keys.documents.naming_strategy) ) @property def pandoc(self) -> CommandConfig: - return self._command_config(self.EXTERNAL_SECTION, self.PANDOC_SUBSECTION) + return CommandConfig( + executable=self.get(self.keys.cmd_pandoc.executable), + args=self.get(self.keys.cmd_pandoc.args), + timeout=self.get(self.keys.cmd_pandoc.timeout), + ) @property def wkhtmltopdf(self) -> CommandConfig: - return self._command_config(self.EXTERNAL_SECTION, self.WKHTMLTOPDF_SUBSECTION) + return CommandConfig( + executable=self.get(self.keys.cmd_wkhtmltopdf.executable), + args=self.get(self.keys.cmd_wkhtmltopdf.args), + timeout=self.get(self.keys.cmd_wkhtmltopdf.timeout), + ) @property def templates(self) -> TemplatesConfig: - templates_data = self.get_or_default(self.TEMPLATES_SECTION) - templates = [TemplateConfig.load(data) for data in templates_data] return TemplatesConfig( - templates=templates, + templates=[ + TemplateConfig.load(data) + for data in self.cfg.get(self.TEMPLATES_SECTION, []) + ], ) @property def experimental(self) -> ExperimentalConfig: return ExperimentalConfig( - pdf_only=self.get_or_default(self.EXPERIMENTAL_SECTION, 'pdfOnly'), - job_timeout=self.get_or_default(self.EXPERIMENTAL_SECTION, 'jobTimeout'), - max_doc_size=self.get_or_default(self.EXPERIMENTAL_SECTION, 'maxDocumentSize'), - pdf_watermark=self.get_or_default(self.EXPERIMENTAL_SECTION, 'pdfWatermark'), - pdf_watermark_top=self.get_or_default(self.EXPERIMENTAL_SECTION, 'pdfWatermarkTop'), + pdf_only=self.get(self.keys.experimental.pdf_only), + job_timeout=self.get(self.keys.experimental.job_timeout), + max_doc_size=self.get(self.keys.experimental.max_doc_size), + pdf_watermark=self.get(self.keys.experimental.pdf_watermark), + pdf_watermark_top=self.get(self.keys.experimental.pdf_watermark_top), ) @property diff --git a/packages/dsw-mailer/dsw/mailer/cli.py b/packages/dsw-mailer/dsw/mailer/cli.py index 2ef2813e..63703d38 100644 --- a/packages/dsw-mailer/dsw/mailer/cli.py +++ b/packages/dsw-mailer/dsw/mailer/cli.py @@ -2,7 +2,7 @@ import json import pathlib -from typing import IO +from typing import IO, Optional from dsw.config.parser import MissingConfigurationError @@ -12,8 +12,12 @@ from .logging import prepare_logging -def validate_config(ctx, param, value: IO): - content = value.read() +def validate_config(ctx, param, value: Optional[IO]): + if value is None: + content = '' + else: + content = value.read() + value.close() parser = MailerConfigParser() if not parser.can_read(content): click.echo('Error: Cannot parse config file', err=True) @@ -44,8 +48,8 @@ def extract_message_request(ctx, param, value: IO): @click.pass_context @click.version_option(version=VERSION) @click.option('-c', '--config', envvar='DSW_CONFIG', - type=click.File('r', encoding='utf-8'), - callback=validate_config) + required=False, callback=validate_config, + type=click.File('r', encoding='utf-8')) @click.option('-w', '--workdir', envvar='MAILER_WORKDIR', type=click.Path(dir_okay=True, exists=True)) @click.option('-m', '--mode', envvar='MAILER_MODE', diff --git a/packages/dsw-mailer/dsw/mailer/config.py b/packages/dsw-mailer/dsw/mailer/config.py index 6a14ccba..f459e01e 100644 --- a/packages/dsw-mailer/dsw/mailer/config.py +++ b/packages/dsw-mailer/dsw/mailer/config.py @@ -1,4 +1,5 @@ from dsw.config import DSWConfigParser +from dsw.config.keys import ConfigKeys from dsw.config.model import GeneralConfig, SentryConfig, \ DatabaseConfig, LoggingConfig, MailConfig @@ -27,11 +28,11 @@ def __str__(self): class MailerConfigParser(DSWConfigParser): - REQUIRED = [ - [DSWConfigParser.MAIL_SECTION, 'email'], - [DSWConfigParser.MAIL_SECTION, 'host'], - [DSWConfigParser.MAIL_SECTION, 'port'], - ] + def __init__(self): + ConfigKeys.mail.email.required = True + ConfigKeys.mail.host.required = True + ConfigKeys.mail.port.required = True + super().__init__(keys=ConfigKeys) @property def config(self) -> MailerConfig: From cc01ea2cbf0069c16d32bcac17c8f92088551966 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Wed, 28 Dec 2022 09:02:55 +0100 Subject: [PATCH 6/8] Release 3.19.0-RC.1 --- packages/dsw-command-queue/CHANGELOG.md | 5 +++++ packages/dsw-command-queue/pyproject.toml | 4 ++-- packages/dsw-config/CHANGELOG.md | 11 +++++++++++ packages/dsw-config/pyproject.toml | 2 +- packages/dsw-data-seeder/CHANGELOG.md | 7 +++++++ packages/dsw-data-seeder/pyproject.toml | 10 +++++----- packages/dsw-database/CHANGELOG.md | 5 +++++ packages/dsw-database/pyproject.toml | 4 ++-- packages/dsw-document-worker/CHANGELOG.md | 7 +++++++ packages/dsw-document-worker/pyproject.toml | 10 +++++----- packages/dsw-mailer/CHANGELOG.md | 7 +++++++ packages/dsw-mailer/pyproject.toml | 8 ++++---- packages/dsw-storage/CHANGELOG.md | 5 +++++ packages/dsw-storage/pyproject.toml | 4 ++-- packages/dsw-tdk/CHANGELOG.md | 5 +++++ packages/dsw-tdk/pyproject.toml | 2 +- 16 files changed, 74 insertions(+), 22 deletions(-) diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md index 7cd644e6..7e951c3e 100644 --- a/packages/dsw-command-queue/CHANGELOG.md +++ b/packages/dsw-command-queue/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +Released for version consistency with other DSW tools. + ## [3.18.0] Released for version consistency with other DSW tools. @@ -59,3 +63,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index 9b6a66a9..2017f448 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -27,7 +27,7 @@ classifiers = [ requires-python = '>=3.7, <4' dependencies = [ # DSW - 'dsw-database==3.19.0.dev0', + 'dsw-database==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md index e5ab402a..10acda2c 100644 --- a/packages/dsw-config/CHANGELOG.md +++ b/packages/dsw-config/CHANGELOG.md @@ -8,6 +8,16 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +### Added + +- Support for environment variables configuration + +### Changed + +- Specification of config keys with various options + ## [3.18.0] Released for version consistency with other DSW tools. @@ -57,3 +67,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 81911ad0..906b6ed9 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md index 5bb03138..477e1b89 100644 --- a/packages/dsw-data-seeder/CHANGELOG.md +++ b/packages/dsw-data-seeder/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +### Changed + +- Adjusted for new `dsw-config` + ## [3.18.0] Released for version consistency with other DSW tools. @@ -100,3 +106,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index d324db57..7095e571 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -27,10 +27,10 @@ dependencies = [ 'click', 'tenacity', # DSW - 'dsw-command-queue==3.19.0.dev0', - 'dsw-config==3.19.0.dev0', - 'dsw-database==3.19.0.dev0', - 'dsw-storage==3.19.0.dev0', + 'dsw-command-queue==3.19.0rc1', + 'dsw-config==3.19.0rc1', + 'dsw-database==3.19.0rc1', + 'dsw-storage==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md index 1446a083..b939f1b0 100644 --- a/packages/dsw-database/CHANGELOG.md +++ b/packages/dsw-database/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +Released for version consistency with other DSW tools. + ## [3.18.0] Released for version consistency with other DSW tools. @@ -65,3 +69,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index 9f4e761d..ea850318 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -28,7 +28,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - 'dsw-config==3.19.0.dev0', + 'dsw-config==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md index 175070f9..dc8f5ec5 100644 --- a/packages/dsw-document-worker/CHANGELOG.md +++ b/packages/dsw-document-worker/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +### Changed + +- Adjusted for new `dsw-config` + ## [3.18.0] ### Fixed @@ -74,3 +80,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 16b4e1a3..8f5295f6 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -39,10 +39,10 @@ dependencies = [ 'tenacity', 'XlsxWriter', # DSW - 'dsw-command-queue==3.19.0.dev0', - 'dsw-config==3.19.0.dev0', - 'dsw-database==3.19.0.dev0', - 'dsw-storage==3.19.0.dev0', + 'dsw-command-queue==3.19.0rc1', + 'dsw-config==3.19.0rc1', + 'dsw-database==3.19.0rc1', + 'dsw-storage==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md index 05945d34..07c46662 100644 --- a/packages/dsw-mailer/CHANGELOG.md +++ b/packages/dsw-mailer/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +### Changed + +- Adjusted for new `dsw-config` + ## [3.18.0] Released for version consistency with other DSW tools. @@ -66,3 +72,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index d6e47e2e..215bc04c 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -29,9 +29,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - 'dsw-command-queue==3.19.0.dev0', - 'dsw-config==3.19.0.dev0', - 'dsw-database==3.19.0.dev0', + 'dsw-command-queue==3.19.0rc1', + 'dsw-config==3.19.0rc1', + 'dsw-database==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-storage/CHANGELOG.md b/packages/dsw-storage/CHANGELOG.md index 0112c33c..2441b0f4 100644 --- a/packages/dsw-storage/CHANGELOG.md +++ b/packages/dsw-storage/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [3.19.0] + +Released for version consistency with other DSW tools. + ## [3.18.0] Released for version consistency with other DSW tools. @@ -57,3 +61,4 @@ Released for version consistency with other DSW tools. [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index da0e810e..5926d91e 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -28,7 +28,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - 'dsw-config==3.19.0.dev0', + 'dsw-config==3.19.0rc1', ] [project.urls] diff --git a/packages/dsw-tdk/CHANGELOG.md b/packages/dsw-tdk/CHANGELOG.md index d495e150..bae069fe 100644 --- a/packages/dsw-tdk/CHANGELOG.md +++ b/packages/dsw-tdk/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [3.19.0] + +Released for version consistency with other DSW tools. + ## [3.18.0] Released for version consistency with other DSW tools. @@ -243,3 +247,4 @@ Initial DSW Template Development Kit (versioned as part of the [DSW platform](ht [3.16.0]: /../../tree/v3.16.0 [3.17.0]: /../../tree/v3.17.0 [3.18.0]: /../../tree/v3.18.0 +[3.19.0]: /../../tree/v3.19.0 diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index 8418092f..08bc3e09 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = '3.19.0.dev0' +version = '3.19.0rc1' description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit'] From ca703a9da4f4d5065e8fdf06cf42dc40a43de617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Wed, 28 Dec 2022 12:37:50 +0100 Subject: [PATCH 7/8] fix(mailer): Use DB config only for Wizard mode --- packages/dsw-config/dsw/config/keys.py | 10 +++++----- packages/dsw-config/dsw/config/model.py | 3 ++- packages/dsw-mailer/dsw/mailer/connection/smtp.py | 8 +++++--- packages/dsw-mailer/dsw/mailer/mailer.py | 10 ++++++---- 4 files changed, 18 insertions(+), 13 deletions(-) diff --git a/packages/dsw-config/dsw/config/keys.py b/packages/dsw-config/dsw/config/keys.py index 710216a0..c0b039bf 100644 --- a/packages/dsw-config/dsw/config/keys.py +++ b/packages/dsw-config/dsw/config/keys.py @@ -164,7 +164,7 @@ class _DatabaseKeys(ConfigKeysContainer): queue_timeout = ConfigKey( yaml_path=['database', 'queueTimeout'], var_names=['DATABASE_QUEUE_TIMEOUT'], - default=300, + default=180, cast=cast_int, ) @@ -235,12 +235,12 @@ class _MailKeys(ConfigKeysContainer): ssl = ConfigKey( yaml_path=['mail', 'ssl'], var_names=[], - cast=cast_str, + cast=cast_optional_str, ) security = ConfigKey( yaml_path=['mail', 'security'], var_names=['MAIL_SECURITY'], - cast=cast_str, + cast=cast_optional_str, ) auth_enabled = ConfigKey( yaml_path=['mail', 'authEnabled'], @@ -271,9 +271,9 @@ class _MailKeys(ConfigKeysContainer): cast=cast_int, ) timeout = ConfigKey( - yaml_path=['mail', 'enabled'], + yaml_path=['mail', 'timeout'], var_names=['MAIL_TIMEOUT'], - default=5, + default=10, cast=cast_int, ) diff --git a/packages/dsw-config/dsw/config/model.py b/packages/dsw-config/dsw/config/model.py index 6d556028..cb7fadd2 100644 --- a/packages/dsw-config/dsw/config/model.py +++ b/packages/dsw-config/dsw/config/model.py @@ -129,4 +129,5 @@ def __str__(self): f'- security = {self.security}\n' \ f'- auth = {self.auth}\n' \ f'- rate_limit_window = {self.rate_limit_window}\n' \ - f'- rate_limit_count = {self.rate_limit_count}\n' + f'- rate_limit_count = {self.rate_limit_count}\n' \ + f'- timeout = {self.timeout}\n' diff --git a/packages/dsw-mailer/dsw/mailer/connection/smtp.py b/packages/dsw-mailer/dsw/mailer/connection/smtp.py index 6ce4b691..07e816fa 100644 --- a/packages/dsw-mailer/dsw/mailer/connection/smtp.py +++ b/packages/dsw-mailer/dsw/mailer/connection/smtp.py @@ -42,11 +42,13 @@ def send(self, message: MailMessage, cfg: Optional[MailConfig]): Context.logger.info(f'Sending via SMTP: {used_cfg.host}:{used_cfg.port}') self._send(message, used_cfg) - def _send(self, mail: MailMessage, cfg: MailConfig): + @classmethod + def _send(cls, mail: MailMessage, cfg: MailConfig): if cfg.is_ssl: - return self._send_smtp_ssl(mail=mail, cfg=cfg) - return self._send_smtp(mail=mail, cfg=cfg) + return cls._send_smtp_ssl(mail=mail, cfg=cfg) + return cls._send_smtp(mail=mail, cfg=cfg) + @classmethod def _send_smtp_ssl(cls, mail: MailMessage, cfg: MailConfig): context = ssl.create_default_context() with smtplib.SMTP_SSL( diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index 0acce0ec..ccae2a4d 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -97,9 +97,11 @@ def _process_command(self, cmd: PersistentCommand): trigger='PersistentComment', ) # get mailer config from DB - cfg = _transform_mail_config( - cfg=app_ctx.db.get_mail_config(app_uuid=cmd.app_uuid), - ) + cfg = None + if Context.is_wizard_mode(): + cfg = _transform_mail_config( + cfg=app_ctx.db.get_mail_config(app_uuid=cmd.app_uuid), + ) Context.logger.debug(f'Config from DB: {cfg}') # update Sentry info SentryReporter.set_context('template', rq.template_name) @@ -507,7 +509,7 @@ def registry_link(self) -> str: def callback_link(self) -> Optional[str]: if self.callback_url is None: return None - return f'{self.client_url}/registry/signup/{self.org.id}/{self.code}' + return f'{self.callback_url}/registry/signup/{self.org.id}/{self.code}' @property def activation_link(self) -> Optional[str]: From 28abc6fa8ecbcc4f82165a13a8b8106da008b445 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 3 Jan 2023 09:02:31 +0100 Subject: [PATCH 8/8] Release 3.19.0 --- packages/dsw-command-queue/pyproject.toml | 4 ++-- packages/dsw-config/pyproject.toml | 2 +- packages/dsw-data-seeder/pyproject.toml | 10 +++++----- packages/dsw-database/pyproject.toml | 4 ++-- packages/dsw-document-worker/pyproject.toml | 10 +++++----- packages/dsw-mailer/pyproject.toml | 8 ++++---- packages/dsw-storage/pyproject.toml | 4 ++-- packages/dsw-tdk/pyproject.toml | 2 +- 8 files changed, 22 insertions(+), 22 deletions(-) diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index 2017f448..0c22a863 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = '3.19.0rc1' +version = '3.19.0' description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -27,7 +27,7 @@ classifiers = [ requires-python = '>=3.7, <4' dependencies = [ # DSW - 'dsw-database==3.19.0rc1', + 'dsw-database==3.19.0', ] [project.urls] diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 906b6ed9..77d1b699 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = '3.19.0rc1' +version = '3.19.0' description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index 7095e571..a6e05055 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = '3.19.0rc1' +version = '3.19.0' description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -27,10 +27,10 @@ dependencies = [ 'click', 'tenacity', # DSW - 'dsw-command-queue==3.19.0rc1', - 'dsw-config==3.19.0rc1', - 'dsw-database==3.19.0rc1', - 'dsw-storage==3.19.0rc1', + 'dsw-command-queue==3.19.0', + 'dsw-config==3.19.0', + 'dsw-database==3.19.0', + 'dsw-storage==3.19.0', ] [project.urls] diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index ea850318..65290046 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = '3.19.0rc1' +version = '3.19.0' description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -28,7 +28,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - 'dsw-config==3.19.0rc1', + 'dsw-config==3.19.0', ] [project.urls] diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 8f5295f6..f4f686a1 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = '3.19.0rc1' +version = '3.19.0' description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -39,10 +39,10 @@ dependencies = [ 'tenacity', 'XlsxWriter', # DSW - 'dsw-command-queue==3.19.0rc1', - 'dsw-config==3.19.0rc1', - 'dsw-database==3.19.0rc1', - 'dsw-storage==3.19.0rc1', + 'dsw-command-queue==3.19.0', + 'dsw-config==3.19.0', + 'dsw-database==3.19.0', + 'dsw-storage==3.19.0', ] [project.urls] diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index 215bc04c..3d166652 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = '3.19.0rc1' +version = '3.19.0' description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -29,9 +29,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - 'dsw-command-queue==3.19.0rc1', - 'dsw-config==3.19.0rc1', - 'dsw-database==3.19.0rc1', + 'dsw-command-queue==3.19.0', + 'dsw-config==3.19.0', + 'dsw-database==3.19.0', ] [project.urls] diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 5926d91e..fe425340 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = '3.19.0rc1' +version = '3.19.0' description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -28,7 +28,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - 'dsw-config==3.19.0rc1', + 'dsw-config==3.19.0', ] [project.urls] diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index 08bc3e09..58390740 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = '3.19.0rc1' +version = '3.19.0' description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit']