From 4c1dc0e8b11ddc0e150ec7fea14f6aec5e9d2dc1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Tue, 3 Oct 2023 12:41:39 +0200 Subject: [PATCH 1/8] [DSW-2095] feat: Rename app to tenant --- .../dsw-data-seeder/dsw/data_seeder/cli.py | 6 +- .../dsw-data-seeder/dsw/data_seeder/consts.py | 2 +- .../dsw-data-seeder/dsw/data_seeder/seeder.py | 58 +++++----- .../example/app_config_seed.sql | 1 - .../dsw-data-seeder/example/example.seed.json | 6 +- .../example/tenant_config_seed.sql | 1 + .../dsw-data-seeder/example/user_seed.sql | 4 +- .../dsw-database/dsw/database/database.py | 100 +++++++++--------- packages/dsw-database/dsw/database/model.py | 44 ++++---- .../dsw/document_worker/limits.py | 12 +-- .../document_worker/templates/templates.py | 56 +++++----- .../dsw/document_worker/worker.py | 36 +++---- packages/dsw-mailer/dsw/mailer/mailer.py | 8 +- packages/dsw-storage/dsw/storage/s3storage.py | 8 +- 14 files changed, 171 insertions(+), 171 deletions(-) delete mode 100644 packages/dsw-data-seeder/example/app_config_seed.sql create mode 100644 packages/dsw-data-seeder/example/tenant_config_seed.sql diff --git a/packages/dsw-data-seeder/dsw/data_seeder/cli.py b/packages/dsw-data-seeder/dsw/data_seeder/cli.py index 9c19f8c9..122a22e3 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/cli.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/cli.py @@ -57,13 +57,13 @@ def run(ctx: click.Context, recipe: str): @cli.command(name='seed', help='Seed data directly.') @click.option('-r', '--recipe', envvar='SEEDER_RECIPE') -@click.option('-a', '--app_uuid', default=NULL_UUID) +@click.option('-t', '--tenant-uuid', default=NULL_UUID) @click.pass_context -def seed(ctx: click.Context, recipe: str, app_uuid: str): +def seed(ctx: click.Context, recipe: str, tenant_uuid: str): cfg = ctx.obj['cfg'] workdir = ctx.obj['workdir'] seeder = DataSeeder(cfg=cfg, workdir=workdir) - seeder.seed(recipe_name=recipe, app_uuid=app_uuid) + seeder.seed(recipe_name=recipe, tenant_uuid=tenant_uuid) @cli.command(name='list', help='List recipes for data seeding.') diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py index 0932e68a..bdce74ce 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/consts.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/consts.py @@ -3,7 +3,7 @@ CMD_CHANNEL = 'data_seeder' DEFAULT_ENCODING = 'utf-8' DEFAULT_MIMETYPE = 'application/octet-stream' -DEFAULT_PLACEHOLDER = '<<|APP-ID|>>' +DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-data-seeder' VERSION = '3.28.0' diff --git a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py index 72134a79..40d22f57 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py @@ -48,14 +48,14 @@ class SeedRecipe: def __init__(self, name: str, description: str, root: pathlib.Path, db_scripts: dict[str, DBScript], db_placeholder: str, - s3_app_dir: Optional[pathlib.Path], s3_fname_replace: dict[str, str], + s3_dir: Optional[pathlib.Path], s3_fname_replace: dict[str, str], uuids_count: int, uuids_placeholder: Optional[str]): self.name = name self.description = description self.root = root self.db_scripts = db_scripts self.db_placeholder = db_placeholder - self.s3_app_dir = s3_app_dir + self.s3_dir = s3_dir self.s3_fname_replace = s3_fname_replace self._db_scripts_data = collections.OrderedDict() # type: dict[str, str] self.s3_objects = collections.OrderedDict() # type: dict[pathlib.Path, str] @@ -71,12 +71,12 @@ def _load_db_scripts(self): ) def _load_s3_object_names(self): - if self.s3_app_dir is None: + if self.s3_dir is None: return - for s3_object_path in self.s3_app_dir.glob('**/*'): + for s3_object_path in self.s3_dir.glob('**/*'): if s3_object_path.is_file(): target_object_name = str( - s3_object_path.relative_to(self.s3_app_dir).as_posix() + s3_object_path.relative_to(self.s3_dir).as_posix() ) for r_from, r_to in self.s3_fname_replace.items(): target_object_name = target_object_name.replace(r_from, r_to) @@ -99,15 +99,15 @@ def prepare(self): def run_prepare(self): self._prepare_uuids() - def _replace_db_script(self, script: str, app_uuid: str) -> str: - result = script.replace(self.db_placeholder, app_uuid) + def _replace_db_script(self, script: str, tenant_uuid: str) -> str: + result = script.replace(self.db_placeholder, tenant_uuid) for uuid_key, uuid_value in self.uuids_replacement.items(): result = result.replace(uuid_key, uuid_value) return result - def iterate_db_scripts(self, app_uuid: str): + def iterate_db_scripts(self, tenant_uuid: str): return ( - (script_id, self._replace_db_script(script, app_uuid)) + (script_id, self._replace_db_script(script, tenant_uuid)) for script_id, script in self._db_scripts_data.items() ) @@ -133,9 +133,9 @@ def __str__(self): f'{self.description}\n\n' \ f'DB SQL Scripts:\n' \ f'{scripts}\n' \ - f'DB APP UUID Placeholder: "{self.db_placeholder}"\n\n' \ - f'S3 App Dir:\n' \ - f'{self.s3_app_dir if self.s3_app_dir is not None else "[nothing]"}\n' \ + f'DB Tenant UUID Placeholder: "{self.db_placeholder}"\n\n' \ + f'S3 Directory:\n' \ + f'{self.s3_dir if self.s3_dir is not None else "[nothing]"}\n' \ f'S3 Filename Replace:\n' \ f'{replaces}' @@ -164,16 +164,16 @@ def load_from_json(recipe_file: pathlib.Path) -> 'SeedRecipe': else: s = DBScript(recipe_file.parent / filepath, target, index) db_scripts[s.id] = s - s3_app_dir = None - if 'appDir' in s3.keys(): - s3_app_dir = recipe_file.parent / s3['appDir'] + s3_dir = None + if 'dir' in s3.keys(): + s3_dir = recipe_file.parent / s3['dir'] return SeedRecipe( name=data['name'], description=data.get('description', ''), root=recipe_file.parent, db_scripts=db_scripts, - db_placeholder=db.get('appIdPlaceholder', DEFAULT_PLACEHOLDER), - s3_app_dir=s3_app_dir, + db_placeholder=db.get('tenantIdPlaceholder', DEFAULT_PLACEHOLDER), + s3_dir=s3_dir, s3_fname_replace=s3.get('filenameReplace', {}), uuids_count=data.get('uuids', {}).get('count', 0), uuids_placeholder=data.get('uuids', {}).get('placeholder', None), @@ -192,8 +192,8 @@ def create_default(): description='Default dummy recipe', root=pathlib.Path('/dev/null'), db_scripts={}, - db_placeholder='<<|APP-ID|>>', - s3_app_dir=pathlib.Path('/dev/null'), + db_placeholder=DEFAULT_PLACEHOLDER, + s3_dir=pathlib.Path('/dev/null'), s3_fname_replace={}, uuids_count=0, uuids_placeholder=None, @@ -262,10 +262,10 @@ def work(self, cmd: PersistentCommand): Context.get().update_trace_id(cmd.uuid) SentryReporter.set_context('cmd_uuid', cmd.uuid) self.recipe.run_prepare() - app_uuid = cmd.body['appUuid'] + tenant_uuid = cmd.body['tenantUuid'] LOG.info(f'Seeding recipe "{self.recipe.name}" ' - f'to app with UUID "{app_uuid}"') - self.execute(app_uuid) + f'to tenant with UUID "{tenant_uuid}"') + self.execute(tenant_uuid) Context.get().update_trace_id('-') SentryReporter.set_context('cmd_uuid', '-') @@ -280,13 +280,13 @@ def _update_component_info(): built_at=built_at, ) - def seed(self, recipe_name: str, app_uuid: str): + def seed(self, recipe_name: str, tenant_uuid: str): self._prepare_recipe(recipe_name) LOG.info(f'Executing recipe "{recipe_name}"') - self.execute(app_uuid=app_uuid) + self.execute(tenant_uuid=tenant_uuid) - def execute(self, app_uuid: str): - SentryReporter.set_context('app_uuid', app_uuid) + def execute(self, tenant_uuid: str): + SentryReporter.set_context('tenant_uuid', tenant_uuid) # Run SQL scripts app_ctx = Context.get().app cursor = app_ctx.db.conn_query.new_cursor(use_dict=True) @@ -294,7 +294,7 @@ def execute(self, app_uuid: str): used_targets = set() try: LOG.info('Running SQL scripts') - for script_id, sql_script in self.recipe.iterate_db_scripts(app_uuid): + for script_id, sql_script in self.recipe.iterate_db_scripts(tenant_uuid): LOG.debug(f' -> Executing script: {script_id}') script = self.recipe.db_scripts[script_id] if script.target in self.dbs.keys(): @@ -311,7 +311,7 @@ def execute(self, app_uuid: str): data = local_file.read_bytes() LOG.debug(f' -> Sending: {object_name}') app_ctx.s3.store_object( - app_uuid=app_uuid, + tenant_uuid=tenant_uuid, object_name=object_name, content_type=_guess_mimetype(local_file.name), data=data, @@ -332,5 +332,5 @@ def execute(self, app_uuid: str): for target in used_targets: self.dbs[target].conn_query.connection.commit() LOG.info('Data seeding done') - SentryReporter.set_context('app_uuid', '-') + SentryReporter.set_context('tenant_uuid', '-') cursor.close() diff --git a/packages/dsw-data-seeder/example/app_config_seed.sql b/packages/dsw-data-seeder/example/app_config_seed.sql deleted file mode 100644 index 75b446da..00000000 --- a/packages/dsw-data-seeder/example/app_config_seed.sql +++ /dev/null @@ -1 +0,0 @@ -UPDATE app_config SET organization = '{"affiliations": [], "name": "Seeded Organization", "organizationId": "seeded.org", "description": "My seeded organization description"}' WHERE uuid = '<<|APP-ID|>>'; diff --git a/packages/dsw-data-seeder/example/example.seed.json b/packages/dsw-data-seeder/example/example.seed.json index e200ef1d..41bf0754 100644 --- a/packages/dsw-data-seeder/example/example.seed.json +++ b/packages/dsw-data-seeder/example/example.seed.json @@ -11,13 +11,13 @@ "target": "other" }, { - "filename": "app_config_seed.sql" + "filename": "tenant_config_seed.sql" } ], - "appIdPlaceholder": "<<|APP-ID|>>" + "tenantIdPlaceholder": "<<|TENANT-ID|>>" }, "s3": { - "appDir": "app", + "dir": "app", "filenameReplace": { ":": "_" } diff --git a/packages/dsw-data-seeder/example/tenant_config_seed.sql b/packages/dsw-data-seeder/example/tenant_config_seed.sql new file mode 100644 index 00000000..ff05592e --- /dev/null +++ b/packages/dsw-data-seeder/example/tenant_config_seed.sql @@ -0,0 +1 @@ +UPDATE tenant_config SET organization = '{"affiliations": [], "name": "Seeded Organization", "organizationId": "seeded.org", "description": "My seeded organization description"}' WHERE uuid = '<<|TENANT-ID|>>'; diff --git a/packages/dsw-data-seeder/example/user_seed.sql b/packages/dsw-data-seeder/example/user_seed.sql index bdf0a429..498cd7d3 100644 --- a/packages/dsw-data-seeder/example/user_seed.sql +++ b/packages/dsw-data-seeder/example/user_seed.sql @@ -1,2 +1,2 @@ -DELETE FROM user_entity WHERE email = 'nikola.tesla@example.com' and app_uuid = '<<|APP-ID|>>'; -DELETE FROM user_entity WHERE email = 'isaac.newton@example.com' and app_uuid = '<<|APP-ID|>>'; +DELETE FROM user_entity WHERE email = 'nikola.tesla@example.com' and tenant_uuid = '<<|TENANT-ID|>>'; +DELETE FROM user_entity WHERE email = 'isaac.newton@example.com' and tenant_uuid = '<<|TENANT-ID|>>'; diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py index ddaeab73..904f1f3f 100644 --- a/packages/dsw-database/dsw/database/database.py +++ b/packages/dsw-database/dsw/database/database.py @@ -11,7 +11,7 @@ from .model import DBDocumentTemplate, DBDocumentTemplateFile, \ DBDocumentTemplateAsset, DBDocument, DBComponent, \ - DocumentState, DBAppConfig, DBAppLimits, DBSubmission, \ + DocumentState, DBTenantConfig, DBTenantLimits, DBSubmission, \ DBInstanceConfigMail, DBQuestionnaireSimple LOG = logging.getLogger(__name__) @@ -30,40 +30,40 @@ def wrap_json_data(data: dict): class Database: # TODO: refactor queries and models - SELECT_DOCUMENT = 'SELECT * FROM document WHERE uuid = %s AND app_uuid = %s LIMIT 1;' - SELECT_QTN_DOCUMENTS = 'SELECT * FROM document WHERE questionnaire_uuid = %s AND app_uuid = %s;' - SELECT_DOCUMENT_SUBMISSIONS = 'SELECT * FROM submission WHERE document_uuid = %s AND app_uuid = %s;' + SELECT_DOCUMENT = 'SELECT * FROM document WHERE uuid = %s AND tenant_uuid = %s LIMIT 1;' + SELECT_QTN_DOCUMENTS = 'SELECT * FROM document WHERE questionnaire_uuid = %s AND tenant_uuid = %s;' + SELECT_DOCUMENT_SUBMISSIONS = 'SELECT * FROM submission WHERE document_uuid = %s AND tenant_uuid = %s;' SELECT_QTN_SUBMISSIONS = 'SELECT s.* FROM document d JOIN submission s ON d.uuid = s.document_uuid ' \ - 'WHERE d.questionnaire_uuid = %s AND d.app_uuid = %s;' + 'WHERE d.questionnaire_uuid = %s AND d.tenant_uuid = %s;' SELECT_QTN_SIMPLE = 'SELECT qtn.* FROM questionnaire qtn ' \ - 'WHERE qtn.uuid = %s AND qtn.app_uuid = %s;' - SELECT_APP_CONFIG = 'SELECT * FROM app_config WHERE uuid = %(app_uuid)s LIMIT 1;' - SELECT_APP_LIMIT = 'SELECT uuid, storage FROM app_limit WHERE uuid = %(app_uuid)s LIMIT 1;' + 'WHERE qtn.uuid = %s AND qtn.tenant_uuid = %s;' + SELECT_TENANT_CONFIG = 'SELECT * FROM tenant_config WHERE uuid = %(tenant_uuid)s LIMIT 1;' + SELECT_TENANT_LIMIT = 'SELECT uuid, storage FROM tenant_limit_bundle WHERE uuid = %(tenant_uuid)s LIMIT 1;' UPDATE_DOCUMENT_STATE = 'UPDATE document SET state = %s, worker_log = %s WHERE uuid = %s;' UPDATE_DOCUMENT_RETRIEVED = 'UPDATE document SET retrieved_at = %s, state = %s WHERE uuid = %s;' UPDATE_DOCUMENT_FINISHED = 'UPDATE document SET finished_at = %s, state = %s, ' \ 'file_name = %s, content_type = %s, worker_log = %s, ' \ 'file_size = %s WHERE uuid = %s;' - SELECT_TEMPLATE = 'SELECT * FROM document_template WHERE id = %s AND app_uuid = %s LIMIT 1;' + SELECT_TEMPLATE = 'SELECT * FROM document_template WHERE id = %s AND tenant_uuid = %s LIMIT 1;' SELECT_TEMPLATE_FILES = 'SELECT * FROM document_template_file ' \ - 'WHERE document_template_id = %s AND app_uuid = %s;' + 'WHERE document_template_id = %s AND tenant_uuid = %s;' SELECT_TEMPLATE_ASSETS = 'SELECT * FROM document_template_asset ' \ - 'WHERE document_template_id = %s AND app_uuid = %s;' + 'WHERE document_template_id = %s AND tenant_uuid = %s;' CHECK_TABLE_EXISTS = 'SELECT EXISTS(SELECT * FROM information_schema.tables' \ ' WHERE table_name = %(table_name)s)' SELECT_MAIL_CONFIG = 'SELECT icm.* ' \ - 'FROM app_config ac JOIN instance_config_mail icm ' \ - 'ON ac.mail_config_uuid = icm.uuid ' \ - 'WHERE ac.uuid = %(app_uuid)s;' + 'FROM tenant_config tc JOIN instance_config_mail icm ' \ + 'ON tc.mail_config_uuid = icm.uuid ' \ + 'WHERE tc.uuid = %(tenant_uuid)s;' UPDATE_COMPONENT_INFO = 'INSERT INTO component (name, version, built_at, created_at, updated_at) ' \ 'VALUES (%(name)s, %(version)s, %(built_at)s, %(created_at)s, %(updated_at)s)' \ 'ON CONFLICT (name) DO ' \ 'UPDATE SET version = %(version)s, built_at = %(built_at)s, updated_at = %(updated_at)s;' SELECT_COMPONENT_INFO = 'SELECT * FROM component WHERE name = %(name)s;' SUM_FILE_SIZES = 'SELECT (SELECT COALESCE(SUM(file_size)::bigint, 0) ' \ - 'FROM document WHERE app_uuid = %(app_uuid)s) ' \ + 'FROM document WHERE tenant_uuid = %(tenant_uuid)s) ' \ '+ (SELECT COALESCE(SUM(file_size)::bigint, 0) ' \ - 'FROM document_template_asset WHERE app_uuid = %(app_uuid)s) ' \ + 'FROM document_template_asset WHERE tenant_uuid = %(tenant_uuid)s) ' \ 'as result;' def __init__(self, cfg: DatabaseConfig, connect: bool = True, @@ -120,11 +120,11 @@ def _check_table_exists(self, table_name: str) -> bool: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_document(self, document_uuid: str, app_uuid: str) -> Optional[DBDocument]: + def fetch_document(self, document_uuid: str, tenant_uuid: str) -> Optional[DBDocument]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_DOCUMENT, - params=(document_uuid, app_uuid), + params=(document_uuid, tenant_uuid), ) result = cursor.fetchall() if len(result) != 1: @@ -138,8 +138,8 @@ def fetch_document(self, document_uuid: str, app_uuid: str) -> Optional[DBDocume before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_app_config(self, app_uuid: str) -> Optional[DBAppConfig]: - return self.get_app_config(app_uuid) + def fetch_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: + return self.get_tenant_config(tenant_uuid) @tenacity.retry( reraise=True, @@ -148,16 +148,16 @@ def fetch_app_config(self, app_uuid: str) -> Optional[DBAppConfig]: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_app_limits(self, app_uuid: str) -> Optional[DBAppLimits]: + def fetch_tenant_limits(self, tenant_uuid: str) -> Optional[DBTenantLimits]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( - query=self.SELECT_APP_LIMIT, - params={'app_uuid': app_uuid}, + query=self.SELECT_TENANT_LIMIT, + params={'tenant_uuid': tenant_uuid}, ) result = cursor.fetchall() if len(result) != 1: return None - return DBAppLimits.from_dict_row(result[0]) + return DBTenantLimits.from_dict_row(result[0]) @tenacity.retry( reraise=True, @@ -167,12 +167,12 @@ def fetch_app_limits(self, app_uuid: str) -> Optional[DBAppLimits]: after=tenacity.after_log(LOG, logging.DEBUG), ) def fetch_template( - self, template_id: str, app_uuid: str + self, template_id: str, tenant_uuid: str ) -> Optional[DBDocumentTemplate]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE, - params=(template_id, app_uuid), + params=(template_id, tenant_uuid), ) result = cursor.fetchall() if len(result) != 1: @@ -187,12 +187,12 @@ def fetch_template( after=tenacity.after_log(LOG, logging.DEBUG), ) def fetch_template_files( - self, template_id: str, app_uuid: str + self, template_id: str, tenant_uuid: str ) -> List[DBDocumentTemplateFile]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE_FILES, - params=(template_id, app_uuid), + params=(template_id, tenant_uuid), ) return [DBDocumentTemplateFile.from_dict_row(x) for x in cursor.fetchall()] @@ -204,12 +204,12 @@ def fetch_template_files( after=tenacity.after_log(LOG, logging.DEBUG), ) def fetch_template_assets( - self, template_id: str, app_uuid: str + self, template_id: str, tenant_uuid: str ) -> List[DBDocumentTemplateAsset]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_TEMPLATE_ASSETS, - params=(template_id, app_uuid), + params=(template_id, tenant_uuid), ) return [DBDocumentTemplateAsset.from_dict_row(x) for x in cursor.fetchall()] @@ -220,11 +220,11 @@ def fetch_template_assets( before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_qtn_documents(self, questionnaire_uuid: str, app_uuid: str) -> List[DBDocument]: + def fetch_qtn_documents(self, questionnaire_uuid: str, tenant_uuid: str) -> List[DBDocument]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_QTN_DOCUMENTS, - params=(questionnaire_uuid, app_uuid), + params=(questionnaire_uuid, tenant_uuid), ) return [DBDocument.from_dict_row(x) for x in cursor.fetchall()] @@ -235,11 +235,11 @@ def fetch_qtn_documents(self, questionnaire_uuid: str, app_uuid: str) -> List[DB before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_document_submissions(self, document_uuid: str, app_uuid: str) -> List[DBSubmission]: + def fetch_document_submissions(self, document_uuid: str, tenant_uuid: str) -> List[DBSubmission]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_DOCUMENT_SUBMISSIONS, - params=(document_uuid, app_uuid), + params=(document_uuid, tenant_uuid), ) return [DBSubmission.from_dict_row(x) for x in cursor.fetchall()] @@ -250,11 +250,11 @@ def fetch_document_submissions(self, document_uuid: str, app_uuid: str) -> List[ before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_questionnaire_submissions(self, questionnaire_uuid: str, app_uuid: str) -> List[DBSubmission]: + def fetch_questionnaire_submissions(self, questionnaire_uuid: str, tenant_uuid: str) -> List[DBSubmission]: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_QTN_SUBMISSIONS, - params=(questionnaire_uuid, app_uuid), + params=(questionnaire_uuid, tenant_uuid), ) return [DBSubmission.from_dict_row(x) for x in cursor.fetchall()] @@ -265,11 +265,11 @@ def fetch_questionnaire_submissions(self, questionnaire_uuid: str, app_uuid: str before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def fetch_questionnaire_simple(self, questionnaire_uuid: str, app_uuid: str) -> DBQuestionnaireSimple: + def fetch_questionnaire_simple(self, questionnaire_uuid: str, tenant_uuid: str) -> DBQuestionnaireSimple: with self.conn_query.new_cursor(use_dict=True) as cursor: cursor.execute( query=self.SELECT_QTN_SIMPLE, - params=(questionnaire_uuid, app_uuid), + params=(questionnaire_uuid, tenant_uuid), ) return DBQuestionnaireSimple.from_dict_row(cursor.fetchone()) @@ -341,11 +341,11 @@ def update_document_finished( before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_currently_used_size(self, app_uuid: str): + def get_currently_used_size(self, tenant_uuid: str): with self.conn_query.new_cursor() as cursor: cursor.execute( query=self.SUM_FILE_SIZES, - params={'app_uuid': app_uuid}, + params={'tenant_uuid': tenant_uuid}, ) row = cursor.fetchone() return row[0] @@ -357,18 +357,18 @@ def get_currently_used_size(self, app_uuid: str): before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_app_config(self, app_uuid: str) -> Optional[DBAppConfig]: + def get_tenant_config(self, tenant_uuid: str) -> Optional[DBTenantConfig]: with self.conn_query.new_cursor(use_dict=True) as cursor: try: cursor.execute( - query=self.SELECT_APP_CONFIG, - params={'app_uuid': app_uuid}, + query=self.SELECT_TENANT_CONFIG, + params={'tenant_uuid': tenant_uuid}, ) result = cursor.fetchone() - return DBAppConfig.from_dict_row(data=result) + return DBTenantConfig.from_dict_row(data=result) except Exception as e: - LOG.warning(f'Could not retrieve app_config for app' - f' "{app_uuid}": {str(e)}') + LOG.warning(f'Could not retrieve tenant_config for tenant' + f' "{tenant_uuid}": {str(e)}') return None @tenacity.retry( @@ -378,22 +378,22 @@ def get_app_config(self, app_uuid: str) -> Optional[DBAppConfig]: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def get_mail_config(self, app_uuid: str) -> Optional[DBInstanceConfigMail]: + def get_mail_config(self, tenant_uuid: str) -> Optional[DBInstanceConfigMail]: with self.conn_query.new_cursor(use_dict=True) as cursor: if not self._check_table_exists(table_name='instance_config_mail'): return None try: cursor.execute( query=self.SELECT_MAIL_CONFIG, - params={'app_uuid': app_uuid}, + params={'tenant_uuid': tenant_uuid}, ) result = cursor.fetchone() if result is None: return None return DBInstanceConfigMail.from_dict_row(data=result) except Exception as e: - LOG.warning(f'Could not retrieve instance_config_mail for app' - f' "{app_uuid}": {str(e)}') + LOG.warning(f'Could not retrieve instance_config_mail for tenant' + f' "{tenant_uuid}": {str(e)}') return None @tenacity.retry( diff --git a/packages/dsw-database/dsw/database/model.py b/packages/dsw-database/dsw/database/model.py index 46df4854..94441a21 100644 --- a/packages/dsw-database/dsw/database/model.py +++ b/packages/dsw-database/dsw/database/model.py @@ -58,7 +58,7 @@ class DBDocument: retrieved_at: Optional[datetime.datetime] finished_at: Optional[datetime.datetime] created_at: datetime.datetime - app_uuid: str + tenant_uuid: str file_size: int @staticmethod @@ -80,7 +80,7 @@ def from_dict_row(data: dict): file_name=data['file_name'], content_type=data['content_type'], worker_log=data['worker_log'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), file_size=data['file_size'], ) @@ -101,7 +101,7 @@ class DBDocumentTemplate: phase: str created_at: datetime.datetime updated_at: datetime.datetime - app_uuid: str + tenant_uuid: str @property def is_draft(self): @@ -132,7 +132,7 @@ def from_dict_row(data: dict) -> 'DBDocumentTemplate': phase=data['phase'], created_at=data['created_at'], updated_at=data['updated_at'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) @@ -144,7 +144,7 @@ class DBDocumentTemplateFile: content: str created_at: datetime.datetime updated_at: datetime.datetime - app_uuid: str + tenant_uuid: str @staticmethod def from_dict_row(data: dict) -> 'DBDocumentTemplateFile': @@ -155,7 +155,7 @@ def from_dict_row(data: dict) -> 'DBDocumentTemplateFile': content=data['content'], created_at=data['created_at'], updated_at=data['updated_at'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) @@ -168,7 +168,7 @@ class DBDocumentTemplateAsset: file_size: int created_at: datetime.datetime updated_at: datetime.datetime - app_uuid: str + tenant_uuid: str @staticmethod def from_dict_row(data: dict) -> 'DBDocumentTemplateAsset': @@ -180,7 +180,7 @@ def from_dict_row(data: dict) -> 'DBDocumentTemplateAsset': file_size=data['file_size'], created_at=data['created_at'], updated_at=data['updated_at'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) @@ -194,7 +194,7 @@ class PersistentCommand: last_error_message: Optional[str] attempts: int max_attempts: int - app_uuid: str + tenant_uuid: str created_by: Optional[str] created_at: datetime.datetime updated_at: datetime.datetime @@ -213,12 +213,12 @@ def from_dict_row(data: dict): created_by=str(data['created_by']), created_at=data['created_at'], updated_at=data['updated_at'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) @dataclasses.dataclass -class DBAppConfig: +class DBTenantConfig: uuid: str organization: dict authentication: dict @@ -253,7 +253,7 @@ def support_email(self) -> Optional[str]: @staticmethod def from_dict_row(data: dict): - return DBAppConfig( + return DBTenantConfig( uuid=str(data['uuid']), organization=data['organization'], authentication=data['authentication'], @@ -273,14 +273,14 @@ def from_dict_row(data: dict): @dataclasses.dataclass -class DBAppLimits: - app_uuid: str +class DBTenantLimits: + tenant_uuid: str storage: Optional[int] @staticmethod def from_dict_row(data: dict): - return DBAppLimits( - app_uuid=str(data['uuid']), + return DBTenantLimits( + tenant_uuid=str(data['uuid']), storage=data['storage'], ) @@ -298,7 +298,7 @@ class DBSubmission: created_by: str created_at: datetime.datetime updated_at: datetime.datetime - app_uuid: str + tenant_uuid: str @staticmethod def from_dict_row(data: dict): @@ -312,7 +312,7 @@ def from_dict_row(data: dict): created_by=str(data['created_by']), created_at=data['created_at'], updated_at=data['updated_at'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) def to_dict(self) -> dict: @@ -326,7 +326,7 @@ def to_dict(self) -> dict: 'created_by': self.created_by, 'created_at': self.created_at.isoformat(timespec='milliseconds'), 'updated_at': self.updated_at.isoformat(timespec='milliseconds'), - 'app_uuid': self.app_uuid, + 'tenant_uuid': self.tenant_uuid, } @@ -349,7 +349,7 @@ class DBQuestionnaireSimple: description: str is_template: bool project_tags: list[str] - app_uuid: str + tenant_uuid: str @staticmethod def from_dict_row(data: dict): @@ -367,7 +367,7 @@ def from_dict_row(data: dict): description=data['description'], is_template=data['is_template'], project_tags=data['project_tags'], - app_uuid=str(data.get('app_uuid', NULL_UUID)), + tenant_uuid=str(data.get('tenant_uuid', NULL_UUID)), ) def to_dict(self) -> dict: @@ -385,7 +385,7 @@ def to_dict(self) -> dict: 'description': self.description, 'is_template': self.is_template, 'project_tags': self.project_tags, - 'app_uuid': self.app_uuid, + 'tenant_uuid': self.tenant_uuid, } diff --git a/packages/dsw-document-worker/dsw/document_worker/limits.py b/packages/dsw-document-worker/dsw/document_worker/limits.py index fb3255e5..532a8f03 100644 --- a/packages/dsw-document-worker/dsw/document_worker/limits.py +++ b/packages/dsw-document-worker/dsw/document_worker/limits.py @@ -1,4 +1,4 @@ -from dsw.database.database import DBAppConfig +from dsw.database.database import DBTenantConfig from .context import Context from .exceptions import JobException @@ -47,10 +47,10 @@ def timeout_exceeded(job_id: str): @staticmethod def check_format(job_id: str, doc_format: Format, - app_config: Optional[DBAppConfig]): + tenant_config: Optional[DBTenantConfig]): pdf_only = Context.get().app.cfg.experimental.pdf_only - if app_config is not None: - pdf_only = pdf_only or app_config.feature_pdf_only + if tenant_config is not None: + pdf_only = pdf_only or tenant_config.feature_pdf_only if not pdf_only or doc_format.is_pdf: return raise JobException( @@ -60,8 +60,8 @@ def check_format(job_id: str, doc_format: Format, @staticmethod def make_watermark(doc_pdf: bytes, - app_config: Optional[DBAppConfig]) -> bytes: + tenant_config: Optional[DBTenantConfig]) -> bytes: watermark = Context.get().app.cfg.experimental.pdf_watermark - if watermark is None or app_config is None or not app_config.feature_pdf_watermark: + if watermark is None or tenant_config is None or not tenant_config.feature_pdf_watermark: return doc_pdf return PdfWaterMarker.create_watermark(doc_pdf=doc_pdf) diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/templates.py b/packages/dsw-document-worker/dsw/document_worker/templates/templates.py index 00616ed3..427ac847 100644 --- a/packages/dsw-document-worker/dsw/document_worker/templates/templates.py +++ b/packages/dsw-document-worker/dsw/document_worker/templates/templates.py @@ -57,9 +57,9 @@ def __init__(self, db_template, db_files, db_assets): class Template: - def __init__(self, app_uuid: str, template_dir: pathlib.Path, + def __init__(self, tenant_uuid: str, template_dir: pathlib.Path, db_template: TemplateComposite): - self.app_uuid = app_uuid + self.tenant_uuid = tenant_uuid self.template_dir = template_dir self.last_used = datetime.datetime.utcnow() self.db_template = db_template @@ -67,7 +67,7 @@ def __init__(self, app_uuid: str, template_dir: pathlib.Path, self.formats = dict() # type: dict[str, Format] self.asset_prefix = f'templates/{self.db_template.template.id}' if Context.get().app.cfg.cloud.multi_tenant: - self.asset_prefix = f'{self.app_uuid}/{self.asset_prefix}' + self.asset_prefix = f'{self.tenant_uuid}/{self.asset_prefix}' def raise_exc(self, message: str): raise TemplateException(self.template_id, message) @@ -233,40 +233,40 @@ def get(cls) -> 'TemplateRegistry': def __init__(self): self._templates = dict() # type: dict[str, dict[str, Template]] - def has_template(self, app_uuid: str, template_id: str) -> bool: - return app_uuid in self._templates.keys() and \ - template_id in self._templates[app_uuid].keys() + def has_template(self, tenant_uuid: str, template_id: str) -> bool: + return tenant_uuid in self._templates.keys() and \ + template_id in self._templates[tenant_uuid].keys() - def _set_template(self, app_uuid: str, template_id: str, template: Template): - if app_uuid not in self._templates.keys(): - self._templates[app_uuid] = dict() - self._templates[app_uuid][template_id] = template + def _set_template(self, tenant_uuid: str, template_id: str, template: Template): + if tenant_uuid not in self._templates.keys(): + self._templates[tenant_uuid] = dict() + self._templates[tenant_uuid][template_id] = template - def get_template(self, app_uuid: str, template_id: str) -> Template: - return self._templates[app_uuid][template_id] + def get_template(self, tenant_uuid: str, template_id: str) -> Template: + return self._templates[tenant_uuid][template_id] - def _init_new_template(self, app_uuid: str, template_id: str, + def _init_new_template(self, tenant_uuid: str, template_id: str, db_template: TemplateComposite): workdir = Context.get().app.workdir - template_dir = workdir / app_uuid / template_id.replace(':', '_') + template_dir = workdir / tenant_uuid / template_id.replace(':', '_') template = Template( - app_uuid=app_uuid, + tenant_uuid=tenant_uuid, template_dir=template_dir, db_template=db_template, ) template.prepare_fs() - self._set_template(app_uuid, template_id, template) + self._set_template(tenant_uuid, template_id, template) - def _refresh_template(self, app_uuid: str, template_id: str, + def _refresh_template(self, tenant_uuid: str, template_id: str, db_template: TemplateComposite): - template = self.get_template(app_uuid, template_id) + template = self.get_template(tenant_uuid, template_id) template.update_template(db_template) - def prepare_template(self, app_uuid: str, template_id: str) -> Template: + def prepare_template(self, tenant_uuid: str, template_id: str) -> Template: ctx = Context.get() query_args = dict( template_id=template_id, - app_uuid=app_uuid, + tenant_uuid=tenant_uuid, ) db_template = ctx.app.db.fetch_template(**query_args) if db_template is None: @@ -279,22 +279,22 @@ def prepare_template(self, app_uuid: str, template_id: str) -> Template: db_assets={f.uuid: f for f in db_assets}, ) - if self.has_template(app_uuid, template_id): - self._refresh_template(app_uuid, template_id, template_composite) + if self.has_template(tenant_uuid, template_id): + self._refresh_template(tenant_uuid, template_id, template_composite) else: - self._init_new_template(app_uuid, template_id, template_composite) + self._init_new_template(tenant_uuid, template_id, template_composite) - return self.get_template(app_uuid, template_id) + return self.get_template(tenant_uuid, template_id) - def _clear_template(self, app_uuid: str, template_id: str): - template = self._templates[app_uuid].pop(template_id) + def _clear_template(self, tenant_uuid: str, template_id: str): + template = self._templates[tenant_uuid].pop(template_id) if template.template_dir.exists(): shutil.rmtree(template.template_dir) def cleanup(self): # TODO: configurable threshold = datetime.datetime.utcnow() - datetime.timedelta(days=7) - for app_uuid, templates in self._templates.items(): + for tenant_uuid, templates in self._templates.items(): for template_id, template in templates.items(): if template.last_used < threshold: - self._clear_template(app_uuid, template_id) + self._clear_template(tenant_uuid, template_id) diff --git a/packages/dsw-document-worker/dsw/document_worker/worker.py b/packages/dsw-document-worker/dsw/document_worker/worker.py index ed094fe5..3db0fa8b 100644 --- a/packages/dsw-document-worker/dsw/document_worker/worker.py +++ b/packages/dsw-document-worker/dsw/document_worker/worker.py @@ -9,8 +9,8 @@ from dsw.command_queue import CommandWorker, CommandQueue from dsw.config.sentry import SentryReporter from dsw.database.database import Database -from dsw.database.model import DBDocument, DBAppConfig, DBAppLimits, \ - PersistentCommand +from dsw.database.model import DBDocument, DBTenantConfig, \ + DBTenantLimits, PersistentCommand from dsw.storage import S3Storage from .build_info import BUILD_INFO @@ -54,13 +54,13 @@ def __init__(self, command: PersistentCommand): self.ctx = Context.get() self.template = None # type: Optional[Template] self.format = None # type: Optional[Format] - self.app_uuid = command.app_uuid # type: str + self.tenant_uuid = command.tenant_uuid # type: str self.doc_uuid = command.body['uuid'] # type: str self.doc_context = command.body # type: dict self.doc = None # type: Optional[DBDocument] self.final_file = None # type: Optional[DocumentFile] - self.app_config = None # type: Optional[DBAppConfig] - self.app_limits = None # type: Optional[DBAppLimits] + self.tenant_config = None # type: Optional[DBTenantConfig] + self.tenant_limits = None # type: Optional[DBTenantLimits] @property def safe_doc(self) -> DBDocument: @@ -91,12 +91,12 @@ def get_document(self): SentryReporter.set_context('template', '') SentryReporter.set_context('format', '') SentryReporter.set_context('document', '') - if self.app_uuid != NULL_UUID: - LOG.info(f'Limiting to app with UUID: {self.app_uuid}') + if self.tenant_uuid != NULL_UUID: + LOG.info(f'Limiting to tenant with UUID: {self.tenant_uuid}') LOG.info(f'Getting the document "{self.doc_uuid}" details from DB') self.doc = self.ctx.app.db.fetch_document( document_uuid=self.doc_uuid, - app_uuid=self.app_uuid, + tenant_uuid=self.tenant_uuid, ) if self.doc is None: raise create_job_exception( @@ -129,19 +129,19 @@ def prepare_template(self): SentryReporter.set_context('document', self.doc_uuid) # prepare template template = TemplateRegistry.get().prepare_template( - app_uuid=self.app_uuid, + tenant_uuid=self.tenant_uuid, template_id=template_id, ) # prepare format template.prepare_format(format_uuid) self.format = template.formats.get(format_uuid) # check limits (PDF-only) - self.app_config = self.ctx.app.db.fetch_app_config(app_uuid=self.app_uuid) - self.app_limits = self.ctx.app.db.fetch_app_limits(app_uuid=self.app_uuid) + self.tenant_config = self.ctx.app.db.fetch_tenant_config(tenant_uuid=self.tenant_uuid) + self.tenant_limits = self.ctx.app.db.fetch_tenant_limits(tenant_uuid=self.tenant_uuid) LimitsEnforcer.check_format( job_id=self.doc_uuid, doc_format=self.safe_format, - app_config=self.app_config, + tenant_config=self.tenant_config, ) # finalize self.template = template @@ -151,13 +151,13 @@ def _enrich_context(self): if self.safe_format.requires_via_extras('submissions'): submissions = self.ctx.app.db.fetch_questionnaire_submissions( questionnaire_uuid=self.safe_doc.questionnaire_uuid, - app_uuid=self.app_uuid, + tenant_uuid=self.tenant_uuid, ) extras['submissions'] = [s.to_dict() for s in submissions] if self.safe_format.requires_via_extras('questionnaire'): questionnaire = self.ctx.app.db.fetch_questionnaire_simple( questionnaire_uuid=self.safe_doc.questionnaire_uuid, - app_uuid=self.app_uuid, + tenant_uuid=self.tenant_uuid, ) extras['questionnaire'] = questionnaire.to_dict() self.doc_context['extras'] = extras @@ -178,8 +178,8 @@ def build_document(self): job_id=self.doc_uuid, doc_size=final_file.byte_size, ) - limit_size = None if self.app_limits is None else self.app_limits.storage - used_size = self.ctx.app.db.get_currently_used_size(app_uuid=self.app_uuid) + limit_size = None if self.tenant_limits is None else self.tenant_limits.storage + used_size = self.ctx.app.db.get_currently_used_size(tenant_uuid=self.tenant_uuid) LimitsEnforcer.check_size_usage( job_id=self.doc_uuid, doc_size=final_file.byte_size, @@ -190,7 +190,7 @@ def build_document(self): if self.safe_format.is_pdf: final_file.content = LimitsEnforcer.make_watermark( doc_pdf=final_file.content, - app_config=self.app_config, + tenant_config=self.tenant_config, ) # finalize self.final_file = final_file @@ -203,7 +203,7 @@ def store_document(self): self.ctx.app.s3.ensure_bucket() LOG.info(f'Storing document to S3 bucket {s3_id}') self.ctx.app.s3.store_document( - app_uuid=self.app_uuid, + tenant_uuid=self.tenant_uuid, file_name=self.doc_uuid, content_type=final_file.object_content_type, data=final_file.content, diff --git a/packages/dsw-mailer/dsw/mailer/mailer.py b/packages/dsw-mailer/dsw/mailer/mailer.py index bbf5aa0f..3de29b0d 100644 --- a/packages/dsw-mailer/dsw/mailer/mailer.py +++ b/packages/dsw-mailer/dsw/mailer/mailer.py @@ -94,7 +94,7 @@ def work(self, cmd: PersistentCommand): ) # get mailer config from DB cfg = _transform_mail_config( - cfg=app_ctx.db.get_mail_config(app_uuid=cmd.app_uuid), + cfg=app_ctx.db.get_mail_config(tenant_uuid=cmd.tenant_uuid), ) LOG.debug(f'Config from DB: {cfg}') # client URL @@ -175,12 +175,12 @@ def hit(self): class MailerCommand: def __init__(self, recipients: list[str], mode: str, template: str, ctx: dict, - app_uuid: str, cmd_uuid: str): + tenant_uuid: str, cmd_uuid: str): self.mode = mode self.template = template self.recipients = recipients self.ctx = ctx - self.app_uuid = app_uuid + self.tenant_uuid = tenant_uuid self.cmd_uuid = cmd_uuid self._enrich_context() @@ -214,7 +214,7 @@ def load(cmd: PersistentCommand) -> 'MailerCommand': template=cmd.body['template'], recipients=cmd.body['recipients'], ctx=cmd.body['parameters'], - app_uuid=cmd.app_uuid, + tenant_uuid=cmd.tenant_uuid, cmd_uuid=cmd.uuid, ) except KeyError as e: diff --git a/packages/dsw-storage/dsw/storage/s3storage.py b/packages/dsw-storage/dsw/storage/s3storage.py index c43a2d9e..ac0b8a9f 100644 --- a/packages/dsw-storage/dsw/storage/s3storage.py +++ b/packages/dsw-storage/dsw/storage/s3storage.py @@ -69,12 +69,12 @@ def ensure_bucket(self): before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def store_document(self, app_uuid: str, file_name: str, + def store_document(self, tenant_uuid: str, file_name: str, content_type: str, data: bytes, metadata: Optional[dict] = None): object_name = f'{DOCUMENTS_DIR}/{file_name}' if self.multi_tenant: - object_name = f'{app_uuid}/{object_name}' + object_name = f'{tenant_uuid}/{object_name}' with temp_binary_file(data=data) as file: self.client.put_object( bucket_name=self.cfg.bucket, @@ -112,11 +112,11 @@ def download_file(self, file_name: str, target_path: pathlib.Path) -> bool: before=tenacity.before_log(LOG, logging.DEBUG), after=tenacity.after_log(LOG, logging.DEBUG), ) - def store_object(self, app_uuid: str, object_name: str, + def store_object(self, tenant_uuid: str, object_name: str, content_type: str, data: bytes, metadata: Optional[dict] = None): if self.multi_tenant: - object_name = f'{app_uuid}/{object_name}' + object_name = f'{tenant_uuid}/{object_name}' with io.BytesIO(data) as file: self.client.put_object( bucket_name=self.cfg.bucket, From f83fb1ee028f19b68e14699bbb6e3025396e3bdc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 6 Oct 2023 08:58:16 +0200 Subject: [PATCH 2/8] deps: Update dependencies --- packages/dsw-command-queue/requirements.txt | 4 ++-- packages/dsw-data-seeder/requirements.txt | 6 +++--- packages/dsw-database/requirements.txt | 4 ++-- packages/dsw-document-worker/requirements.txt | 18 +++++++++--------- packages/dsw-mailer/requirements.txt | 4 ++-- packages/dsw-storage/requirements.txt | 2 +- packages/dsw-tdk/requirements.txt | 2 +- requirements.txt | 18 +++++++++--------- 8 files changed, 29 insertions(+), 29 deletions(-) diff --git a/packages/dsw-command-queue/requirements.txt b/packages/dsw-command-queue/requirements.txt index 7047ef23..dfd7de40 100644 --- a/packages/dsw-command-queue/requirements.txt +++ b/packages/dsw-command-queue/requirements.txt @@ -1,5 +1,5 @@ -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 PyYAML==6.0.1 tenacity==8.2.3 typing_extensions==4.8.0 diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt index 7e498bca..5fb54a8f 100644 --- a/packages/dsw-data-seeder/requirements.txt +++ b/packages/dsw-data-seeder/requirements.txt @@ -1,9 +1,9 @@ certifi==2023.7.22 click==8.1.7 -minio==7.1.16 +minio==7.1.17 python-dateutil==2.8.2 -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 PyYAML==6.0.1 sentry-sdk==1.31.0 six==1.16.0 diff --git a/packages/dsw-database/requirements.txt b/packages/dsw-database/requirements.txt index 7047ef23..dfd7de40 100644 --- a/packages/dsw-database/requirements.txt +++ b/packages/dsw-database/requirements.txt @@ -1,5 +1,5 @@ -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 PyYAML==6.0.1 tenacity==8.2.3 typing_extensions==4.8.0 diff --git a/packages/dsw-document-worker/requirements.txt b/packages/dsw-document-worker/requirements.txt index ef8f2c20..de22d74c 100644 --- a/packages/dsw-document-worker/requirements.txt +++ b/packages/dsw-document-worker/requirements.txt @@ -1,10 +1,10 @@ Brotli==1.0.9 certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 +cffi==1.16.0 +charset-normalizer==3.3.0 click==8.1.7 cssselect2==0.7.0 -fonttools==4.42.1 +fonttools==4.43.0 html5lib==1.1 idna==3.4 isodate==0.6.1 @@ -12,14 +12,14 @@ Jinja2==3.1.2 Markdown==3.4.4 MarkupSafe==2.1.3 mdx-breakless-lists==1.0.1 -minio==7.1.16 +minio==7.1.17 pathvalidate==3.2.0 pdfrw==0.4 Pillow==10.0.1 -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 pycparser==2.21 -pydyf==0.7.0 +pydyf==0.8.0 pyparsing==3.1.1 pyphen==0.14.0 python-dateutil==2.8.2 @@ -36,7 +36,7 @@ tinycss2==1.2.1 typing_extensions==4.8.0 tzdata==2023.3 urllib3==2.0.6 -weasyprint==59.0 +weasyprint==60.1 webencodings==0.5.1 -XlsxWriter==3.1.4 +XlsxWriter==3.1.6 zopfli==0.2.3 diff --git a/packages/dsw-mailer/requirements.txt b/packages/dsw-mailer/requirements.txt index 082f7115..aa4bcc09 100644 --- a/packages/dsw-mailer/requirements.txt +++ b/packages/dsw-mailer/requirements.txt @@ -5,8 +5,8 @@ dnspython==2.4.2 Jinja2==3.1.2 MarkupSafe==2.1.3 pathvalidate==3.2.0 -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 python-dateutil==2.8.2 PyYAML==6.0.1 sentry-sdk==1.31.0 diff --git a/packages/dsw-storage/requirements.txt b/packages/dsw-storage/requirements.txt index e6c49803..a5a92574 100644 --- a/packages/dsw-storage/requirements.txt +++ b/packages/dsw-storage/requirements.txt @@ -1,5 +1,5 @@ certifi==2023.7.22 -minio==7.1.16 +minio==7.1.17 PyYAML==6.0.1 tenacity==8.2.3 urllib3==2.0.6 diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 4aa52924..635b4909 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -3,7 +3,7 @@ aiosignal==1.3.1 anyio==3.7.1 async-timeout==4.0.3 attrs==23.1.0 -charset-normalizer==3.2.0 +charset-normalizer==3.3.0 click==8.1.7 colorama==0.4.6 frozenlist==1.4.0 diff --git a/requirements.txt b/requirements.txt index db42faf3..3dc2a505 100644 --- a/requirements.txt +++ b/requirements.txt @@ -5,14 +5,14 @@ async-timeout==4.0.3 attrs==23.1.0 Brotli==1.0.9 certifi==2023.7.22 -cffi==1.15.1 -charset-normalizer==3.2.0 +cffi==1.16.0 +charset-normalizer==3.3.0 click==8.1.7 colorama==0.4.6 cssselect2==0.7.0 dkimpy==1.1.5 dnspython==2.4.2 -fonttools==4.42.1 +fonttools==4.43.0 frozenlist==1.4.0 html5lib==1.1 humanize==4.8.0 @@ -22,16 +22,16 @@ Jinja2==3.1.2 Markdown==3.4.4 MarkupSafe==2.1.3 mdx-breakless-lists==1.0.1 -minio==7.1.16 +minio==7.1.17 multidict==6.0.4 pathspec==0.11.2 pathvalidate==3.2.0 pdfrw==0.4 Pillow==10.0.1 -psycopg==3.1.10 -psycopg-binary==3.1.10 +psycopg==3.1.12 +psycopg-binary==3.1.12 pycparser==2.21 -pydyf==0.7.0 +pydyf==0.8.0 pyparsing==3.1.1 pyphen==0.14.0 python-dateutil==2.8.2 @@ -51,8 +51,8 @@ typing_extensions==4.8.0 tzdata==2023.3 urllib3==2.0.6 watchgod==0.8.2 -weasyprint==59.0 +weasyprint==60.1 webencodings==0.5.1 -XlsxWriter==3.1.4 +XlsxWriter==3.1.6 yarl==1.9.2 zopfli==0.2.3 From f43679902c19c2a18d48d697caa1676c2e905840 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 6 Oct 2023 10:18:54 +0200 Subject: [PATCH 3/8] [DSW-2102] fix(tdk): Fix TDK watch mode termination --- .github/test.sh | 4 ++-- packages/dsw-tdk/dsw/tdk/cli.py | 25 ++++++++++++++++++------- packages/dsw-tdk/dsw/tdk/core.py | 21 ++++++++++++++------- packages/dsw-tdk/pyproject.toml | 2 +- packages/dsw-tdk/requirements.txt | 4 ++-- requirements.txt | 3 +-- 6 files changed, 38 insertions(+), 21 deletions(-) diff --git a/.github/test.sh b/.github/test.sh index 71914add..512b15e2 100644 --- a/.github/test.sh +++ b/.github/test.sh @@ -5,11 +5,11 @@ if make verify; then echo "## Verify: passed" else echo "## Verify: failed" - echo "VERIFY_FAILS=command-queue:$VERIFY_FAILS" >> $GITHUB_ENV + echo "VERIFY_FAILS=$1:$VERIFY_FAILS" >> $GITHUB_ENV fi if make test; then echo "## Test: passed" else echo "## Test: failed" - echo "TEST_FAILS=command-queue:$TEST_FAILS" >> $GITHUB_ENV + echo "TEST_FAILS=$1:$TEST_FAILS" >> $GITHUB_ENV fi diff --git a/packages/dsw-tdk/dsw/tdk/cli.py b/packages/dsw-tdk/dsw/tdk/cli.py index 3a02fa4c..5d192585 100644 --- a/packages/dsw-tdk/dsw/tdk/cli.py +++ b/packages/dsw-tdk/dsw/tdk/cli.py @@ -1,4 +1,5 @@ import asyncio +import signal import click # type: ignore import datetime @@ -8,7 +9,7 @@ import mimetypes import pathlib import slugify -import watchgod # type: ignore +import watchfiles # type: ignore from typing import Dict @@ -31,9 +32,9 @@ class ClickPrinter: CHANGE_SIGNS = { - watchgod.Change.added: click.style('+', fg='green'), - watchgod.Change.modified: click.style('*', fg='yellow'), - watchgod.Change.deleted: click.style('-', fg='red'), + watchfiles.Change.added: click.style('+', fg='green'), + watchfiles.Change.modified: click.style('*', fg='yellow'), + watchfiles.Change.deleted: click.style('-', fg='red'), } @staticmethod @@ -61,7 +62,7 @@ def watch(message: str): click.echo(f': {message}') @classmethod - def watch_change(cls, change_type: watchgod.Change, filepath: pathlib.Path, root: pathlib.Path): + def watch_change(cls, change_type: watchfiles.Change, filepath: pathlib.Path, root: pathlib.Path): timestamp = datetime.datetime.now().isoformat(timespec='milliseconds') sign = cls.CHANGE_SIGNS[change_type] click.secho('WATCH', fg='blue', bold=True, nl=False) @@ -348,6 +349,7 @@ async def main_routine(): @click.pass_context def put_template(ctx, api_url, template_dir, api_key, force, watch): tdk = TDKCore(logger=ctx.obj.logger) + stop_event = asyncio.Event() async def watch_callback(changes): changes = list(changes) @@ -370,7 +372,7 @@ async def main_routine(): if watch: ClickPrinter.watch('Entering watch mode... (press Ctrl+C to abort)') - await tdk.watch_project(watch_callback) + await tdk.watch_project(watch_callback, stop_event) await tdk.safe_client.close() except TDKProcessingError as e: @@ -387,8 +389,17 @@ async def main_routine(): await tdk.safe_client.safe_close() exit(1) + def set_stop_event(signum, frame): + signame = signal.Signals(signum).name + ClickPrinter.warning(f'Got {signame}, finishing... Bye!') + stop_event.set() + + signal.signal(signal.SIGINT, set_stop_event) + signal.signal(signal.SIGABRT, set_stop_event) + loop = asyncio.get_event_loop() - loop.run_until_complete(main_routine()) + main_task = asyncio.ensure_future(main_routine()) + loop.run_until_complete(main_task) @main.command(help='Create ZIP package for a template.', name='package') diff --git a/packages/dsw-tdk/dsw/tdk/core.py b/packages/dsw-tdk/dsw/tdk/core.py index 18ee0896..b4f5867a 100644 --- a/packages/dsw-tdk/dsw/tdk/core.py +++ b/packages/dsw-tdk/dsw/tdk/core.py @@ -7,9 +7,10 @@ import re import shutil import tempfile -import watchgod # type: ignore +import watchfiles # type: ignore import zipfile + from typing import List, Optional, Tuple from .api_client import DSWAPIClient, DSWCommunicationError @@ -19,7 +20,7 @@ from .validation import ValidationError, TemplateValidator -ChangeItem = Tuple[watchgod.Change, pathlib.Path] +ChangeItem = Tuple[watchfiles.Change, pathlib.Path] class TDKProcessingError(RuntimeError): @@ -81,6 +82,9 @@ def __init__(self, template: Optional[Template] = None, project: Optional[Templa self.changes_processor = ChangesProcessor(self) self.remote_id = 'unknown' + async def close(self): + await self.safe_client.close() + @property def safe_template(self) -> Template: if self.template is None: @@ -368,8 +372,11 @@ def create_dot_env(self, output: pathlib.Path, force: bool, api_url: str, api_ke encoding=DEFAULT_ENCODING, ) - async def watch_project(self, callback): - async for changes in watchgod.awatch(self.safe_project.template_dir): + async def watch_project(self, callback, stop_event: asyncio.Event): + async for changes in watchfiles.awatch( + self.safe_project.template_dir, + stop_event=stop_event, + ): await callback(( change for change in ((change[0], pathlib.Path(change[1])) for change in changes) if self.safe_project.is_template_file( @@ -460,7 +467,7 @@ async def _process_file_changes(self): self.tdk.logger.debug(f'Processing {file_change}') change_type = file_change[0] filepath = file_change[1] - if change_type == watchgod.Change.deleted and filepath not in deleted: + if change_type == watchfiles.Change.deleted and filepath not in deleted: self.tdk.logger.debug('Scheduling delete operation') deleted.add(filepath) await self.tdk._delete_file(filepath) @@ -472,7 +479,7 @@ async def _process_file_changes(self): async def _reload_descriptor(self, force: bool) -> bool: if self.descriptor_change is None: return False - if self.descriptor_change[0] == watchgod.Change.deleted: + if self.descriptor_change[0] == watchfiles.Change.deleted: raise RuntimeError(f'Deleted template descriptor {self.tdk.safe_project.descriptor_path} ... the end') self.tdk.logger.debug(f'Reloading {TemplateProject.TEMPLATE_FILE} file') previous_id = self.tdk.safe_project.safe_template.id @@ -490,7 +497,7 @@ async def _reload_descriptor(self, force: bool) -> bool: async def _reload_readme(self) -> bool: if self.readme_change is None: return False - if self.readme_change[0] == watchgod.Change.deleted: + if self.readme_change[0] == watchfiles.Change.deleted: raise RuntimeError(f'Deleted used README file {self.tdk.safe_project.used_readme}') self.tdk.logger.debug('Reloading README file') self.tdk.safe_project.load_readme() diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index 4f4d38d5..dbe9364e 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -34,7 +34,7 @@ dependencies = [ 'pathspec', 'python-dotenv', 'python-slugify', - 'watchgod', + 'watchfiles', ] [project.optional-dependencies] diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 635b4909..16add327 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -1,6 +1,6 @@ aiohttp==3.8.5 aiosignal==1.3.1 -anyio==3.7.1 +anyio==4.0.0 async-timeout==4.0.3 attrs==23.1.0 charset-normalizer==3.3.0 @@ -17,5 +17,5 @@ python-dotenv==1.0.0 python-slugify==8.0.1 sniffio==1.3.0 text-unidecode==1.3 -watchgod==0.8.2 +watchfiles==0.20.0 yarl==1.9.2 diff --git a/requirements.txt b/requirements.txt index 3dc2a505..4dfc96c5 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,6 @@ aiohttp==3.8.5 aiosignal==1.3.1 -anyio==3.7.1 +anyio==4.0.0 async-timeout==4.0.3 attrs==23.1.0 Brotli==1.0.9 @@ -50,7 +50,6 @@ tinycss2==1.2.1 typing_extensions==4.8.0 tzdata==2023.3 urllib3==2.0.6 -watchgod==0.8.2 weasyprint==60.1 webencodings==0.5.1 XlsxWriter==3.1.6 From bb25fe0bcca04131c4c9d6c47e279b019f71b1df Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Fri, 6 Oct 2023 20:49:32 +0200 Subject: [PATCH 4/8] [DSW-2102] fix(tdk): Fix TDK when creating template with brackets --- packages/dsw-tdk/dsw/tdk/cli.py | 5 +++-- packages/dsw-tdk/dsw/tdk/utils.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/packages/dsw-tdk/dsw/tdk/cli.py b/packages/dsw-tdk/dsw/tdk/cli.py index 5d192585..d32ddacc 100644 --- a/packages/dsw-tdk/dsw/tdk/cli.py +++ b/packages/dsw-tdk/dsw/tdk/cli.py @@ -17,7 +17,7 @@ from .core import TDKCore, TDKProcessingError from .consts import VERSION, DEFAULT_LIST_FORMAT from .model import Template -from .utils import TemplateBuilder, FormatSpec +from .utils import TemplateBuilder, FormatSpec, safe_utf8 from .validation import ValidationError CURRENT_DIR = pathlib.Path.cwd() @@ -72,7 +72,8 @@ def watch_change(cls, change_type: watchfiles.Change, filepath: pathlib.Path, ro def prompt_fill(text: str, obj, attr, **kwargs): while True: try: - setattr(obj, attr, click.prompt(text, **kwargs).strip()) + value = safe_utf8(click.prompt(text, **kwargs).strip()) + setattr(obj, attr, value) break except ValidationError as e: ClickPrinter.error(e.message) diff --git a/packages/dsw-tdk/dsw/tdk/utils.py b/packages/dsw-tdk/dsw/tdk/utils.py index 5ba26caa..496e60da 100644 --- a/packages/dsw-tdk/dsw/tdk/utils.py +++ b/packages/dsw-tdk/dsw/tdk/utils.py @@ -193,3 +193,7 @@ def build(self) -> Template: def create_dot_env(api_url: Optional[str] = None, api_key: Optional[str] = None) -> str: return j2_env.get_template('env.j2').render(api_url=api_url, api_key=api_key) + + +def safe_utf8(text: str) -> str: + return text.encode(encoding='utf-8', errors='ignore').decode(encoding='utf-8') From c7d0837df324bb0a19191fcef6c7ae1eb540aea9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Thu, 12 Oct 2023 06:59:46 +0200 Subject: [PATCH 5/8] fix(db): Adapt to DB rename in documents table --- packages/dsw-database/dsw/database/model.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/packages/dsw-database/dsw/database/model.py b/packages/dsw-database/dsw/database/model.py index 94441a21..9427751c 100644 --- a/packages/dsw-database/dsw/database/model.py +++ b/packages/dsw-database/dsw/database/model.py @@ -54,7 +54,7 @@ class DBDocument: file_name: str content_type: str worker_log: str - creator_uuid: str + created_by: str retrieved_at: Optional[datetime.datetime] finished_at: Optional[datetime.datetime] created_at: datetime.datetime @@ -73,7 +73,7 @@ def from_dict_row(data: dict): questionnaire_replies_hash=data['questionnaire_replies_hash'], document_template_id=data['document_template_id'], format_uuid=str(data['format_uuid']), - creator_uuid=str(data['creator_uuid']), + created_by=str(data['created_by']), retrieved_at=data['retrieved_at'], finished_at=data['finished_at'], created_at=data['created_at'], From 36363a0c74737a9f96ac2c671c5c689d2c69bc13 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Mon, 30 Oct 2023 09:44:51 +0100 Subject: [PATCH 6/8] deps: Update dependencies --- packages/dsw-config/requirements.txt | 4 ++-- packages/dsw-data-seeder/requirements.txt | 4 ++-- packages/dsw-document-worker/requirements.txt | 14 +++++++------- packages/dsw-mailer/requirements.txt | 4 ++-- packages/dsw-storage/requirements.txt | 2 +- packages/dsw-tdk/requirements.txt | 4 ++-- requirements.txt | 16 ++++++++-------- 7 files changed, 24 insertions(+), 24 deletions(-) diff --git a/packages/dsw-config/requirements.txt b/packages/dsw-config/requirements.txt index c902c204..fa32eafd 100644 --- a/packages/dsw-config/requirements.txt +++ b/packages/dsw-config/requirements.txt @@ -1,4 +1,4 @@ certifi==2023.7.22 PyYAML==6.0.1 -sentry-sdk==1.31.0 -urllib3==2.0.6 +sentry-sdk==1.32.0 +urllib3==2.0.7 diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt index 5fb54a8f..dae30f2d 100644 --- a/packages/dsw-data-seeder/requirements.txt +++ b/packages/dsw-data-seeder/requirements.txt @@ -5,9 +5,9 @@ python-dateutil==2.8.2 psycopg==3.1.12 psycopg-binary==3.1.12 PyYAML==6.0.1 -sentry-sdk==1.31.0 +sentry-sdk==1.32.0 six==1.16.0 tenacity==8.2.3 typing_extensions==4.8.0 tzdata==2023.3 -urllib3==2.0.6 +urllib3==2.0.7 diff --git a/packages/dsw-document-worker/requirements.txt b/packages/dsw-document-worker/requirements.txt index de22d74c..ad481e40 100644 --- a/packages/dsw-document-worker/requirements.txt +++ b/packages/dsw-document-worker/requirements.txt @@ -1,21 +1,21 @@ Brotli==1.0.9 certifi==2023.7.22 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 click==8.1.7 cssselect2==0.7.0 -fonttools==4.43.0 +fonttools==4.43.1 html5lib==1.1 idna==3.4 isodate==0.6.1 Jinja2==3.1.2 -Markdown==3.4.4 +Markdown==3.5 MarkupSafe==2.1.3 mdx-breakless-lists==1.0.1 minio==7.1.17 pathvalidate==3.2.0 pdfrw==0.4 -Pillow==10.0.1 +Pillow==10.1.0 psycopg==3.1.12 psycopg-binary==3.1.12 pycparser==2.21 @@ -28,15 +28,15 @@ PyYAML==6.0.1 rdflib==7.0.0 rdflib-jsonld==0.6.2 requests==2.31.0 -sentry-sdk==1.31.0 +sentry-sdk==1.32.0 six==1.16.0 tenacity==8.2.3 text-unidecode==1.3 tinycss2==1.2.1 typing_extensions==4.8.0 tzdata==2023.3 -urllib3==2.0.6 +urllib3==2.0.7 weasyprint==60.1 webencodings==0.5.1 -XlsxWriter==3.1.6 +XlsxWriter==3.1.9 zopfli==0.2.3 diff --git a/packages/dsw-mailer/requirements.txt b/packages/dsw-mailer/requirements.txt index aa4bcc09..385587d1 100644 --- a/packages/dsw-mailer/requirements.txt +++ b/packages/dsw-mailer/requirements.txt @@ -9,9 +9,9 @@ psycopg==3.1.12 psycopg-binary==3.1.12 python-dateutil==2.8.2 PyYAML==6.0.1 -sentry-sdk==1.31.0 +sentry-sdk==1.32.0 six==1.16.0 tenacity==8.2.3 typing_extensions==4.8.0 tzdata==2023.3 -urllib3==2.0.6 +urllib3==2.0.7 diff --git a/packages/dsw-storage/requirements.txt b/packages/dsw-storage/requirements.txt index a5a92574..ca87fc3d 100644 --- a/packages/dsw-storage/requirements.txt +++ b/packages/dsw-storage/requirements.txt @@ -2,4 +2,4 @@ certifi==2023.7.22 minio==7.1.17 PyYAML==6.0.1 tenacity==8.2.3 -urllib3==2.0.6 +urllib3==2.0.7 diff --git a/packages/dsw-tdk/requirements.txt b/packages/dsw-tdk/requirements.txt index 16add327..fe409801 100644 --- a/packages/dsw-tdk/requirements.txt +++ b/packages/dsw-tdk/requirements.txt @@ -1,9 +1,9 @@ -aiohttp==3.8.5 +aiohttp==3.8.6 aiosignal==1.3.1 anyio==4.0.0 async-timeout==4.0.3 attrs==23.1.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 click==8.1.7 colorama==0.4.6 frozenlist==1.4.0 diff --git a/requirements.txt b/requirements.txt index 4dfc96c5..6cd52a8e 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,4 +1,4 @@ -aiohttp==3.8.5 +aiohttp==3.8.6 aiosignal==1.3.1 anyio==4.0.0 async-timeout==4.0.3 @@ -6,20 +6,20 @@ attrs==23.1.0 Brotli==1.0.9 certifi==2023.7.22 cffi==1.16.0 -charset-normalizer==3.3.0 +charset-normalizer==3.3.1 click==8.1.7 colorama==0.4.6 cssselect2==0.7.0 dkimpy==1.1.5 dnspython==2.4.2 -fonttools==4.43.0 +fonttools==4.43.1 frozenlist==1.4.0 html5lib==1.1 humanize==4.8.0 idna==3.4 isodate==0.6.1 Jinja2==3.1.2 -Markdown==3.4.4 +Markdown==3.5 MarkupSafe==2.1.3 mdx-breakless-lists==1.0.1 minio==7.1.17 @@ -27,7 +27,7 @@ multidict==6.0.4 pathspec==0.11.2 pathvalidate==3.2.0 pdfrw==0.4 -Pillow==10.0.1 +Pillow==10.1.0 psycopg==3.1.12 psycopg-binary==3.1.12 pycparser==2.21 @@ -41,7 +41,7 @@ PyYAML==6.0.1 rdflib==7.0.0 rdflib-jsonld==0.6.2 requests==2.31.0 -sentry-sdk==1.31.0 +sentry-sdk==1.32.0 six==1.16.0 sniffio==1.3.0 tenacity==8.2.3 @@ -49,9 +49,9 @@ text-unidecode==1.3 tinycss2==1.2.1 typing_extensions==4.8.0 tzdata==2023.3 -urllib3==2.0.6 +urllib3==2.0.7 weasyprint==60.1 webencodings==0.5.1 -XlsxWriter==3.1.6 +XlsxWriter==3.1.9 yarl==1.9.2 zopfli==0.2.3 From 3c0c0b682a6ceb95ecbbe1d948ee53fc4e6b4a1a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Mon, 30 Oct 2023 10:09:52 +0100 Subject: [PATCH 7/8] Release 4.0.0-RC.1 --- packages/dsw-command-queue/CHANGELOG.md | 5 +++++ packages/dsw-command-queue/pyproject.toml | 4 ++-- packages/dsw-config/CHANGELOG.md | 7 +++++++ packages/dsw-config/pyproject.toml | 2 +- packages/dsw-data-seeder/CHANGELOG.md | 5 +++++ packages/dsw-data-seeder/pyproject.toml | 10 +++++----- packages/dsw-database/CHANGELOG.md | 7 +++++++ packages/dsw-database/pyproject.toml | 4 ++-- packages/dsw-document-worker/CHANGELOG.md | 7 +++++++ packages/dsw-document-worker/pyproject.toml | 10 +++++----- packages/dsw-mailer/CHANGELOG.md | 5 +++++ packages/dsw-mailer/pyproject.toml | 8 ++++---- packages/dsw-models/CHANGELOG.md | 5 +++++ packages/dsw-models/pyproject.toml | 2 +- packages/dsw-storage/CHANGELOG.md | 9 +++++++++ packages/dsw-storage/pyproject.toml | 4 ++-- packages/dsw-tdk/CHANGELOG.md | 5 +++++ packages/dsw-tdk/pyproject.toml | 2 +- 18 files changed, 78 insertions(+), 23 deletions(-) diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md index 35933b9c..f537be4c 100644 --- a/packages/dsw-command-queue/CHANGELOG.md +++ b/packages/dsw-command-queue/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] Released for version consistency with other DSW tools. @@ -147,3 +151,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index ecf75760..feedc581 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = "3.28.0" +version = "4.0.0rc1" description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -25,7 +25,7 @@ classifiers = [ requires-python = '>=3.10, <4' dependencies = [ # DSW - "dsw-database==3.28.0", + "dsw-database==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md index 2027d77f..090e9e24 100644 --- a/packages/dsw-config/CHANGELOG.md +++ b/packages/dsw-config/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +### Changed + +- Adapt DB renames in documents table + ## [3.28.0] Released for version consistency with other DSW tools. @@ -155,3 +161,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 8881a53f..0f5e7bec 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = "3.28.0" +version = "4.0.0rc1" description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md index 03a0566f..2e19cf23 100644 --- a/packages/dsw-data-seeder/CHANGELOG.md +++ b/packages/dsw-data-seeder/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] ### Changed @@ -197,3 +201,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index d67a4693..c15623bf 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = "3.28.0" +version = "4.0.0rc1" description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -29,10 +29,10 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==3.28.0", - "dsw-config==3.28.0", - "dsw-database==3.28.0", - "dsw-storage==3.28.0", + "dsw-command-queue==4.0.0rc1", + "dsw-config==4.0.0rc1", + "dsw-database==4.0.0rc1", + "dsw-storage==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md index 2c25e706..e84d3eb7 100644 --- a/packages/dsw-database/CHANGELOG.md +++ b/packages/dsw-database/CHANGELOG.md @@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +### Changed + +- Adapt DB renames in documents table + ## [3.28.0] Released for version consistency with other DSW tools. @@ -162,3 +168,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index cb9ebae3..ac02df9d 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = "3.28.0" +version = "4.0.0rc1" description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -26,7 +26,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - "dsw-config==3.28.0", + "dsw-config==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md index f9deefea..adb3864d 100644 --- a/packages/dsw-document-worker/CHANGELOG.md +++ b/packages/dsw-document-worker/CHANGELOG.md @@ -8,16 +8,22 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] ### Added - Send charset as part of S3 object content-type + ## [3.27.1] ### Fixed - Fix PDF output format detection + ## [3.27.0] Released for version consistency with other DSW tools. @@ -188,3 +194,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index 8c50ff48..de4a7cd4 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = "3.28.0" +version = "4.0.0rc1" description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -40,10 +40,10 @@ dependencies = [ 'weasyprint', 'XlsxWriter', # DSW - "dsw-command-queue==3.28.0", - "dsw-config==3.28.0", - "dsw-database==3.28.0", - "dsw-storage==3.28.0", + "dsw-command-queue==4.0.0rc1", + "dsw-config==4.0.0rc1", + "dsw-database==4.0.0rc1", + "dsw-storage==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md index c0cdb966..fde8eceb 100644 --- a/packages/dsw-mailer/CHANGELOG.md +++ b/packages/dsw-mailer/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] Released for version consistency with other DSW tools. @@ -172,3 +176,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index e260f3df..34f9aa35 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = "3.28.0" +version = "4.0.0rc1" description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -31,9 +31,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==3.28.0", - "dsw-config==3.28.0", - "dsw-database==3.28.0", + "dsw-command-queue==4.0.0rc1", + "dsw-config==4.0.0rc1", + "dsw-database==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-models/CHANGELOG.md b/packages/dsw-models/CHANGELOG.md index ec62f911..87b3dce9 100644 --- a/packages/dsw-models/CHANGELOG.md +++ b/packages/dsw-models/CHANGELOG.md @@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] Released for version consistency with other DSW tools. @@ -80,3 +84,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-models/pyproject.toml b/packages/dsw-models/pyproject.toml index 6103be3c..5bc1bcfd 100644 --- a/packages/dsw-models/pyproject.toml +++ b/packages/dsw-models/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-models' -version = "3.28.0" +version = "4.0.0rc1" description = 'Library with DSW models and basic IO operations' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-storage/CHANGELOG.md b/packages/dsw-storage/CHANGELOG.md index 1607516b..a2cf8ae0 100644 --- a/packages/dsw-storage/CHANGELOG.md +++ b/packages/dsw-storage/CHANGELOG.md @@ -8,11 +8,19 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). ## [Unreleased] +## [4.0.0] + +### Fixed + +- Fixed creating template with special symbols +- Fixed watch mode termination + ## [3.28.0] ### Added - Support for object metadata + ## [3.27.1] Released for version consistency with other DSW tools. @@ -143,3 +151,4 @@ Released for version consistency with other DSW tools. [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 4103bd69..162d556c 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = "3.28.0" +version = "4.0.0rc1" description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -26,7 +26,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - "dsw-config==3.28.0", + "dsw-config==4.0.0rc1", ] [project.urls] diff --git a/packages/dsw-tdk/CHANGELOG.md b/packages/dsw-tdk/CHANGELOG.md index e473c61c..befbed18 100644 --- a/packages/dsw-tdk/CHANGELOG.md +++ b/packages/dsw-tdk/CHANGELOG.md @@ -8,6 +8,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +## [4.0.0] + +Released for version consistency with other DSW tools. + ## [3.28.0] Released for version consistency with other DSW tools. @@ -357,3 +361,4 @@ Initial DSW Template Development Kit (versioned as part of the [DSW platform](ht [3.27.0]: /../../tree/v3.27.0 [3.27.1]: /../../tree/v3.27.1 [3.28.0]: /../../tree/v3.28.0 +[4.0.0]: /../../tree/v4.0.0 diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index dbe9364e..8591707a 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = "3.28.0" +version = "4.0.0rc1" description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit'] From 997ab8dba072d517ad10dd0a2bc789c6d862bef2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Marek=20Such=C3=A1nek?= Date: Thu, 9 Nov 2023 11:59:57 +0100 Subject: [PATCH 8/8] Release 4.0.0-RC.2 --- packages/dsw-command-queue/pyproject.toml | 4 ++-- packages/dsw-config/pyproject.toml | 2 +- packages/dsw-data-seeder/dsw/data_seeder/consts.py | 2 +- packages/dsw-data-seeder/pyproject.toml | 10 +++++----- packages/dsw-database/pyproject.toml | 4 ++-- .../dsw-document-worker/dsw/document_worker/consts.py | 2 +- packages/dsw-document-worker/pyproject.toml | 10 +++++----- packages/dsw-mailer/dsw/mailer/consts.py | 2 +- packages/dsw-mailer/pyproject.toml | 8 ++++---- packages/dsw-models/pyproject.toml | 2 +- packages/dsw-storage/pyproject.toml | 4 ++-- packages/dsw-tdk/CHANGELOG.md | 6 +++++- packages/dsw-tdk/dsw/tdk/cli.py | 4 +++- packages/dsw-tdk/dsw/tdk/consts.py | 2 +- packages/dsw-tdk/pyproject.toml | 2 +- 15 files changed, 35 insertions(+), 29 deletions(-) diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml index feedc581..947e5b56 100644 --- a/packages/dsw-command-queue/pyproject.toml +++ b/packages/dsw-command-queue/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-command-queue' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Library for working with command queue and persistent commands' readme = 'README.md' keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing'] @@ -25,7 +25,7 @@ classifiers = [ requires-python = '>=3.10, <4' dependencies = [ # DSW - "dsw-database==4.0.0rc1", + "dsw-database==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml index 0f5e7bec..8bb35424 100644 --- a/packages/dsw-config/pyproject.toml +++ b/packages/dsw-config/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-config' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Library for DSW config manipulation' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py index bdce74ce..f1e813ad 100644 --- a/packages/dsw-data-seeder/dsw/data_seeder/consts.py +++ b/packages/dsw-data-seeder/dsw/data_seeder/consts.py @@ -6,4 +6,4 @@ DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-data-seeder' -VERSION = '3.28.0' +VERSION = '4.0.0' diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml index c15623bf..56955d4f 100644 --- a/packages/dsw-data-seeder/pyproject.toml +++ b/packages/dsw-data-seeder/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-data-seeder' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Worker for seeding DSW data' readme = 'README.md' keywords = ['data', 'database', 'seed', 'storage'] @@ -29,10 +29,10 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==4.0.0rc1", - "dsw-config==4.0.0rc1", - "dsw-database==4.0.0rc1", - "dsw-storage==4.0.0rc1", + "dsw-command-queue==4.0.0rc2", + "dsw-config==4.0.0rc2", + "dsw-database==4.0.0rc2", + "dsw-storage==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml index ac02df9d..a43c96f8 100644 --- a/packages/dsw-database/pyproject.toml +++ b/packages/dsw-database/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-database' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Library for managing DSW database' readme = 'README.md' keywords = ['dsw', 'database'] @@ -26,7 +26,7 @@ dependencies = [ 'psycopg[binary]', 'tenacity', # DSW - "dsw-config==4.0.0rc1", + "dsw-config==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-document-worker/dsw/document_worker/consts.py b/packages/dsw-document-worker/dsw/document_worker/consts.py index 6657c810..ae87bbff 100644 --- a/packages/dsw-document-worker/dsw/document_worker/consts.py +++ b/packages/dsw-document-worker/dsw/document_worker/consts.py @@ -6,7 +6,7 @@ EXIT_SUCCESS = 0 NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'docworker' -VERSION = '3.28.0' +VERSION = '4.0.0' class DocumentState: diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml index de4a7cd4..7606eb74 100644 --- a/packages/dsw-document-worker/pyproject.toml +++ b/packages/dsw-document-worker/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-document-worker' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Worker for assembling and transforming documents' readme = 'README.md' keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker'] @@ -40,10 +40,10 @@ dependencies = [ 'weasyprint', 'XlsxWriter', # DSW - "dsw-command-queue==4.0.0rc1", - "dsw-config==4.0.0rc1", - "dsw-database==4.0.0rc1", - "dsw-storage==4.0.0rc1", + "dsw-command-queue==4.0.0rc2", + "dsw-config==4.0.0rc2", + "dsw-database==4.0.0rc2", + "dsw-storage==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-mailer/dsw/mailer/consts.py b/packages/dsw-mailer/dsw/mailer/consts.py index 582a02ae..0064577e 100644 --- a/packages/dsw-mailer/dsw/mailer/consts.py +++ b/packages/dsw-mailer/dsw/mailer/consts.py @@ -5,4 +5,4 @@ DEFAULT_ENCODING = 'utf-8' NULL_UUID = '00000000-0000-0000-0000-000000000000' PROG_NAME = 'dsw-mailer' -VERSION = '3.28.0' +VERSION = '4.0.0' diff --git a/packages/dsw-mailer/pyproject.toml b/packages/dsw-mailer/pyproject.toml index 34f9aa35..4f651958 100644 --- a/packages/dsw-mailer/pyproject.toml +++ b/packages/dsw-mailer/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-mailer' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Worker for sending email notifications' readme = 'README.md' keywords = ['email', 'jinja2', 'notification', 'template'] @@ -31,9 +31,9 @@ dependencies = [ 'sentry-sdk', 'tenacity', # DSW - "dsw-command-queue==4.0.0rc1", - "dsw-config==4.0.0rc1", - "dsw-database==4.0.0rc1", + "dsw-command-queue==4.0.0rc2", + "dsw-config==4.0.0rc2", + "dsw-database==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-models/pyproject.toml b/packages/dsw-models/pyproject.toml index 5bc1bcfd..82dad126 100644 --- a/packages/dsw-models/pyproject.toml +++ b/packages/dsw-models/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-models' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Library with DSW models and basic IO operations' readme = 'README.md' keywords = ['dsw', 'config', 'yaml', 'parser'] diff --git a/packages/dsw-storage/pyproject.toml b/packages/dsw-storage/pyproject.toml index 162d556c..05a3f600 100644 --- a/packages/dsw-storage/pyproject.toml +++ b/packages/dsw-storage/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-storage' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Library for managing DSW S3 storage' readme = 'README.md' keywords = ['dsw', 's3', 'bucket', 'storage'] @@ -26,7 +26,7 @@ dependencies = [ 'minio', 'tenacity', # DSW - "dsw-config==4.0.0rc1", + "dsw-config==4.0.0rc2", ] [project.urls] diff --git a/packages/dsw-tdk/CHANGELOG.md b/packages/dsw-tdk/CHANGELOG.md index befbed18..20837601 100644 --- a/packages/dsw-tdk/CHANGELOG.md +++ b/packages/dsw-tdk/CHANGELOG.md @@ -10,7 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [4.0.0] -Released for version consistency with other DSW tools. +### Fixed + +- Fixed `unpackage` with default output directory +- Fixed terminating watch mode +- Fixed non-printable characters handling ## [3.28.0] diff --git a/packages/dsw-tdk/dsw/tdk/cli.py b/packages/dsw-tdk/dsw/tdk/cli.py index d32ddacc..f6db01e1 100644 --- a/packages/dsw-tdk/dsw/tdk/cli.py +++ b/packages/dsw-tdk/dsw/tdk/cli.py @@ -434,10 +434,12 @@ def extract_package(ctx, template_package, output, force: bool): data = pathlib.Path(template_package).read_bytes() tdk.extract_package( zip_data=data, - template_dir=pathlib.Path(output), + template_dir=pathlib.Path(output) if output is not None else output, force=force, ) except Exception as e: + import traceback + traceback.print_exc() ClickPrinter.failure('Failed to extract the package') ClickPrinter.error(f'> {e}') exit(1) diff --git a/packages/dsw-tdk/dsw/tdk/consts.py b/packages/dsw-tdk/dsw/tdk/consts.py index bf4bfaaa..ac05dff8 100644 --- a/packages/dsw-tdk/dsw/tdk/consts.py +++ b/packages/dsw-tdk/dsw/tdk/consts.py @@ -3,7 +3,7 @@ import re APP = 'dsw-tdk' -VERSION = '3.28.0' +VERSION = '4.0.0' METAMODEL_VERSION = 11 REGEX_SEMVER = re.compile(r'^[0-9]+\.[0-9]+\.[0-9]+$') diff --git a/packages/dsw-tdk/pyproject.toml b/packages/dsw-tdk/pyproject.toml index 8591707a..c5a663b6 100644 --- a/packages/dsw-tdk/pyproject.toml +++ b/packages/dsw-tdk/pyproject.toml @@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta' [project] name = 'dsw-tdk' -version = "4.0.0rc1" +version = "4.0.0rc2" description = 'Data Stewardship Wizard Template Development Toolkit' readme = 'README.md' keywords = ['documents', 'dsw', 'jinja2', 'template', 'toolkit']