Merge pull request #18 from ds-wizard/hotfix/3.15.3

Hotfix 3.15.3
ds-wizard · Sep 17, 2022 · bdb9768 · bdb9768
2 parents 84fabfd + c32407f
commit bdb9768
Show file tree

Hide file tree

Showing 27 changed files with 103 additions and 47 deletions.
diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+Released for version consistency with other DSW tools.
+
 ## [3.15.2]
 
 Released for version consistency with other DSW tools.
@@ -37,3 +41,4 @@ Released for version consistency with other DSW tools.
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = '3.15.2'
+version = '3.15.3'
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -27,7 +27,7 @@ classifiers = [
 requires-python = '>=3.7, <4'
 dependencies = [
  # DSW
- 'dsw-database==3.15.2',
+ 'dsw-database==3.15.3',
 ]
 
 [project.urls]

diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+Released for version consistency with other DSW tools.
+
 ## [3.15.2]
 
 Released for version consistency with other DSW tools.
@@ -37,3 +41,4 @@ Released for version consistency with other DSW tools.
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = '3.15.2'
+version = '3.15.3'
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md
@@ -8,6 +8,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+### Fixed
+
+- Update of the database dependency to fix on-start query memory leaks
+- Setting of log level to all internal loggers
+
 ## [3.15.2]
 
 ### Fixed
@@ -64,14 +71,15 @@ Released for version consistency with other DSW tools.
 
 
 [Unreleased]: /../../compare/main...develop
-[3.10.0]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
-[3.11.0]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
-[3.12.0]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
-[3.12.1]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
-[3.12.2]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
+[3.10.0]: https://github.com/ds-wizard/data-seeder/tree/v3.10.0
+[3.11.0]: https://github.com/ds-wizard/data-seeder/tree/v3.11.0
+[3.12.0]: https://github.com/ds-wizard/data-seeder/tree/v3.12.0
+[3.12.1]: https://github.com/ds-wizard/data-seeder/tree/v3.12.1
+[3.12.2]: https://github.com/ds-wizard/data-seeder/tree/v3.12.2
 [3.13.0]: https://github.com/ds-wizard/data-seeder/tree/v3.13.0
 [3.14.0]: /../../tree/v3.14.0
 [3.14.1]: /../../tree/v3.14.1
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-data-seeder/dsw/data_seeder/cli.py b/packages/dsw-data-seeder/dsw/data_seeder/cli.py
@@ -7,7 +7,7 @@
 
 from .config import SeederConfig, SeederConfigParser
 from .consts import PROG_NAME, VERSION, NULL_UUID
-from .seeder import SeedRecipe, DataSeeder
+from .logging import prepare_logging
 
 
 def validate_config(ctx, param, value: IO) -> SeederConfig:
@@ -38,6 +38,7 @@ def validate_config(ctx, param, value: IO) -> SeederConfig:
 def cli(ctx: click.Context, config: SeederConfig, workdir: str):
  ctx.obj['cfg'] = config
  ctx.obj['workdir'] = pathlib.Path(workdir).absolute()
+ prepare_logging(cfg=config)
 
 
 @cli.command()
@@ -47,6 +48,7 @@ def run(ctx: click.Context, recipe: str):
  """Run worker that listens to persistent commands"""
  cfg = ctx.obj['cfg']
  workdir = ctx.obj['workdir']
+ from .seeder import DataSeeder
  seeder = DataSeeder(cfg=cfg, workdir=workdir)
  seeder.run(recipe)
 
@@ -59,6 +61,7 @@ def seed(ctx: click.Context, recipe: str, app_uuid: str):
  """Seed data in DSW directly"""
  cfg = ctx.obj['cfg']
  workdir = ctx.obj['workdir']
+ from .seeder import DataSeeder
  seeder = DataSeeder(cfg=cfg, workdir=workdir)
  seeder.seed(recipe_name=recipe, app_uuid=app_uuid)
 
@@ -68,6 +71,7 @@ def seed(ctx: click.Context, recipe: str, app_uuid: str):
 def list(ctx: click.Context):
  """List recipes for data seeding"""
  workdir = ctx.obj['workdir']
+ from .seeder import SeedRecipe
  recipes = SeedRecipe.load_from_dir(workdir)
  for recipe in recipes.values():
  click.echo(recipe)

diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py
@@ -1,5 +1,5 @@
 PROG_NAME = 'dsw-seeder'
-VERSION = '3.15.0'
+VERSION = '3.15.3'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 
 LOGGER_NAME = 'DSW_DATA_SEEDER'

diff --git a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py
@@ -15,7 +15,6 @@
 from .consts import DEFAULT_ENCODING, DEFAULT_MIMETYPE, \
  DEFAULT_PLACEHOLDER, Queries
 from .context import Context
-from .logging import prepare_logging
 
 
 def _guess_mimetype(filename: str) -> str:
@@ -198,7 +197,6 @@ def _init_context(self, workdir: pathlib.Path):
  )
 
  def _prepare_logging(self):
- prepare_logging(cfg=self.cfg)
  Context.logger.set_level(self.cfg.log.level)
 
  def _prepare_recipe(self, recipe_name: str):

diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = '3.15.2'
+version = '3.15.3'
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -27,10 +27,10 @@ dependencies = [
  'click',
  'tenacity',
  # DSW
- 'dsw-command-queue==3.15.2',
- 'dsw-config==3.15.2',
- 'dsw-database==3.15.2',
- 'dsw-storage==3.15.2',
+ 'dsw-command-queue==3.15.3',
+ 'dsw-config==3.15.3',
+ 'dsw-database==3.15.3',
+ 'dsw-storage==3.15.3',
 ]
 
 [project.urls]

diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md
@@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+### Fixed
+
+- Avoid on-start query memory leaks
+
 ## [3.15.2]
 
 Released for version consistency with other DSW tools.
@@ -37,3 +43,4 @@ Released for version consistency with other DSW tools.
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py
@@ -522,9 +522,9 @@ def _connect_db(self):
  connection.set_isolation_level(self.isolation)
  # test connection
  cursor = connection.cursor()
- cursor.execute(query='SELECT * FROM persistent_command;')
- result = cursor.fetchall()
- LOG.debug(f'Jobs in queue: {result}')
+ cursor.execute(query='SELECT 1;')
+ result = cursor.fetchone()
+ LOG.debug(f'DB connection verified (result={result[0]})')
  cursor.close()
  connection.commit()
  self._connection = connection

diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-database'
-version = '3.15.2'
+version = '3.15.3'
 description = 'Library for managing DSW database'
 readme = 'README.md'
 keywords = ['dsw', 'database']
@@ -28,7 +28,7 @@ dependencies = [
  'psycopg2',
  'tenacity',
  # DSW
- 'dsw-config==3.15.2',
+ 'dsw-config==3.15.3',
 ]
 
 [project.urls]

diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md
@@ -8,6 +8,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+### Fixed
+
+- Update of the database dependency to fix on-start query memory leaks
+- Setting of log level to all internal loggers
+
 ## [3.15.2]
 
 ### Fixed
@@ -43,3 +50,4 @@ Released for version consistency with other DSW tools.
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-document-worker/dsw/document_worker/cli.py b/packages/dsw-document-worker/dsw/document_worker/cli.py
@@ -1,5 +1,7 @@
 import click
 import pathlib
+import logging
+import sys
 
 from typing import IO
 
@@ -8,7 +10,6 @@
 from .config import DocumentWorkerConfig, DocumentWorkerConfigParser
 from .sentry import SentryReporter
 from .consts import VERSION
-from .worker import DocumentWorker
 
 
 def validate_config(ctx, param, value: IO):
@@ -37,6 +38,12 @@ def validate_config(ctx, param, value: IO):
 @click.argument('workdir', envvar='DOCWORKER_WORKDIR',
  type=click.Path(dir_okay=True, exists=True))
 def main(config: DocumentWorkerConfig, workdir: str):
+ logging.basicConfig(
+ stream=sys.stdout,
+ level=config.log.global_level,
+ format=config.log.message_format
+ )
+ from .worker import DocumentWorker
  worker = DocumentWorker(config, pathlib.Path(workdir))
  try:
  worker.run()

diff --git a/packages/dsw-document-worker/dsw/document_worker/consts.py b/packages/dsw-document-worker/dsw/document_worker/consts.py
@@ -1,6 +1,6 @@
 DEFAULT_ENCODING = 'utf-8'
 EXIT_SUCCESS = 0
-VERSION = '3.15.0'
+VERSION = '3.15.3'
 PROG_NAME = 'docworker'
 LOGGER_NAME = 'docworker'
 CURRENT_METAMODEL = 10

diff --git a/packages/dsw-document-worker/dsw/document_worker/worker.py b/packages/dsw-document-worker/dsw/document_worker/worker.py
@@ -2,7 +2,6 @@
 import functools
 import logging
 import pathlib
-import sys
 import uuid
 
 from typing import Optional
@@ -260,11 +259,6 @@ def _init_context(self, workdir: pathlib.Path):
  )
 
  def _prepare_logging(self):
- logging.basicConfig(
- stream=sys.stdout,
- level=self.config.log.global_level,
- format=self.config.log.message_format
- )
  Context.logger.set_level(self.config.log.level)
  log_filter = DocWorkerLogFilter()
  logging.getLogger().addFilter(filter=log_filter)

diff --git a/packages/dsw-document-worker/pyproject.toml b/packages/dsw-document-worker/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-document-worker'
-version = '3.15.2'
+version = '3.15.3'
 description = 'Worker for assembling and transforming documents'
 readme = 'README.md'
 keywords = ['documents', 'generation', 'jinja2', 'pandoc', 'worker']
@@ -40,10 +40,10 @@ dependencies = [
  'sentry-sdk',
  'tenacity',
  # DSW
- 'dsw-command-queue==3.15.2',
- 'dsw-config==3.15.2',
- 'dsw-database==3.15.2',
- 'dsw-storage==3.15.2',
+ 'dsw-command-queue==3.15.3',
+ 'dsw-config==3.15.3',
+ 'dsw-database==3.15.3',
+ 'dsw-storage==3.15.3',
 ]
 
 [project.urls]

diff --git a/packages/dsw-mailer/CHANGELOG.md b/packages/dsw-mailer/CHANGELOG.md
@@ -8,6 +8,13 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.15.3]
+
+### Fixed
+
+- Update of the database dependency to fix on-start query memory leaks
+- Setting of log level to all internal loggers
+
 ## [3.15.2]
 
 ### Fixed
@@ -41,3 +48,4 @@ Released for version consistency with other DSW tools.
 [3.15.0]: /../../tree/v3.15.0
 [3.15.1]: /../../tree/v3.15.1
 [3.15.2]: /../../tree/v3.15.2
+[3.15.3]: /../../tree/v3.15.3
diff --git a/packages/dsw-mailer/dsw/mailer/cli.py b/packages/dsw-mailer/dsw/mailer/cli.py
@@ -9,8 +9,8 @@
 
 from .config import MailerConfig, MailerConfigParser
 from .consts import VERSION
-from .mailer import Mailer
 from .model import MessageRequest
+from .logging import prepare_logging
 
 
 def validate_config(ctx, param, value: IO):
@@ -58,6 +58,8 @@ def cli(ctx, config: MailerConfig, workdir: str, mode: str):
  click.echo('Mail is set to disabled, why even running mailer?')
  sys.exit(1)
  path_workdir = pathlib.Path(workdir)
+ prepare_logging(cfg=config)
+ from .mailer import Mailer
  ctx.obj['mailer'] = Mailer(config, path_workdir, mode)
 
 
@@ -67,6 +69,7 @@ def cli(ctx, config: MailerConfig, workdir: str, mode: str):
  callback=extract_message_request)
 def send(ctx, msg_request: MessageRequest):
  """Send message(s) from given file directly"""
+ from .mailer import Mailer
  mailer = ctx.obj['mailer'] # type: Mailer
  mailer.send(rq=msg_request)
 
@@ -75,6 +78,7 @@ def send(ctx, msg_request: MessageRequest):
 @click.pass_context
 def run(ctx):
  """Run mailer worker processing message jobs"""
+ from .mailer import Mailer
  mailer = ctx.obj['mailer'] # type: Mailer
  mailer.run()
 

diff --git a/packages/dsw-mailer/dsw/mailer/consts.py b/packages/dsw-mailer/dsw/mailer/consts.py
@@ -1,5 +1,5 @@
 PROG_NAME = 'dsw-mailer'
-VERSION = '3.15.0'
+VERSION = '3.15.3'
 
 LOGGER_NAME = 'mailer'