Merge pull request #35 from ds-wizard/release/3.18.0

Release 3.18.0
ds-wizard · Nov 29, 2022 · c94c68e · c94c68e
2 parents c9c36f0 + dae71c6
commit c94c68e
Show file tree

Hide file tree

Showing 41 changed files with 338 additions and 105 deletions.
diff --git a/.github/workflows/code-style.yml b/.github/workflows/code-style.yml
@@ -42,7 +42,7 @@ jobs:
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip
- pip install flake8
+ pip install flake8==5.0.4
  cd packages/${{ matrix.package }}
  pip install -r requirements.txt
  make local-deps
@@ -94,7 +94,7 @@ jobs:
  - name: Install dependencies
  run: |
  python -m pip install --upgrade pip
- pip install mypy
+ pip install mypy==0.991
  cd packages/${{ matrix.package }}
  pip install -r requirements.txt
  make local-deps
@@ -107,4 +107,4 @@ jobs:
  - name: Typecheck with mypy
  run: |
  cd packages/${{ matrix.package }}
- mypy --install-types --ignore-missing-imports --non-interactive dsw
+ mypy --install-types --ignore-missing-imports --check-untyped-defs --non-interactive dsw
diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.18.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.17.0]
 
 ### Changed
@@ -54,3 +58,4 @@ Released for version consistency with other DSW tools.
 [3.15.3]: /../../tree/v3.15.3
 [3.16.0]: /../../tree/v3.16.0
 [3.17.0]: /../../tree/v3.17.0
+[3.18.0]: /../../tree/v3.18.0
diff --git a/packages/dsw-command-queue/dsw/command_queue/command_queue.py b/packages/dsw-command-queue/dsw/command_queue/command_queue.py
@@ -67,7 +67,7 @@ def run(self):
  queue_conn.listening = True
  LOG.info('Listening for jobs in command queue')
 
- notifications = list()
+ notifications = list() # type: list[str]
  timeout = self.db.cfg.queue_timout
 
  LOG.info('Entering working cycle, waiting for notifications')

diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = '3.17.0'
+version = '3.18.0'
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -27,7 +27,7 @@ classifiers = [
 requires-python = '>=3.7, <4'
 dependencies = [
  # DSW
- 'dsw-database==3.17.0',
+ 'dsw-database==3.18.0',
 ]
 
 [project.urls]

diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.18.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.17.0]
 
 Released for version consistency with other DSW tools.
@@ -52,3 +56,4 @@ Released for version consistency with other DSW tools.
 [3.15.3]: /../../tree/v3.15.3
 [3.16.0]: /../../tree/v3.16.0
 [3.17.0]: /../../tree/v3.17.0
+[3.18.0]: /../../tree/v3.18.0
diff --git a/packages/dsw-config/dsw/config/parser.py b/packages/dsw-config/dsw/config/parser.py
@@ -1,6 +1,6 @@
 import yaml
 
-from typing import List
+from typing import List, Any
 
 from .model import GeneralConfig, SentryConfig, S3Config, \
  DatabaseConfig, LoggingConfig, CloudConfig, MailConfig
@@ -98,13 +98,13 @@ def has(self, *path):
  return True
 
  def _get_default(self, *path):
- x = self.DEFAULTS
+ x = self.DEFAULTS # type: Any
  for p in path:
  x = x[p]
  return x
 
  def get_or_default(self, *path):
- x = self.cfg
+ x = self.cfg # type: Any
  for p in path:
  if not hasattr(x, 'keys') or p not in x.keys():
  return self._get_default(*path)

diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = '3.17.0'
+version = '3.18.0'
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.18.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.17.0]
 
 ### Changed
@@ -95,3 +99,4 @@ Released for version consistency with other DSW tools.
 [3.15.3]: /../../tree/v3.15.3
 [3.16.0]: /../../tree/v3.16.0
 [3.17.0]: /../../tree/v3.17.0
+[3.18.0]: /../../tree/v3.18.0
diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py
@@ -1,5 +1,5 @@
 PROG_NAME = 'dsw-seeder'
-VERSION = '3.17.0'
+VERSION = '3.18.0'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 
 LOGGER_NAME = 'DSW_DATA_SEEDER'

diff --git a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py
@@ -65,9 +65,10 @@ def _load_s3_object_names(self):
  self.s3_objects[s3_object_path] = target_object_name
 
  def _prepare_uuids(self):
- for i in range(self.uuids_count):
- key = self.uuids_placeholder.replace('[n]', f'[{i}]')
- self.uuids_replacement[key] = str(uuid.uuid4())
+ if self.uuids_placeholder is not None:
+ for i in range(self.uuids_count):
+ key = self.uuids_placeholder.replace('[n]', f'[{i}]')
+ self.uuids_replacement[key] = str(uuid.uuid4())
 
  def prepare(self):
  if self.prepared:

diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = '3.17.0'
+version = '3.18.0rc1'
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -27,10 +27,10 @@ dependencies = [
  'click',
  'tenacity',
  # DSW
- 'dsw-command-queue==3.17.0',
- 'dsw-config==3.17.0',
- 'dsw-database==3.17.0',
- 'dsw-storage==3.17.0',
+ 'dsw-command-queue==3.18.0',
+ 'dsw-config==3.18.0',
+ 'dsw-database==3.18.0',
+ 'dsw-storage==3.18.0',
 ]
 
 [project.urls]

diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.18.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.17.0]
 
 ### Added
@@ -60,3 +64,4 @@ Released for version consistency with other DSW tools.
 [3.15.3]: /../../tree/v3.15.3
 [3.16.0]: /../../tree/v3.16.0
 [3.17.0]: /../../tree/v3.17.0
+[3.18.0]: /../../tree/v3.18.0
diff --git a/packages/dsw-database/dsw/database/database.py b/packages/dsw-database/dsw/database/database.py
@@ -344,7 +344,7 @@ def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False):
  connect_timeout=timeout,
  )
  self.autocommit = autocommit
- self._connection = None
+ self._connection = None # type: Optional[psycopg.Connection]
 
  @tenacity.retry(
  reraise=True,
@@ -356,10 +356,14 @@ def __init__(self, name: str, dsn: str, timeout=30000, autocommit=False):
  def _connect_db(self):
  LOG.info(f'Creating connection to PostgreSQL database "{self.name}"')
  connection = psycopg.connect(conninfo=self.dsn, autocommit=self.autocommit)
+ if connection is None:
+ raise RuntimeError('Failed to init DB connection')
  # test connection
  cursor = connection.cursor()
  cursor.execute(query='SELECT 1;')
  result = cursor.fetchone()
+ if result is None:
+ raise RuntimeError('Failed to verify DB connection')
  LOG.debug(f'DB connection verified (result={result[0]})')
  cursor.close()
  connection.commit()

diff --git a/packages/dsw-database/pyproject.toml b/packages/dsw-database/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-database'
-version = '3.17.0'
+version = '3.18.0'
 description = 'Library for managing DSW database'
 readme = 'README.md'
 keywords = ['dsw', 'database']
@@ -28,7 +28,7 @@ dependencies = [
  'psycopg[binary]',
  'tenacity',
  # DSW
- 'dsw-config==3.17.0',
+ 'dsw-config==3.18.0',
 ]
 
 [project.urls]

diff --git a/packages/dsw-document-worker/CHANGELOG.md b/packages/dsw-document-worker/CHANGELOG.md
@@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [3.18.0]
+
+### Fixed
+
+- Fix Excel step for datetime
+
 ## [3.17.0]
 
 ### Added
@@ -67,3 +73,4 @@ Released for version consistency with other DSW tools.
 [3.15.3]: /../../tree/v3.15.3
 [3.16.0]: /../../tree/v3.16.0
 [3.17.0]: /../../tree/v3.17.0
+[3.18.0]: /../../tree/v3.18.0
diff --git a/packages/dsw-document-worker/dsw/document_worker/config.py b/packages/dsw-document-worker/dsw/document_worker/config.py
@@ -20,7 +20,7 @@ def __str__(self):
 
 class ExperimentalConfig:
 
- def __init__(self, pdf_only: bool, job_timeout: Optional[float],
+ def __init__(self, pdf_only: bool, job_timeout: Optional[int],
  max_doc_size: Optional[float],
  pdf_watermark: str, pdf_watermark_top: bool):
  self.pdf_only = pdf_only

diff --git a/packages/dsw-document-worker/dsw/document_worker/consts.py b/packages/dsw-document-worker/dsw/document_worker/consts.py
@@ -1,6 +1,6 @@
 DEFAULT_ENCODING = 'utf-8'
 EXIT_SUCCESS = 0
-VERSION = '3.17.0'
+VERSION = '3.18.0'
 PROG_NAME = 'docworker'
 LOGGER_NAME = 'docworker'
 CURRENT_METAMODEL = 10

diff --git a/packages/dsw-document-worker/dsw/document_worker/model/context.py b/packages/dsw-document-worker/dsw/document_worker/model/context.py
@@ -378,7 +378,11 @@ def _resolve_links_parent(self, ctx):
  question_uuid = self.fragments[-1]
  if question_uuid in ctx.e.questions.keys():
  self.question = ctx.e.questions[question_uuid]
- self.question.replies[self.path] = self
+ if self.question is not None:
+ self.question.replies[self.path] = self
+
+ def _resolve_links(self, ctx):
+ pass
 
 
 class AnswerReply(Reply):
@@ -657,6 +661,9 @@ def _resolve_links_parent(self, ctx):
  self.required_phase = ctx.e.phases.get(self.required_phase_uuid, PHASE_NEVER)
  self.is_required = ctx.current_phase.order >= self.required_phase.order
 
+ def _resolve_links(self, ctx):
+ pass
+
  @property
  def url_references(self) -> list[URLReference]:
  return [r for r in self.references if isinstance(r, URLReference)]
@@ -789,7 +796,7 @@ def _resolve_links(self, ctx):
  for key in self.choice_uuids
  if key in ctx.e.choices.keys()]
  for choice in self.choices:
- choice.question = self
+ choice.parent = self
 
  @staticmethod
  def load(data: dict, **options):
@@ -1300,7 +1307,8 @@ def _resolve_links(self, ctx):
  m._resolve_links(ctx)
  if self.chapter_uuid is not None and self.chapter_uuid in ctx.e.chapters.keys():
  self.chapter = ctx.e.chapters[self.chapter_uuid]
- self.chapter.reports.append(self)
+ if self.chapter is not None:
+ self.chapter.reports.append(self)
 
  @staticmethod
  def load(data: dict, **options):

diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/__init__.py b/packages/dsw-document-worker/dsw/document_worker/templates/__init__.py
@@ -1,3 +1,4 @@
 from .templates import TemplateRegistry, Template
+from .formats import Format
 
-__all__ = ['TemplateRegistry', 'Template']
+__all__ = ['TemplateRegistry', 'Template', 'Format']
diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/formats.py b/packages/dsw-document-worker/dsw/document_worker/templates/formats.py
@@ -66,7 +66,7 @@ def execute(self, context: dict) -> DocumentFile:
  result = self.steps[0].execute_first(context)
  for step in self.steps[1:]:
  if result is not None:
- result = step.execute_follow(result)
+ result = step.execute_follow(result, context)
  else:
  break
  return result
diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/__init__.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/__init__.py
@@ -3,11 +3,13 @@
 from .conversion import WkHtmlToPdfStep, PandocStep, RdfLibConvertStep
 from .excel import ExcelStep
 from .template import JSONStep, Jinja2Step
+from .word import EnrichDocxStep
 
 __all__ = [
  'create_step', 'Step', 'FormatStepException',
  'ArchiveStep',
  'PandocStep', 'RdfLibConvertStep', 'WkHtmlToPdfStep',
  'ExcelStep',
  'JSONStep', 'Jinja2Step',
+ 'EnrichDocxStep',
 ]
diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/archive.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/archive.py
@@ -130,7 +130,7 @@ def _create_tar(self) -> DocumentFile:
  content=data,
  )
 
- def execute_follow(self, document: DocumentFile) -> DocumentFile:
+ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile:
  # (future) allow multiple files to be archived
  self.input_file_src = document.filename('document')
  if self.input_file_dst == '':

diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/base.py
@@ -31,7 +31,7 @@ def requires_via_extras(self, requirement: str) -> bool:
  def execute_first(self, context: dict) -> DocumentFile:
  return self.raise_exc('Called execute_follow on Step class')
 
- def execute_follow(self, document: DocumentFile) -> DocumentFile:
+ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile:
  return self.raise_exc('Called execute_follow on Step class')
 
  def raise_exc(self, message: str):

diff --git a/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py b/packages/dsw-document-worker/dsw/document_worker/templates/steps/conversion.py
@@ -16,7 +16,7 @@ def __init__(self, template, options: dict):
  def execute_first(self, context: dict) -> DocumentFile:
  return self.raise_exc(f'Step "{self.NAME}" cannot be first')
 
- def execute_follow(self, document: DocumentFile) -> DocumentFile:
+ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile:
  if document.file_format != FileFormats.HTML:
  self.raise_exc(f'WkHtmlToPdf does not support {document.file_format.name} format as input')
  data = self.wkhtmltopdf(
@@ -71,7 +71,7 @@ def __init__(self, template, options: dict):
  def execute_first(self, context: dict) -> DocumentFile:
  return self.raise_exc(f'Step "{self.NAME}" cannot be first')
 
- def execute_follow(self, document: DocumentFile) -> DocumentFile:
+ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile:
  if document.file_format != self.input_format:
  self.raise_exc(f'Unexpected input {document.file_format.name} as input for pandoc')
  data = self.pandoc(
@@ -114,7 +114,7 @@ def __init__(self, template, options: dict):
  def execute_first(self, context: dict) -> DocumentFile:
  return self.raise_exc(f'Step "{self.NAME}" cannot be first')
 
- def execute_follow(self, document: DocumentFile) -> DocumentFile:
+ def execute_follow(self, document: DocumentFile, context: dict) -> DocumentFile:
  if document.file_format != self.input_format:
  self.raise_exc(f'Unexpected input {document.file_format.name} '
  f'as input for rdflib-convert '