Merge pull request #156 from ds-wizard/release/4.1.0

Release 4.1.0

packages/dsw-command-queue/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.1.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.0.0]
 
 Released for version consistency with other DSW tools.
@@ -152,3 +156,4 @@ Released for version consistency with other DSW tools.
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
 [4.0.0]: /../../tree/v4.0.0
+[4.1.0]: /../../tree/v4.1.0

packages/dsw-command-queue/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = "4.0.0rc2"
+version = "4.1.0"
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -25,7 +25,7 @@ classifiers = [
 requires-python = '>=3.10, <4'
 dependencies = [
  # DSW
- "dsw-database==4.0.0rc2",
+ "dsw-database==4.1.0",
 ]
 
 [project.urls]

packages/dsw-command-queue/requirements.txt

@@ -1,5 +1,5 @@
-psycopg==3.1.12
-psycopg-binary==3.1.12
+psycopg==3.1.13
+psycopg-binary==3.1.13
 PyYAML==6.0.1
 tenacity==8.2.3
 typing_extensions==4.8.0

packages/dsw-config/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.1.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.0.0]
 
 ### Changed
@@ -162,3 +166,4 @@ Released for version consistency with other DSW tools.
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
 [4.0.0]: /../../tree/v4.0.0
+[4.1.0]: /../../tree/v4.1.0

packages/dsw-config/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = "4.0.0rc2"
+version = "4.1.0"
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

packages/dsw-config/requirements.txt

@@ -1,4 +1,4 @@
-certifi==2023.7.22
+certifi==2023.11.17
 PyYAML==6.0.1
-sentry-sdk==1.32.0
-urllib3==2.0.7
+sentry-sdk==1.37.1
+urllib3==2.1.0

packages/dsw-data-seeder/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.1.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.0.0]
 
 Released for version consistency with other DSW tools.
@@ -202,3 +206,4 @@ Released for version consistency with other DSW tools.
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
 [4.0.0]: /../../tree/v4.0.0
+[4.1.0]: /../../tree/v4.1.0

packages/dsw-data-seeder/Dockerfile

@@ -1,4 +1,4 @@
-FROM datastewardshipwizard/python-base:3.11-alpine as builder
+FROM datastewardshipwizard/python-base:4.1.0-3.11-basic as builder
 
 WORKDIR /app
 
@@ -12,7 +12,7 @@ RUN python -m pip wheel --no-cache-dir --wheel-dir=/app/wheels -r /app/packages/
  && python -m pip wheel --no-cache-dir --no-deps --wheel-dir=/app/wheels /app/packages/dsw-data-seeder
 
 
-FROM datastewardshipwizard/python-base:3.11-alpine
+FROM datastewardshipwizard/python-base:4.1.0-3.11-basic
 
 ENV APPLICATION_CONFIG_PATH /app/config/application.yml
 ENV WORKDIR_PATH /home/user/data

packages/dsw-data-seeder/dsw/data_seeder/consts.py

@@ -6,4 +6,4 @@
 DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'dsw-data-seeder'
-VERSION = '4.0.0'
+VERSION = '4.1.0'

packages/dsw-data-seeder/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = "4.0.0rc2"
+version = "4.1.0"
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -29,10 +29,10 @@ dependencies = [
  'sentry-sdk',
  'tenacity',
  # DSW
- "dsw-command-queue==4.0.0rc2",
- "dsw-config==4.0.0rc2",
- "dsw-database==4.0.0rc2",
- "dsw-storage==4.0.0rc2",
+ "dsw-command-queue==4.1.0",
+ "dsw-config==4.1.0",
+ "dsw-database==4.1.0",
+ "dsw-storage==4.1.0",
 ]
 
 [project.urls]

packages/dsw-data-seeder/requirements.txt

@@ -1,13 +1,12 @@
-certifi==2023.7.22
+certifi==2023.11.17
 click==8.1.7
 minio==7.1.17
+psycopg==3.1.13
+psycopg-binary==3.1.13
 python-dateutil==2.8.2
-psycopg==3.1.12
-psycopg-binary==3.1.12
 PyYAML==6.0.1
-sentry-sdk==1.32.0
+sentry-sdk==1.37.1
 six==1.16.0
 tenacity==8.2.3
 typing_extensions==4.8.0
-tzdata==2023.3
-urllib3==2.0.7
+urllib3==2.1.0

packages/dsw-database/CHANGELOG.md

@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.1.0]
+
+Released for version consistency with other DSW tools.
+
 ## [4.0.0]
 
 ### Changed
@@ -169,3 +173,4 @@ Released for version consistency with other DSW tools.
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
 [4.0.0]: /../../tree/v4.0.0
+[4.1.0]: /../../tree/v4.1.0

packages/dsw-database/pyproject.toml

@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-database'
-version = "4.0.0rc2"
+version = "4.1.0"
 description = 'Library for managing DSW database'
 readme = 'README.md'
 keywords = ['dsw', 'database']
@@ -26,7 +26,7 @@ dependencies = [
  'psycopg[binary]',
  'tenacity',
  # DSW
- "dsw-config==4.0.0rc2",
+ "dsw-config==4.1.0",
 ]
 
 [project.urls]

packages/dsw-database/requirements.txt

@@ -1,5 +1,5 @@
-psycopg==3.1.12
-psycopg-binary==3.1.12
+psycopg==3.1.13
+psycopg-binary==3.1.13
 PyYAML==6.0.1
 tenacity==8.2.3
 typing_extensions==4.8.0

packages/dsw-document-worker/CHANGELOG.md

@@ -8,6 +8,20 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.1.0]
+
+### Changed
+
+- Adjusted with template metamodel version 12 (integration and integration reply changes)
+
+### Removed
+
+- Support of `wkhtmltopdf` step (marked as deprecated in v3.24)
+
+### Fixed
+
+- Reporting Jinja2 template syntax errors
+
 ## [4.0.0]
 
 Released for version consistency with other DSW tools.
@@ -195,3 +209,4 @@ Released for version consistency with other DSW tools.
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
 [4.0.0]: /../../tree/v4.0.0
+[4.1.0]: /../../tree/v4.1.0

packages/dsw-document-worker/Dockerfile

@@ -1,4 +1,4 @@
-FROM datastewardshipwizard/python-base:3.11-alpine-docworker as builder
+FROM datastewardshipwizard/python-base:4.1.0-3.11-docworker as builder
 
 ARG TARGETARCH
 
@@ -18,7 +18,7 @@ RUN python -m pip wheel --wheel-dir=/app/wheels -r /app/packages/dsw-document-wo
  && python -m pip install --no-index /app/wheels/* \
  && python -m pip wheel --no-deps --wheel-dir=/app/wheels /app/packages/dsw-document-worker
 
-FROM datastewardshipwizard/python-base:3.11-alpine-docworker
+FROM datastewardshipwizard/python-base:4.1.0-3.11-docworker
 
 ENV APPLICATION_CONFIG_PATH /app/config/application.yml
 ENV WORKDIR_PATH /tmp/docworker

packages/dsw-document-worker/README.md

@@ -13,7 +13,6 @@
 
 - PostgreSQL
 - S3 storage (e.g. [Minio](https://min.io))
-- [wkhtmltopdf](https://github.com/wkhtmltopdf/wkhtmltopdf)
 - [pandoc](https://github.com/jgm/pandoc)
 
 For more information, see [deployment example](https://github.com/ds-wizard/dsw-deployment-example).

packages/dsw-document-worker/config.example.yml

@@ -41,7 +41,3 @@ templates:
 # executable: pandoc
 # args: --standalone
 # timeout:
-# wkhtmltopdf:
-# executable: wkhtmltopdf
-# args:
-# timeout:

packages/dsw-document-worker/dsw/document_worker/config.py

@@ -73,32 +73,10 @@ class _CommandPandocKeys(ConfigKeysContainer):
  )
 
 
-class _CommandWkhtmltopdfKeys(ConfigKeysContainer):
- executable = ConfigKey(
- yaml_path=['externals', 'wkhtmltopdf', 'executable'],
- var_names=['WKHTMLTOPDF_EXECUTABLE'],
- default='wkhtmltopdf',
- cast=cast_str,
- )
- args = ConfigKey(
- yaml_path=['externals', 'wkhtmltopdf', 'args'],
- var_names=['WKHTMLTOPDF_ARGS'],
- default='',
- cast=cast_str,
- )
- timeout = ConfigKey(
- yaml_path=['externals', 'wkhtmltopdf', 'timeout'],
- var_names=['WKHTMLTOPDF_TIMEOUT'],
- default=None,
- cast=cast_optional_int,
- )
-
-
 class DocWorkerConfigKeys(ConfigKeys):
  documents = _DocumentsKeys
  experimental = _ExperimentalKeys
  cmd_pandoc = _CommandPandocKeys
- cmd_wkhtmltopdf = _CommandWkhtmltopdfKeys
 
 
 class DocumentsConfig(ConfigModel):
@@ -184,15 +162,14 @@ def get_config(self, template_id: str) -> Optional[TemplateConfig]:
 class DocumentWorkerConfig:
 
  def __init__(self, db: DatabaseConfig, s3: S3Config, log: LoggingConfig,
- doc: DocumentsConfig, pandoc: CommandConfig, wkhtmltopdf: CommandConfig,
+ doc: DocumentsConfig, pandoc: CommandConfig,
  templates: TemplatesConfig, experimental: ExperimentalConfig,
  cloud: CloudConfig, sentry: SentryConfig, general: GeneralConfig):
  self.db = db
  self.s3 = s3
  self.log = log
  self.doc = doc
  self.pandoc = pandoc
- self.wkhtmltopdf = wkhtmltopdf
  self.templates = templates
  self.experimental = experimental
  self.cloud = cloud
@@ -211,7 +188,6 @@ def __str__(self):
  f'{self.sentry}' \
  f'{self.general}' \
  f'Pandoc: {self.pandoc}' \
- f'WkHtmlToPdf: {self.wkhtmltopdf}' \
  f'====================\n'
 
 
@@ -237,14 +213,6 @@ def pandoc(self) -> CommandConfig:
  timeout=self.get(self.keys.cmd_pandoc.timeout),
  )
 
- @property
- def wkhtmltopdf(self) -> CommandConfig:
- return CommandConfig(
- executable=self.get(self.keys.cmd_wkhtmltopdf.executable),
- args=self.get(self.keys.cmd_wkhtmltopdf.args),
- timeout=self.get(self.keys.cmd_wkhtmltopdf.timeout),
- )
-
  @property
  def templates(self) -> TemplatesConfig:
  return TemplatesConfig(
@@ -272,7 +240,6 @@ def config(self) -> DocumentWorkerConfig:
  log=self.logging,
  doc=self.documents,
  pandoc=self.pandoc,
- wkhtmltopdf=self.wkhtmltopdf,
  templates=self.templates,
  experimental=self.experimental,
  cloud=self.cloud,

packages/dsw-document-worker/dsw/document_worker/consts.py

@@ -1,12 +1,12 @@
 CMD_CHANNEL = 'doc_worker'
 CMD_COMPONENT = 'doc_worker'
 COMPONENT_NAME = 'Document Worker'
-CURRENT_METAMODEL = 11
+CURRENT_METAMODEL = 12
 DEFAULT_ENCODING = 'utf-8'
 EXIT_SUCCESS = 0
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'docworker'
-VERSION = '4.0.0'
+VERSION = '4.1.0'
 
 
 class DocumentState:

packages/dsw-document-worker/dsw/document_worker/conversions.py

@@ -43,36 +43,6 @@ def __str__(self):
  f' to {self.target_format} - {self.message}'
 
 
-class WkHtmlToPdf:
-
- ARGS1 = ['--quiet', '--load-error-handling', 'ignore']
- ARGS2 = ['--encoding', DEFAULT_ENCODING, '-', '-']
-
- def __init__(self, config: DocumentWorkerConfig):
- self.config = config
-
- def __call__(self, source_format: FileFormat, target_format: FileFormat,
- data: bytes, metadata: dict, workdir: str) -> bytes:
- config_args = shlex.split(self.config.wkhtmltopdf.args)
- template_args = self.extract_template_args(metadata)
- args_access = ['--disable-local-file-access', '--allow', workdir]
- args = self.ARGS1 + template_args + config_args + args_access + self.ARGS2
- command = self.config.wkhtmltopdf.command + args
- return run_conversion(
- args=command,
- workdir=workdir,
- input_data=data,
- name=type(self).__name__,
- source_format=source_format,
- target_format=target_format,
- timeout=self.config.wkhtmltopdf.timeout,
- )
-
- @staticmethod
- def extract_template_args(metadata: dict):
- return shlex.split(metadata.get('args', ''))
-
-
 class Pandoc:
 
  def __init__(self, config: DocumentWorkerConfig):