Merge pull request #147 from ds-wizard/release/4.0.0

Release 4.0.0
ds-wizard · Nov 13, 2023 · 1d16f01 · 1d16f01
2 parents 59c7d15 + 997ab8d
commit 1d16f01
Show file tree

Hide file tree

Showing 48 changed files with 353 additions and 270 deletions.
diff --git a/.github/test.sh b/.github/test.sh
@@ -5,11 +5,11 @@ if make verify; then
  echo "## Verify: passed"
 else
  echo "## Verify: failed"
- echo "VERIFY_FAILS=command-queue:$VERIFY_FAILS" >> $GITHUB_ENV
+ echo "VERIFY_FAILS=$1:$VERIFY_FAILS" >> $GITHUB_ENV
 fi
 if make test; then
  echo "## Test: passed"
 else
  echo "## Test: failed"
- echo "TEST_FAILS=command-queue:$TEST_FAILS" >> $GITHUB_ENV
+ echo "TEST_FAILS=$1:$TEST_FAILS" >> $GITHUB_ENV
 fi
diff --git a/packages/dsw-command-queue/CHANGELOG.md b/packages/dsw-command-queue/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.0.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.28.0]
 
 Released for version consistency with other DSW tools.
@@ -147,3 +151,4 @@ Released for version consistency with other DSW tools.
 [3.27.0]: /../../tree/v3.27.0
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
+[4.0.0]: /../../tree/v4.0.0
diff --git a/packages/dsw-command-queue/pyproject.toml b/packages/dsw-command-queue/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-command-queue'
-version = "3.28.0"
+version = "4.0.0rc2"
 description = 'Library for working with command queue and persistent commands'
 readme = 'README.md'
 keywords = ['dsw', 'subscriber', 'publisher', 'database', 'queue', 'processing']
@@ -25,7 +25,7 @@ classifiers = [
 requires-python = '>=3.10, <4'
 dependencies = [
  # DSW
- "dsw-database==3.28.0",
+ "dsw-database==4.0.0rc2",
 ]
 
 [project.urls]

diff --git a/packages/dsw-command-queue/requirements.txt b/packages/dsw-command-queue/requirements.txt
@@ -1,5 +1,5 @@
-psycopg==3.1.10
-psycopg-binary==3.1.10
+psycopg==3.1.12
+psycopg-binary==3.1.12
 PyYAML==6.0.1
 tenacity==8.2.3
 typing_extensions==4.8.0

diff --git a/packages/dsw-config/CHANGELOG.md b/packages/dsw-config/CHANGELOG.md
@@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.0.0]
+
+### Changed
+
+- Adapt DB renames in documents table
+
 ## [3.28.0]
 
 Released for version consistency with other DSW tools.
@@ -155,3 +161,4 @@ Released for version consistency with other DSW tools.
 [3.27.0]: /../../tree/v3.27.0
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
+[4.0.0]: /../../tree/v4.0.0
diff --git a/packages/dsw-config/pyproject.toml b/packages/dsw-config/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-config'
-version = "3.28.0"
+version = "4.0.0rc2"
 description = 'Library for DSW config manipulation'
 readme = 'README.md'
 keywords = ['dsw', 'config', 'yaml', 'parser']

diff --git a/packages/dsw-config/requirements.txt b/packages/dsw-config/requirements.txt
@@ -1,4 +1,4 @@
 certifi==2023.7.22
 PyYAML==6.0.1
-sentry-sdk==1.31.0
-urllib3==2.0.6
+sentry-sdk==1.32.0
+urllib3==2.0.7
diff --git a/packages/dsw-data-seeder/CHANGELOG.md b/packages/dsw-data-seeder/CHANGELOG.md
@@ -8,6 +8,10 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.0.0]
+
+Released for version consistency with other DSW tools.
+
 ## [3.28.0]
 
 ### Changed
@@ -197,3 +201,4 @@ Released for version consistency with other DSW tools.
 [3.27.0]: /../../tree/v3.27.0
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
+[4.0.0]: /../../tree/v4.0.0
diff --git a/packages/dsw-data-seeder/dsw/data_seeder/cli.py b/packages/dsw-data-seeder/dsw/data_seeder/cli.py
@@ -57,13 +57,13 @@ def run(ctx: click.Context, recipe: str):
 
 @cli.command(name='seed', help='Seed data directly.')
 @click.option('-r', '--recipe', envvar='SEEDER_RECIPE')
-@click.option('-a', '--app_uuid', default=NULL_UUID)
+@click.option('-t', '--tenant-uuid', default=NULL_UUID)
 @click.pass_context
-def seed(ctx: click.Context, recipe: str, app_uuid: str):
+def seed(ctx: click.Context, recipe: str, tenant_uuid: str):
  cfg = ctx.obj['cfg']
  workdir = ctx.obj['workdir']
  seeder = DataSeeder(cfg=cfg, workdir=workdir)
- seeder.seed(recipe_name=recipe, app_uuid=app_uuid)
+ seeder.seed(recipe_name=recipe, tenant_uuid=tenant_uuid)
 
 
 @cli.command(name='list', help='List recipes for data seeding.')

diff --git a/packages/dsw-data-seeder/dsw/data_seeder/consts.py b/packages/dsw-data-seeder/dsw/data_seeder/consts.py
@@ -3,7 +3,7 @@
 CMD_CHANNEL = 'data_seeder'
 DEFAULT_ENCODING = 'utf-8'
 DEFAULT_MIMETYPE = 'application/octet-stream'
-DEFAULT_PLACEHOLDER = '<<|APP-ID|>>'
+DEFAULT_PLACEHOLDER = '<<|TENANT-ID|>>'
 NULL_UUID = '00000000-0000-0000-0000-000000000000'
 PROG_NAME = 'dsw-data-seeder'
-VERSION = '3.28.0'
+VERSION = '4.0.0'
diff --git a/packages/dsw-data-seeder/dsw/data_seeder/seeder.py b/packages/dsw-data-seeder/dsw/data_seeder/seeder.py
@@ -48,14 +48,14 @@ class SeedRecipe:
 
  def __init__(self, name: str, description: str, root: pathlib.Path,
  db_scripts: dict[str, DBScript], db_placeholder: str,
- s3_app_dir: Optional[pathlib.Path], s3_fname_replace: dict[str, str],
+ s3_dir: Optional[pathlib.Path], s3_fname_replace: dict[str, str],
  uuids_count: int, uuids_placeholder: Optional[str]):
  self.name = name
  self.description = description
  self.root = root
  self.db_scripts = db_scripts
  self.db_placeholder = db_placeholder
- self.s3_app_dir = s3_app_dir
+ self.s3_dir = s3_dir
  self.s3_fname_replace = s3_fname_replace
  self._db_scripts_data = collections.OrderedDict() # type: dict[str, str]
  self.s3_objects = collections.OrderedDict() # type: dict[pathlib.Path, str]
@@ -71,12 +71,12 @@ def _load_db_scripts(self):
  )
 
  def _load_s3_object_names(self):
- if self.s3_app_dir is None:
+ if self.s3_dir is None:
  return
- for s3_object_path in self.s3_app_dir.glob('**/*'):
+ for s3_object_path in self.s3_dir.glob('**/*'):
  if s3_object_path.is_file():
  target_object_name = str(
- s3_object_path.relative_to(self.s3_app_dir).as_posix()
+ s3_object_path.relative_to(self.s3_dir).as_posix()
  )
  for r_from, r_to in self.s3_fname_replace.items():
  target_object_name = target_object_name.replace(r_from, r_to)
@@ -99,15 +99,15 @@ def prepare(self):
  def run_prepare(self):
  self._prepare_uuids()
 
- def _replace_db_script(self, script: str, app_uuid: str) -> str:
- result = script.replace(self.db_placeholder, app_uuid)
+ def _replace_db_script(self, script: str, tenant_uuid: str) -> str:
+ result = script.replace(self.db_placeholder, tenant_uuid)
  for uuid_key, uuid_value in self.uuids_replacement.items():
  result = result.replace(uuid_key, uuid_value)
  return result
 
- def iterate_db_scripts(self, app_uuid: str):
+ def iterate_db_scripts(self, tenant_uuid: str):
  return (
- (script_id, self._replace_db_script(script, app_uuid))
+ (script_id, self._replace_db_script(script, tenant_uuid))
  for script_id, script in self._db_scripts_data.items()
  )
 
@@ -133,9 +133,9 @@ def __str__(self):
  f'{self.description}\n\n' \
  f'DB SQL Scripts:\n' \
  f'{scripts}\n' \
- f'DB APP UUID Placeholder: "{self.db_placeholder}"\n\n' \
- f'S3 App Dir:\n' \
- f'{self.s3_app_dir if self.s3_app_dir is not None else "[nothing]"}\n' \
+ f'DB Tenant UUID Placeholder: "{self.db_placeholder}"\n\n' \
+ f'S3 Directory:\n' \
+ f'{self.s3_dir if self.s3_dir is not None else "[nothing]"}\n' \
  f'S3 Filename Replace:\n' \
  f'{replaces}'
 
@@ -164,16 +164,16 @@ def load_from_json(recipe_file: pathlib.Path) -> 'SeedRecipe':
  else:
  s = DBScript(recipe_file.parent / filepath, target, index)
  db_scripts[s.id] = s
- s3_app_dir = None
- if 'appDir' in s3.keys():
- s3_app_dir = recipe_file.parent / s3['appDir']
+ s3_dir = None
+ if 'dir' in s3.keys():
+ s3_dir = recipe_file.parent / s3['dir']
  return SeedRecipe(
  name=data['name'],
  description=data.get('description', ''),
  root=recipe_file.parent,
  db_scripts=db_scripts,
- db_placeholder=db.get('appIdPlaceholder', DEFAULT_PLACEHOLDER),
- s3_app_dir=s3_app_dir,
+ db_placeholder=db.get('tenantIdPlaceholder', DEFAULT_PLACEHOLDER),
+ s3_dir=s3_dir,
  s3_fname_replace=s3.get('filenameReplace', {}),
  uuids_count=data.get('uuids', {}).get('count', 0),
  uuids_placeholder=data.get('uuids', {}).get('placeholder', None),
@@ -192,8 +192,8 @@ def create_default():
  description='Default dummy recipe',
  root=pathlib.Path('/dev/null'),
  db_scripts={},
- db_placeholder='<<|APP-ID|>>',
- s3_app_dir=pathlib.Path('/dev/null'),
+ db_placeholder=DEFAULT_PLACEHOLDER,
+ s3_dir=pathlib.Path('/dev/null'),
  s3_fname_replace={},
  uuids_count=0,
  uuids_placeholder=None,
@@ -262,10 +262,10 @@ def work(self, cmd: PersistentCommand):
  Context.get().update_trace_id(cmd.uuid)
  SentryReporter.set_context('cmd_uuid', cmd.uuid)
  self.recipe.run_prepare()
- app_uuid = cmd.body['appUuid']
+ tenant_uuid = cmd.body['tenantUuid']
  LOG.info(f'Seeding recipe "{self.recipe.name}" '
- f'to app with UUID "{app_uuid}"')
- self.execute(app_uuid)
+ f'to tenant with UUID "{tenant_uuid}"')
+ self.execute(tenant_uuid)
  Context.get().update_trace_id('-')
  SentryReporter.set_context('cmd_uuid', '-')
 
@@ -280,21 +280,21 @@ def _update_component_info():
  built_at=built_at,
  )
 
- def seed(self, recipe_name: str, app_uuid: str):
+ def seed(self, recipe_name: str, tenant_uuid: str):
  self._prepare_recipe(recipe_name)
  LOG.info(f'Executing recipe "{recipe_name}"')
- self.execute(app_uuid=app_uuid)
+ self.execute(tenant_uuid=tenant_uuid)
 
- def execute(self, app_uuid: str):
- SentryReporter.set_context('app_uuid', app_uuid)
+ def execute(self, tenant_uuid: str):
+ SentryReporter.set_context('tenant_uuid', tenant_uuid)
  # Run SQL scripts
  app_ctx = Context.get().app
  cursor = app_ctx.db.conn_query.new_cursor(use_dict=True)
  phase = 'DB'
  used_targets = set()
  try:
  LOG.info('Running SQL scripts')
- for script_id, sql_script in self.recipe.iterate_db_scripts(app_uuid):
+ for script_id, sql_script in self.recipe.iterate_db_scripts(tenant_uuid):
  LOG.debug(f' -> Executing script: {script_id}')
  script = self.recipe.db_scripts[script_id]
  if script.target in self.dbs.keys():
@@ -311,7 +311,7 @@ def execute(self, app_uuid: str):
  data = local_file.read_bytes()
  LOG.debug(f' -> Sending: {object_name}')
  app_ctx.s3.store_object(
- app_uuid=app_uuid,
+ tenant_uuid=tenant_uuid,
  object_name=object_name,
  content_type=_guess_mimetype(local_file.name),
  data=data,
@@ -332,5 +332,5 @@ def execute(self, app_uuid: str):
  for target in used_targets:
  self.dbs[target].conn_query.connection.commit()
  LOG.info('Data seeding done')
- SentryReporter.set_context('app_uuid', '-')
+ SentryReporter.set_context('tenant_uuid', '-')
  cursor.close()
diff --git a/packages/dsw-data-seeder/example/app_config_seed.sql b/packages/dsw-data-seeder/example/app_config_seed.sql
diff --git a/packages/dsw-data-seeder/example/example.seed.json b/packages/dsw-data-seeder/example/example.seed.json
@@ -11,13 +11,13 @@
  "target": "other"
  },
  {
- "filename": "app_config_seed.sql"
+ "filename": "tenant_config_seed.sql"
  }
  ],
- "appIdPlaceholder": "<<|APP-ID|>>"
+ "tenantIdPlaceholder": "<<|TENANT-ID|>>"
  },
  "s3": {
- "appDir": "app",
+ "dir": "app",
  "filenameReplace": {
  ":": "_"
  }

diff --git a/packages/dsw-data-seeder/example/tenant_config_seed.sql b/packages/dsw-data-seeder/example/tenant_config_seed.sql
@@ -0,0 +1 @@
+UPDATE tenant_config SET organization = '{"affiliations": [], "name": "Seeded Organization", "organizationId": "seeded.org", "description": "My seeded organization description"}' WHERE uuid = '<<|TENANT-ID|>>';
diff --git a/packages/dsw-data-seeder/example/user_seed.sql b/packages/dsw-data-seeder/example/user_seed.sql
@@ -1,2 +1,2 @@
-DELETE FROM user_entity WHERE email = 'nikola.tesla@example.com' and app_uuid = '<<|APP-ID|>>';
-DELETE FROM user_entity WHERE email = 'isaac.newton@example.com' and app_uuid = '<<|APP-ID|>>';
+DELETE FROM user_entity WHERE email = 'nikola.tesla@example.com' and tenant_uuid = '<<|TENANT-ID|>>';
+DELETE FROM user_entity WHERE email = 'isaac.newton@example.com' and tenant_uuid = '<<|TENANT-ID|>>';
diff --git a/packages/dsw-data-seeder/pyproject.toml b/packages/dsw-data-seeder/pyproject.toml
@@ -4,7 +4,7 @@ build-backend = 'setuptools.build_meta'
 
 [project]
 name = 'dsw-data-seeder'
-version = "3.28.0"
+version = "4.0.0rc2"
 description = 'Worker for seeding DSW data'
 readme = 'README.md'
 keywords = ['data', 'database', 'seed', 'storage']
@@ -29,10 +29,10 @@ dependencies = [
  'sentry-sdk',
  'tenacity',
  # DSW
- "dsw-command-queue==3.28.0",
- "dsw-config==3.28.0",
- "dsw-database==3.28.0",
- "dsw-storage==3.28.0",
+ "dsw-command-queue==4.0.0rc2",
+ "dsw-config==4.0.0rc2",
+ "dsw-database==4.0.0rc2",
+ "dsw-storage==4.0.0rc2",
 ]
 
 [project.urls]

diff --git a/packages/dsw-data-seeder/requirements.txt b/packages/dsw-data-seeder/requirements.txt
@@ -1,13 +1,13 @@
 certifi==2023.7.22
 click==8.1.7
-minio==7.1.16
+minio==7.1.17
 python-dateutil==2.8.2
-psycopg==3.1.10
-psycopg-binary==3.1.10
+psycopg==3.1.12
+psycopg-binary==3.1.12
 PyYAML==6.0.1
-sentry-sdk==1.31.0
+sentry-sdk==1.32.0
 six==1.16.0
 tenacity==8.2.3
 typing_extensions==4.8.0
 tzdata==2023.3
-urllib3==2.0.6
+urllib3==2.0.7
diff --git a/packages/dsw-database/CHANGELOG.md b/packages/dsw-database/CHANGELOG.md
@@ -8,6 +8,12 @@ to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 ## [Unreleased]
 
 
+## [4.0.0]
+
+### Changed
+
+- Adapt DB renames in documents table
+
 ## [3.28.0]
 
 Released for version consistency with other DSW tools.
@@ -162,3 +168,4 @@ Released for version consistency with other DSW tools.
 [3.27.0]: /../../tree/v3.27.0
 [3.27.1]: /../../tree/v3.27.1
 [3.28.0]: /../../tree/v3.28.0
+[4.0.0]: /../../tree/v4.0.0