From 59453ff97204c4cf0087410252b4c417804a06f6 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 14:16:34 -0800 Subject: [PATCH 01/26] works for binary uuids --- data_diff/abcs/database_types.py | 6 ++++++ data_diff/databases/_connect.py | 6 +++++- data_diff/databases/base.py | 29 ++++++++++++++++++++++++++++- data_diff/hashdiff_tables.py | 19 +++++++++++++++++-- 4 files changed, 56 insertions(+), 4 deletions(-) diff --git a/data_diff/abcs/database_types.py b/data_diff/abcs/database_types.py index f3c6381a..7ceb7762 100644 --- a/data_diff/abcs/database_types.py +++ b/data_diff/abcs/database_types.py @@ -134,6 +134,12 @@ class String_UUID(ColType_UUID, StringType): pass +# Snowflake Binary UUID +@attrs.define(frozen=True) +class Binary_UUID(ColType_UUID): + python_type = bytes + + @attrs.define(frozen=True) class String_Alphanum(ColType_Alphanum, StringType): @staticmethod diff --git a/data_diff/databases/_connect.py b/data_diff/databases/_connect.py index be55cc2d..8ef7b43a 100644 --- a/data_diff/databases/_connect.py +++ b/data_diff/databases/_connect.py @@ -25,6 +25,8 @@ from data_diff.databases.duckdb import DuckDB from data_diff.databases.mssql import MsSQL +import urllib.parse + @attrs.define(frozen=True) class MatchUriPath: @@ -196,7 +198,9 @@ def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1, **kwargs) if dsn.password: kw["password"] = dsn.password - kw = {k: v for k, v in kw.items() if v is not None} + # snowflake connector can handle unquoted values, but data-diff cannot + # results in error if user or password is encoded + kw = {k: urllib.parse.unqoute(v) for k, v in kw.items() if v is not None} if issubclass(cls, ThreadedDatabase): db = cls(thread_count=thread_count, **kw, **kwargs) diff --git a/data_diff/databases/base.py b/data_diff/databases/base.py index bf165461..380b9d9a 100644 --- a/data_diff/databases/base.py +++ b/data_diff/databases/base.py @@ -23,6 +23,7 @@ from data_diff.queries.api import Expr, table, Select, SKIP, Explain, Code, this from data_diff.queries.ast_classes import ( Alias, + BinBoolOp, BinOp, CaseWhen, Cast, @@ -64,6 +65,7 @@ Float, Native_UUID, String_UUID, + Binary_UUID, String_Alphanum, String_VaryingAlphanum, TemporalType, @@ -482,6 +484,22 @@ def render_tableop(self, parent_c: Compiler, elem: TableOp) -> str: def render__resolvecolumn(self, c: Compiler, elem: _ResolveColumn) -> str: return self.compile(c, elem._get_resolved()) + def modify_string_where_clause(self, col, where_clause): + # NOTE: snowflake specific issue with Binary columns + return where_clause.replace(f'"{col}"', f"TO_VARCHAR(\"{col}\", 'UTF-8')") + + def check_for_binary_cols(self, where_exprs): + binary_uuid_columns = set() + for expr in where_exprs: + if isinstance(expr, BinBoolOp): + for arg in expr.args: + if isinstance(arg, _ResolveColumn): + resolved_column = arg.resolved + if isinstance(resolved_column, Column) and resolved_column.source_table.schema: + if isinstance(resolved_column.type, Binary_UUID): + binary_uuid_columns.add(resolved_column.name) + return binary_uuid_columns + def render_select(self, parent_c: Compiler, elem: Select) -> str: c: Compiler = attrs.evolve(parent_c, in_select=True) # .add_table_context(self.table) compile_fn = functools.partial(self.compile, c) @@ -497,7 +515,13 @@ def render_select(self, parent_c: Compiler, elem: Select) -> str: select += f" FROM {self.PLACEHOLDER_TABLE}" if elem.where_exprs: - select += " WHERE " + " AND ".join(map(compile_fn, elem.where_exprs)) + where_clause = " WHERE " + " AND ".join(map(compile_fn, elem.where_exprs)) + # post processing step for snowfake BINARAY_UUID columns + if parent_c.dialect.name == "Snowflake": + binary_uuids = self.check_for_binary_cols(elem.where_exprs) + for binary_uuid in binary_uuids: + where_clause = self.modify_string_where_clause(binary_uuid, where_clause) + select += where_clause if elem.group_by_exprs: select += " GROUP BY " + ", ".join(map(compile_fn, elem.group_by_exprs)) @@ -836,6 +860,9 @@ def normalize_uuid(self, value: str, coltype: ColType_UUID) -> str: """Creates an SQL expression, that strips uuids of artifacts like whitespace.""" if isinstance(coltype, String_UUID): return f"TRIM({value})" + # converts Binary to VARCHAR for Snowflake + elif isinstance(coltype, Binary_UUID): + return f"TRIM(TO_VARCHAR({value}, 'UTF-8'))" return self.to_string(value) def normalize_json(self, value: str, _coltype: JSON) -> str: diff --git a/data_diff/hashdiff_tables.py b/data_diff/hashdiff_tables.py index b77594cc..f33e5ea1 100644 --- a/data_diff/hashdiff_tables.py +++ b/data_diff/hashdiff_tables.py @@ -6,7 +6,17 @@ import attrs -from data_diff.abcs.database_types import ColType_UUID, NumericType, PrecisionType, StringType, Boolean, JSON +from data_diff.abcs.database_types import ( + Binary_UUID, + ColType_UUID, + NumericType, + PrecisionType, + String_UUID, + StringType, + Boolean, + JSON, + UnknownColType, +) from data_diff.info_tree import InfoTree from data_diff.utils import safezip, diffs_are_equiv_jsons from data_diff.thread_utils import ThreadedYielder @@ -92,7 +102,12 @@ def _validate_and_adjust_columns(self, table1: TableSegment, table2: TableSegmen # Update schemas to minimal mutual precision col1 = table1._schema[c1] col2 = table2._schema[c2] - if isinstance(col1, PrecisionType) and isinstance(col2, PrecisionType): + + # snowflake specific error when comparing binary and string columns + if isinstance(col1, String_UUID) and isinstance(col2, UnknownColType) and col2.text == "BINARY": + table2._schema[c2] = Binary_UUID() + + elif isinstance(col1, PrecisionType) and isinstance(col2, PrecisionType): if strict and not isinstance(col2, PrecisionType): raise TypeError(f"Incompatible types for column '{c1}': {col1} <-> {col2}") From 2dd668ba6b941d1e6f9cb6d2c4e4adaf8a74c92f Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 15:29:04 -0800 Subject: [PATCH 02/26] Update _connect.py add issue on data-fold --- data_diff/databases/_connect.py | 1 + 1 file changed, 1 insertion(+) diff --git a/data_diff/databases/_connect.py b/data_diff/databases/_connect.py index 8ef7b43a..cb73df0e 100644 --- a/data_diff/databases/_connect.py +++ b/data_diff/databases/_connect.py @@ -200,6 +200,7 @@ def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1, **kwargs) # snowflake connector can handle unquoted values, but data-diff cannot # results in error if user or password is encoded + # https://github.com/datafold/data-diff/issues/428 kw = {k: urllib.parse.unqoute(v) for k, v in kw.items() if v is not None} if issubclass(cls, ThreadedDatabase): From ee2cd89fb36a447ac0363425b47d1652d74b2236 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 16:04:45 -0800 Subject: [PATCH 03/26] add TEXT support --- data_diff/hashdiff_tables.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/data_diff/hashdiff_tables.py b/data_diff/hashdiff_tables.py index f33e5ea1..043a32fd 100644 --- a/data_diff/hashdiff_tables.py +++ b/data_diff/hashdiff_tables.py @@ -16,6 +16,7 @@ Boolean, JSON, UnknownColType, + Text, ) from data_diff.info_tree import InfoTree from data_diff.utils import safezip, diffs_are_equiv_jsons @@ -104,7 +105,8 @@ def _validate_and_adjust_columns(self, table1: TableSegment, table2: TableSegmen col2 = table2._schema[c2] # snowflake specific error when comparing binary and string columns - if isinstance(col1, String_UUID) and isinstance(col2, UnknownColType) and col2.text == "BINARY": + # use case is destination table has a binary column + if isinstance(col1, (String_UUID, Text)) and isinstance(col2, UnknownColType) and col2.text == "BINARY": table2._schema[c2] = Binary_UUID() elif isinstance(col1, PrecisionType) and isinstance(col2, PrecisionType): From cbb56b364022086861eb519ec2551d50e67f1997 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 16:41:34 -0800 Subject: [PATCH 04/26] misspelled quote --- data_diff/databases/_connect.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data_diff/databases/_connect.py b/data_diff/databases/_connect.py index cb73df0e..485d1227 100644 --- a/data_diff/databases/_connect.py +++ b/data_diff/databases/_connect.py @@ -25,7 +25,7 @@ from data_diff.databases.duckdb import DuckDB from data_diff.databases.mssql import MsSQL -import urllib.parse +from urllib.parse import unquote @attrs.define(frozen=True) @@ -201,7 +201,7 @@ def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1, **kwargs) # snowflake connector can handle unquoted values, but data-diff cannot # results in error if user or password is encoded # https://github.com/datafold/data-diff/issues/428 - kw = {k: urllib.parse.unqoute(v) for k, v in kw.items() if v is not None} + kw = {k: unquote(v) for k, v in kw.items() if v is not None} if issubclass(cls, ThreadedDatabase): db = cls(thread_count=thread_count, **kw, **kwargs) From 6faf204268fe1130e35d3b85a402bb5e76f21caa Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 23:49:52 -0800 Subject: [PATCH 05/26] Update _connect.py port is integer --- data_diff/databases/_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_diff/databases/_connect.py b/data_diff/databases/_connect.py index 485d1227..2f40ad55 100644 --- a/data_diff/databases/_connect.py +++ b/data_diff/databases/_connect.py @@ -201,7 +201,7 @@ def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1, **kwargs) # snowflake connector can handle unquoted values, but data-diff cannot # results in error if user or password is encoded # https://github.com/datafold/data-diff/issues/428 - kw = {k: unquote(v) for k, v in kw.items() if v is not None} + kw = {k: v if k not in ('password', 'user') else unquote(v) for k, v in kw.items() if v is not None} if issubclass(cls, ThreadedDatabase): db = cls(thread_count=thread_count, **kw, **kwargs) From 524cf02094c76e398c1fc06ae704ea293ac10b60 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Tue, 12 Dec 2023 23:54:16 -0800 Subject: [PATCH 06/26] test kw --- data_diff/databases/_connect.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data_diff/databases/_connect.py b/data_diff/databases/_connect.py index 2f40ad55..873d9cc9 100644 --- a/data_diff/databases/_connect.py +++ b/data_diff/databases/_connect.py @@ -201,7 +201,7 @@ def connect_to_uri(self, db_uri: str, thread_count: Optional[int] = 1, **kwargs) # snowflake connector can handle unquoted values, but data-diff cannot # results in error if user or password is encoded # https://github.com/datafold/data-diff/issues/428 - kw = {k: v if k not in ('password', 'user') else unquote(v) for k, v in kw.items() if v is not None} + kw = {k: v for k, v in kw.items() if v is not None} if issubclass(cls, ThreadedDatabase): db = cls(thread_count=thread_count, **kw, **kwargs) From 82c3ffe0141ed410e6d20a6c884c8a32bb3e0c95 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Wed, 13 Dec 2023 23:03:07 -0800 Subject: [PATCH 07/26] add ci/cd --- .github/workflows/CODEOWNERS | 1 + .github/workflows/_cache-dependencies.yml | 51 +++++++++++ .../_publish-data-platform-data-diff.yml | 85 +++++++++++++++++++ .github/workflows/ci.yml | 11 +-- .github/workflows/ci_full.yml | 10 +-- .github/workflows/formatter.yml | 8 +- .github/workflows/pull-request-checks.yml | 70 +++++++++++++++ 7 files changed, 214 insertions(+), 22 deletions(-) create mode 100644 .github/workflows/CODEOWNERS create mode 100644 .github/workflows/_cache-dependencies.yml create mode 100644 .github/workflows/_publish-data-platform-data-diff.yml create mode 100644 .github/workflows/pull-request-checks.yml diff --git a/.github/workflows/CODEOWNERS b/.github/workflows/CODEOWNERS new file mode 100644 index 00000000..e48175a8 --- /dev/null +++ b/.github/workflows/CODEOWNERS @@ -0,0 +1 @@ +* @coinlist/data diff --git a/.github/workflows/_cache-dependencies.yml b/.github/workflows/_cache-dependencies.yml new file mode 100644 index 00000000..9aa10eb5 --- /dev/null +++ b/.github/workflows/_cache-dependencies.yml @@ -0,0 +1,51 @@ +name: cache_dependencies + +on: + workflow_call: + inputs: + python_version: + required: true + type: string + poetry_version: + required: true + type: string + outputs: + python_cache_key: + description: "The key of the primary cache of the python dependencies" + value: ${{ jobs.python-cache.outputs.key }} + + +jobs: + python-cache: + runs-on: ubuntu-latest + outputs: + key: ${{ steps.define-cache-key.outputs.cache_key }} + steps: + - uses: actions/checkout@v3 + + - name: Setup Python + uses: actions/setup-python@v4 + id: setup-python + with: + python-version: '${{ inputs.python_version }}' + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ inputs.poetry_version }} + virtualenvs-in-project: true + + - name: Define Cache Key + id: define-cache-key + run: | + echo "cache_key=python-${{ runner.os }}--${{ inputs.python_version }}-${{ inputs.poetry_version }}-${{ hashFiles('**/poetry.lock') }}" >> $GITHUB_OUTPUT + + - name: Cache venv + id: cached-python + uses: actions/cache@v3 + with: + path: .venv + key: ${{ steps.define-cache-key.outputs.cache_key }} + + - name: Install dependencies + run: poetry install --no-interaction --no-root diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml new file mode 100644 index 00000000..36522161 --- /dev/null +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -0,0 +1,85 @@ +name: publish_data-platform-data-diff + +on: + workflow_call: + inputs: + python_version: + required: true + type: string + poetry_version: + required: true + type: string + python_cache_key: + required: true + type: string + role_to_assume: + required: true + type: string + aws_region: + required: true + type: string + +permissions: + id-token: write + contents: read + +jobs: + build: + name: 'Publish python SDK cl-data' + runs-on: ubuntu-latest + + steps: + - name: Setup AWS credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + role-to-assume: ${{ inputs.role_to_assume }} + aws-region: ${{ inputs.aws_region }} + mask-aws-account-id: 'yes' + + - uses: actions/checkout@v3 + + - name: Setup Python + id: setup-python + uses: actions/setup-python@v4 + with: + python-version: '${{ env.PYTHON_VERSION }}' + + - name: Install and configure Poetry + uses: snok/install-poetry@v1 + with: + version: ${{ env.POETRY_VERSION }} + virtualenvs-in-project: true + + - name: Restore cached key + id: cache-restore + uses: actions/cache/restore@v3 + with: + path: .venv + key: ${{ inputs.python_cache_key }} + + - name: Set env variables + run: | + echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt + echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV + + echo "VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region ${{ inputs.aws_region }} list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count)" >> $GITHUB_ENV + + - name: Publish dev version + if: ${{ github.ref_name != 'main' }} + run: | + echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt + echo "To install this wheel \`poetry add \"data-platform-data-diff==$(cat version.txt)[all]\" --source artifact\`" + + poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt) + + cd python + poetry build --format wheel + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text) + + - name: Publish new version + if: ${{ github.ref_name == 'main' }} + run: | + if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then + cd python + poetry build --format wheel + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index fd995d5a..322bd659 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -1,20 +1,13 @@ name: CI-COVER-VERSIONS on: -# push: -# paths: -# - '**.py' -# - '.github/workflows/**' -# - '!dev/**' - pull_request: - branches: [ master ] - workflow_dispatch: + workflow_call: jobs: unit_tests: strategy: - fail-fast: false + fail-fast: true matrix: os: [ubuntu-latest] python-version: diff --git a/.github/workflows/ci_full.yml b/.github/workflows/ci_full.yml index d125d114..f6b7b9dc 100644 --- a/.github/workflows/ci_full.yml +++ b/.github/workflows/ci_full.yml @@ -1,14 +1,8 @@ name: CI-COVER-DATABASES on: -# push: -# paths: -# - '**.py' -# - '.github/workflows/**' -# - '!dev/**' - pull_request: - branches: [ master ] workflow_dispatch: + workflow_call: permissions: id-token: write # This is required for requesting the JWT @@ -17,7 +11,7 @@ permissions: jobs: unit_tests: strategy: - fail-fast: false + fail-fast: true matrix: os: [ubuntu-latest] python-version: diff --git a/.github/workflows/formatter.yml b/.github/workflows/formatter.yml index 52ee9919..b60d7195 100644 --- a/.github/workflows/formatter.yml +++ b/.github/workflows/formatter.yml @@ -1,9 +1,7 @@ name: formatter on: - pull_request: - branches: [ master ] - - workflow_dispatch: + workflow_dispatch: + worflow_call: jobs: linter_name: @@ -19,4 +17,4 @@ jobs: - name: Auto commit ruff formatting uses: stefanzweifel/git-auto-commit-action@v5 with: - commit_message: 'style fixes by ruff' \ No newline at end of file + commit_message: 'style fixes by ruff' diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml new file mode 100644 index 00000000..363f4b47 --- /dev/null +++ b/.github/workflows/pull-request-checks.yml @@ -0,0 +1,70 @@ +name: PR checks + +on: + pull_request: {} + +permissions: + id-token: write + contents: read + actions: write + +jobs: + cancel: + description: Cancel any in-progress runs of this workflow execution + runs-on: ubuntu-latest + steps: + - uses: styfle/cancel-workflow-action@0.12.0 + with: + access_token: ${{ github.token }} + + setup: + description: Setsup the necessary variables for Python and Poery Versions + runs-on: ubuntu-latest + needs: [cancel] + outputs: + python_version: ${{ steps.set_var.outputs.python_version }} + poetry_version: ${{ steps.set_var.outputs.poetry_version }} + role_to_assume: ${{ steps.set_var.outputs.role_to_assume }} + aws_region: ${{ steps.set_var.outputs.aws_region }} + steps: + - id: set_var + run: | + echo "python_version=3.8" >> $GITHUB_OUTPUT + echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT + echo "role_to_assume=arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID}}:role/github-actions-data-platform-data-diff" >> $GITHUB_OUTPUT + echo "aws_region=${{ secrets.AWS_REGION }}" >> $GITHUB_OUTPUT + + perform-formatting: + description: Formats the *.py files using ruff + needs: [setup] + uses: ./.github/workflows/formatter.yml + + cache-dependencies: + description: Caches project dependencies for faster workflow execution + needs: [setup, preform-formatting] + uses: ./.github/workflows/_cache-dependencies.yml + secrets: inherit + with: + python_version: ${{ needs.setup.outputs.python_version }} + poetry_version: ${{ needs.setup.outputs.poetry_version }} + + run-unit-test-versions: + description: Runs the unit tests for the different versions of Python + needs: [setup] + uses: ./.github/workflows/ci_full.yml + + run-unit-test-per-database: + description: Runs the unit tests for the different databases + needs: [setup] + uses: ./.github/workflows/ci.yml + + publish-data-platform-data-diff-dev: + needs: [setup, cache-dependencies] + uses: ./.github/workflows/_publish-data-platform-data-diff.yml + secrets: inherit + with: + python_version: ${{ needs.setup.outputs.python_version }} + poetry_version: ${{ needs.setup.outputs.poetry_version }} + python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }} + role_to_assume: ${{ needs.setup.outputs.role_to_assume }} + aws_region: ${{ needs.setup.outputs.aws_region }} From 8a268ecc22b4de736a401badfa21aaaeaa130702 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Wed, 13 Dec 2023 23:44:04 -0800 Subject: [PATCH 08/26] worflow -> workflow --- .github/workflows/formatter.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/formatter.yml b/.github/workflows/formatter.yml index b60d7195..63923ba1 100644 --- a/.github/workflows/formatter.yml +++ b/.github/workflows/formatter.yml @@ -1,7 +1,7 @@ name: formatter on: workflow_dispatch: - worflow_call: + workflow_call: jobs: linter_name: From c01846bc68c5eef534a8461b1e2a501ea673cb2b Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 12:42:08 -0800 Subject: [PATCH 09/26] change name --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 36522161..279af6e0 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -25,7 +25,7 @@ permissions: jobs: build: - name: 'Publish python SDK cl-data' + name: 'Publish python data-platform-data-diff' runs-on: ubuntu-latest steps: From 0505f887137091dde3ff71be8fa42e8092c1839a Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 13:26:09 -0800 Subject: [PATCH 10/26] rmv desc --- .github/workflows/pull-request-checks.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 363f4b47..1020cb9a 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -10,7 +10,6 @@ permissions: jobs: cancel: - description: Cancel any in-progress runs of this workflow execution runs-on: ubuntu-latest steps: - uses: styfle/cancel-workflow-action@0.12.0 @@ -18,7 +17,6 @@ jobs: access_token: ${{ github.token }} setup: - description: Setsup the necessary variables for Python and Poery Versions runs-on: ubuntu-latest needs: [cancel] outputs: @@ -34,13 +32,11 @@ jobs: echo "role_to_assume=arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID}}:role/github-actions-data-platform-data-diff" >> $GITHUB_OUTPUT echo "aws_region=${{ secrets.AWS_REGION }}" >> $GITHUB_OUTPUT - perform-formatting: - description: Formats the *.py files using ruff + perform-ruff-formatting: needs: [setup] uses: ./.github/workflows/formatter.yml cache-dependencies: - description: Caches project dependencies for faster workflow execution needs: [setup, preform-formatting] uses: ./.github/workflows/_cache-dependencies.yml secrets: inherit @@ -49,12 +45,10 @@ jobs: poetry_version: ${{ needs.setup.outputs.poetry_version }} run-unit-test-versions: - description: Runs the unit tests for the different versions of Python needs: [setup] uses: ./.github/workflows/ci_full.yml run-unit-test-per-database: - description: Runs the unit tests for the different databases needs: [setup] uses: ./.github/workflows/ci.yml From 0c3de047695d965e68c1b0142e5c7e6a8c00b35d Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 13:28:47 -0800 Subject: [PATCH 11/26] preform -> perform --- .github/workflows/pull-request-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 1020cb9a..384048f5 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -37,7 +37,7 @@ jobs: uses: ./.github/workflows/formatter.yml cache-dependencies: - needs: [setup, preform-formatting] + needs: [setup, perform-ruff-formatting] uses: ./.github/workflows/_cache-dependencies.yml secrets: inherit with: From 98c85824398f9f88ffd7cb902a6e83b74207f209 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 13:41:09 -0800 Subject: [PATCH 12/26] add dependency --- .github/workflows/pull-request-checks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index 384048f5..ee8a6360 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -53,7 +53,7 @@ jobs: uses: ./.github/workflows/ci.yml publish-data-platform-data-diff-dev: - needs: [setup, cache-dependencies] + needs: [setup, run-unit-test-versions, run-unit-test-per-database, cache-dependencies] uses: ./.github/workflows/_publish-data-platform-data-diff.yml secrets: inherit with: From fbf6f64086adcbb6d7c7b4f9fa367fda62a6f1f9 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 14:11:31 -0800 Subject: [PATCH 13/26] remove secret --- .../_publish-data-platform-data-diff.yml | 19 ++++++++----------- .github/workflows/pull-request-checks.yml | 8 +------- 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 279af6e0..35a0fbcf 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -12,12 +12,6 @@ on: python_cache_key: required: true type: string - role_to_assume: - required: true - type: string - aws_region: - required: true - type: string permissions: id-token: write @@ -32,8 +26,8 @@ jobs: - name: Setup AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: ${{ inputs.role_to_assume }} - aws-region: ${{ inputs.aws_region }} + role-to-assume: ${{ secrets.CROSS_ACCOUNT_ROLE_TO_ASSSUME }} + aws-region: ${{ secrets.AWS_REGION }} mask-aws-account-id: 'yes' - uses: actions/checkout@v3 @@ -58,11 +52,14 @@ jobs: key: ${{ inputs.python_cache_key }} - name: Set env variables + env: + AWS_REGION = ${{ secrets.AWS_REGION }} run: | echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV + VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) - echo "VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region ${{ inputs.aws_region }} list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count)" >> $GITHUB_ENV + echo "VERSION_ALREADY_PUBLISHED=$VERSION_ALREADY_PUBLISHED >> $GITHUB_ENV - name: Publish dev version if: ${{ github.ref_name != 'main' }} @@ -74,7 +71,7 @@ jobs: cd python poetry build --format wheel - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text) + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) - name: Publish new version if: ${{ github.ref_name == 'main' }} @@ -82,4 +79,4 @@ jobs: if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then cd python poetry build --format wheel - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text) + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) diff --git a/.github/workflows/pull-request-checks.yml b/.github/workflows/pull-request-checks.yml index ee8a6360..0050cefe 100644 --- a/.github/workflows/pull-request-checks.yml +++ b/.github/workflows/pull-request-checks.yml @@ -22,15 +22,11 @@ jobs: outputs: python_version: ${{ steps.set_var.outputs.python_version }} poetry_version: ${{ steps.set_var.outputs.poetry_version }} - role_to_assume: ${{ steps.set_var.outputs.role_to_assume }} - aws_region: ${{ steps.set_var.outputs.aws_region }} steps: - id: set_var run: | echo "python_version=3.8" >> $GITHUB_OUTPUT echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT - echo "role_to_assume=arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID}}:role/github-actions-data-platform-data-diff" >> $GITHUB_OUTPUT - echo "aws_region=${{ secrets.AWS_REGION }}" >> $GITHUB_OUTPUT perform-ruff-formatting: needs: [setup] @@ -52,7 +48,7 @@ jobs: needs: [setup] uses: ./.github/workflows/ci.yml - publish-data-platform-data-diff-dev: + publish-data-platform-data-diff: needs: [setup, run-unit-test-versions, run-unit-test-per-database, cache-dependencies] uses: ./.github/workflows/_publish-data-platform-data-diff.yml secrets: inherit @@ -60,5 +56,3 @@ jobs: python_version: ${{ needs.setup.outputs.python_version }} poetry_version: ${{ needs.setup.outputs.poetry_version }} python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }} - role_to_assume: ${{ needs.setup.outputs.role_to_assume }} - aws_region: ${{ needs.setup.outputs.aws_region }} From 384bf54ff8dcc6a57cdc7531dae34b02a50cb7d8 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 14:22:00 -0800 Subject: [PATCH 14/26] formating --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 35a0fbcf..246e5b54 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -53,7 +53,7 @@ jobs: - name: Set env variables env: - AWS_REGION = ${{ secrets.AWS_REGION }} + AWS_REGION: ${{ secrets.AWS_REGION }} run: | echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV From b12312941937599a9ae0f623dc8aa4624e0a1360 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 14:31:35 -0800 Subject: [PATCH 15/26] ASSSUME -> ASSUME --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 246e5b54..ed87aa46 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -26,7 +26,7 @@ jobs: - name: Setup AWS credentials uses: aws-actions/configure-aws-credentials@v1 with: - role-to-assume: ${{ secrets.CROSS_ACCOUNT_ROLE_TO_ASSSUME }} + role-to-assume: ${{ secrets.CROSS_ACCOUNT_ROLE_TO_ASSUME }} aws-region: ${{ secrets.AWS_REGION }} mask-aws-account-id: 'yes' From 7974a8a04112e6702606dad190ca25e6c9333c97 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 15:23:57 -0800 Subject: [PATCH 16/26] add toml & catch error on aws get and put calls --- .../_publish-data-platform-data-diff.yml | 26 +- poetry.lock | 251 +++++++++++++++++- pyproject.toml | 1 + 3 files changed, 265 insertions(+), 13 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index ed87aa46..cd7289da 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -55,10 +55,11 @@ jobs: env: AWS_REGION: ${{ secrets.AWS_REGION }} run: | - echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt + VERSION=$(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) || { echo "FAILED TO GET POETRY VERSION"; exit 1;} + echo $VERSION > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV - VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) - + + VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) || { echo "Error querying AWS CodeArtifact, repository might not exist."; exit 1; } echo "VERSION_ALREADY_PUBLISHED=$VERSION_ALREADY_PUBLISHED >> $GITHUB_ENV - name: Publish dev version @@ -66,17 +67,20 @@ jobs: run: | echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt echo "To install this wheel \`poetry add \"data-platform-data-diff==$(cat version.txt)[all]\" --source artifact\`" - - poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt) - + + poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt) || { echo "Failed to set version in pyproject.toml"; exit 1; } + cd python - poetry build --format wheel - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) - + poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) || { echo "Failed to publish the package"; exit 1; } + - name: Publish new version if: ${{ github.ref_name == 'main' }} run: | if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then cd python - poetry build --format wheel - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) + poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) || { echo "Failed to publish the package"; exit 1; } + else + echo "Version already published, skipping publish step." + fi diff --git a/poetry.lock b/poetry.lock index 16e4495c..495fc3b9 100644 --- a/poetry.lock +++ b/poetry.lock @@ -1,9 +1,10 @@ -# This file is automatically @generated by Poetry 1.7.0 and should not be changed by hand. +# This file is automatically @generated by Poetry 1.4.2 and should not be changed by hand. [[package]] name = "agate" version = "1.7.1" description = "A data analysis library that is optimized for humans instead of machines." +category = "main" optional = false python-versions = "*" files = [ @@ -27,6 +28,7 @@ test = ["PyICU (>=2.4.2)", "coverage (>=3.7.1)", "cssselect (>=0.9.1)", "lxml (> name = "arrow" version = "1.2.3" description = "Better dates & times for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -41,6 +43,7 @@ python-dateutil = ">=2.7.0" name = "asn1crypto" version = "1.5.1" description = "Fast ASN.1 parser and serializer with definitions for private keys, public keys, certificates, CRL, OCSP, CMS, PKCS#3, PKCS#7, PKCS#8, PKCS#12, PKCS#5, X.509 and TSP" +category = "main" optional = false python-versions = "*" files = [ @@ -52,6 +55,7 @@ files = [ name = "attrs" version = "23.1.0" description = "Classes Without Boilerplate" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -70,6 +74,7 @@ tests-no-zope = ["cloudpickle", "hypothesis", "mypy (>=1.1.1)", "pympler", "pyte name = "babel" version = "2.11.0" description = "Internationalization utilities" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -84,6 +89,7 @@ pytz = ">=2015.7" name = "backports.zoneinfo" version = "0.2.1" description = "Backport of the standard library zoneinfo module" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -112,6 +118,7 @@ tzdata = ["tzdata"] name = "certifi" version = "2022.12.7" description = "Python package for providing Mozilla's CA Bundle." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -123,6 +130,7 @@ files = [ name = "cffi" version = "1.15.1" description = "Foreign Function Interface for Python calling C code." +category = "main" optional = false python-versions = "*" files = [ @@ -199,6 +207,7 @@ pycparser = "*" name = "cfgv" version = "3.4.0" description = "Validate configuration and produce human readable error messages." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -210,6 +219,7 @@ files = [ name = "charset-normalizer" version = "2.0.12" description = "The Real First Universal Charset Detector. Open, modern and actively maintained alternative to Chardet." +category = "main" optional = false python-versions = ">=3.5.0" files = [ @@ -224,6 +234,7 @@ unicode-backport = ["unicodedata2"] name = "click" version = "8.1.3" description = "Composable command line interface toolkit" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -238,6 +249,7 @@ colorama = {version = "*", markers = "platform_system == \"Windows\""} name = "clickhouse-driver" version = "0.2.5" description = "Python driver with native interface for ClickHouse" +category = "main" optional = false python-versions = ">=3.6, <4" files = [ @@ -350,6 +362,7 @@ zstd = ["clickhouse-cityhash (>=1.0.2.1)", "zstd"] name = "colorama" version = "0.4.4" description = "Cross-platform colored terminal text." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -361,6 +374,7 @@ files = [ name = "commonmark" version = "0.9.1" description = "Python parser for the CommonMark Markdown spec" +category = "main" optional = false python-versions = "*" files = [ @@ -375,6 +389,7 @@ test = ["flake8 (==3.7.8)", "hypothesis (==3.55.3)"] name = "coverage" version = "6.5.0" description = "Code coverage measurement for Python" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -437,6 +452,7 @@ toml = ["tomli"] name = "cryptography" version = "36.0.2" description = "cryptography is a package which provides cryptographic recipes and primitives to Python developers." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -477,6 +493,7 @@ test = ["hypothesis (>=1.11.4,!=3.79.2)", "iso8601", "pretend", "pytest (>=6.2.0 name = "dbt-core" version = "1.6.5" description = "With dbt, data analysts and engineers can build analytics the way engineers build applications." +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -513,6 +530,7 @@ urllib3 = ">=1.0,<2.0" name = "dbt-extractor" version = "0.4.1" description = "A tool to analyze and extract information from Jinja used in dbt projects." +category = "main" optional = false python-versions = ">=3.6.1" files = [ @@ -538,6 +556,7 @@ files = [ name = "dbt-semantic-interfaces" version = "0.2.0" description = "The shared semantic layer definitions that dbt-core and MetricFlow use" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -560,6 +579,7 @@ typing-extensions = ">=4.0,<5.0" name = "distlib" version = "0.3.7" description = "Distribution utilities" +category = "dev" optional = false python-versions = "*" files = [ @@ -571,6 +591,7 @@ files = [ name = "dsnparse" version = "0.1.15" description = "parse dsn urls" +category = "main" optional = false python-versions = "*" files = [ @@ -581,6 +602,7 @@ files = [ name = "duckdb" version = "0.7.1" description = "DuckDB embedded database" +category = "main" optional = false python-versions = "*" files = [ @@ -637,6 +659,7 @@ files = [ name = "filelock" version = "3.12.2" description = "A platform independent file lock." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -652,6 +675,7 @@ testing = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "diff-cover (>=7.5)", "p name = "future" version = "0.18.3" description = "Clean single-source support for Python 3 and 2" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -662,6 +686,7 @@ files = [ name = "hologram" version = "0.0.16" description = "JSON schema generation from dataclasses" +category = "main" optional = false python-versions = "*" files = [ @@ -677,6 +702,7 @@ python-dateutil = ">=2.8,<2.9" name = "identify" version = "2.5.31" description = "File identification library for Python" +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -691,6 +717,7 @@ license = ["ukkonen"] name = "idna" version = "3.4" description = "Internationalized Domain Names in Applications (IDNA)" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -702,6 +729,7 @@ files = [ name = "importlib-metadata" version = "6.8.0" description = "Read metadata from Python packages" +category = "main" optional = false python-versions = ">=3.8" files = [ @@ -721,6 +749,7 @@ testing = ["flufl.flake8", "importlib-resources (>=1.3)", "packaging", "pyfakefs name = "importlib-resources" version = "5.12.0" description = "Read resources from Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -739,6 +768,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "isodate" version = "0.6.1" description = "An ISO 8601 date/time/duration parser and formatter" +category = "main" optional = false python-versions = "*" files = [ @@ -753,6 +783,7 @@ six = "*" name = "jaraco-classes" version = "3.2.3" description = "Utility functions for Python class constructs" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -771,6 +802,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "jeepney" version = "0.8.0" description = "Low-level, pure Python DBus protocol wrapper." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -786,6 +818,7 @@ trio = ["async_generator", "trio"] name = "jinja2" version = "3.1.2" description = "A very fast and expressive template engine." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -803,6 +836,7 @@ i18n = ["Babel (>=2.7)"] name = "jsonschema" version = "3.1.1" description = "An implementation of JSON Schema validation for Python" +category = "main" optional = false python-versions = "*" files = [ @@ -824,6 +858,7 @@ format = ["idna", "jsonpointer (>1.13)", "rfc3987", "strict-rfc3339", "webcolors name = "keyring" version = "23.13.1" description = "Store and access your passwords safely." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -848,6 +883,7 @@ testing = ["flake8 (<5)", "pytest (>=6)", "pytest-black (>=0.3.7)", "pytest-chec name = "lark-parser" version = "0.11.3" description = "a modern parsing library" +category = "main" optional = false python-versions = "*" files = [ @@ -863,6 +899,7 @@ regex = ["regex"] name = "leather" version = "0.3.4" description = "Python charting for 80% of humans." +category = "main" optional = false python-versions = "*" files = [ @@ -877,6 +914,7 @@ six = ">=1.6.1" name = "logbook" version = "1.5.3" description = "A logging replacement for Python" +category = "main" optional = false python-versions = "*" files = [ @@ -906,6 +944,7 @@ zmq = ["pyzmq"] name = "markupsafe" version = "2.0.1" description = "Safely add untrusted strings to HTML/XML markup." +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -984,6 +1023,7 @@ files = [ name = "mashumaro" version = "3.8.1" description = "Fast serialization library on top of dataclasses" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1005,6 +1045,7 @@ yaml = ["pyyaml (>=3.13)"] name = "minimal-snowplow-tracker" version = "0.0.2" description = "A minimal snowplow event tracker for Python. Add analytics to your Python and Django apps, webapps and games" +category = "main" optional = false python-versions = "*" files = [ @@ -1019,6 +1060,7 @@ six = ">=1.9.0,<2.0" name = "more-itertools" version = "8.14.0" description = "More routines for operating on iterables, beyond itertools" +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1030,6 +1072,7 @@ files = [ name = "msgpack" version = "1.0.4" description = "MessagePack serializer" +category = "main" optional = false python-versions = "*" files = [ @@ -1091,6 +1134,7 @@ files = [ name = "mysql-connector-python" version = "8.0.29" description = "MySQL driver written in Python" +category = "main" optional = false python-versions = "*" files = [ @@ -1126,6 +1170,7 @@ gssapi = ["gssapi (>=1.6.9)"] name = "networkx" version = "2.6.3" description = "Python package for creating and manipulating graphs and networks" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1144,6 +1189,7 @@ test = ["codecov (>=2.1)", "pytest (>=6.2)", "pytest-cov (>=2.12)"] name = "nodeenv" version = "1.8.0" description = "Node.js virtual environment builder" +category = "dev" optional = false python-versions = ">=2.7,!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,!=3.6.*" files = [ @@ -1158,6 +1204,7 @@ setuptools = "*" name = "oracledb" version = "1.3.2" description = "Python interface to Oracle Database" +category = "main" optional = true python-versions = ">=3.6" files = [ @@ -1198,6 +1245,7 @@ cryptography = ">=3.2.1" name = "oscrypto" version = "1.3.0" description = "TLS (SSL) sockets, key generation, encryption, decryption, signing, verification and KDFs using the OS crypto libraries. Does not require a compiler, and relies on the OS for patching. Works on Windows, OS X and Linux/BSD." +category = "main" optional = false python-versions = "*" files = [ @@ -1212,6 +1260,7 @@ asn1crypto = ">=1.5.1" name = "packaging" version = "21.3" description = "Core utilities for Python packages" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1226,6 +1275,7 @@ pyparsing = ">=2.0.2,<3.0.5 || >3.0.5" name = "parameterized" version = "0.8.1" description = "Parameterized testing with any Python test framework" +category = "dev" optional = false python-versions = "*" files = [ @@ -1240,6 +1290,7 @@ dev = ["jinja2"] name = "parsedatetime" version = "2.4" description = "Parse human-readable date/time text." +category = "main" optional = false python-versions = "*" files = [ @@ -1254,6 +1305,7 @@ future = "*" name = "pathspec" version = "0.11.2" description = "Utility library for gitignore style pattern matching of file paths." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1265,6 +1317,7 @@ files = [ name = "platformdirs" version = "3.11.0" description = "A small Python package for determining appropriate platform-specific dirs, e.g. a \"user data dir\"." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1280,6 +1333,7 @@ test = ["appdirs (==1.4.4)", "covdefaults (>=2.3)", "pytest (>=7.4)", "pytest-co name = "pre-commit" version = "3.5.0" description = "A framework for managing and maintaining multi-language pre-commit hooks." +category = "dev" optional = false python-versions = ">=3.8" files = [ @@ -1298,6 +1352,7 @@ virtualenv = ">=20.10.0" name = "preql" version = "0.2.19" description = "An interpreted relational query language that compiles to SQL" +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1323,6 +1378,7 @@ server = ["starlette"] name = "presto-python-client" version = "0.8.3" description = "Client for the Presto distributed SQL Engine" +category = "main" optional = false python-versions = "*" files = [ @@ -1345,6 +1401,7 @@ tests = ["google-auth", "httpretty", "pytest", "pytest-runner", "requests-kerber name = "prompt-toolkit" version = "3.0.36" description = "Library for building powerful interactive command lines in Python" +category = "main" optional = false python-versions = ">=3.6.2" files = [ @@ -1359,6 +1416,7 @@ wcwidth = "*" name = "protobuf" version = "4.22.3" description = "" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1381,6 +1439,7 @@ files = [ name = "psycopg2" version = "2.9.5" description = "psycopg2 - Python-PostgreSQL Database Adapter" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1403,6 +1462,7 @@ files = [ name = "pycparser" version = "2.21" description = "C parser in Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*" files = [ @@ -1414,6 +1474,7 @@ files = [ name = "pycryptodomex" version = "3.16.0" description = "Cryptographic library for Python" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*" files = [ @@ -1449,6 +1510,7 @@ files = [ name = "pydantic" version = "1.10.12" description = "Data validation and settings management using python type hints" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1501,6 +1563,7 @@ email = ["email-validator (>=1.0.3)"] name = "pygments" version = "2.15.1" description = "Pygments is a syntax highlighting package written in Python." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1515,6 +1578,7 @@ plugins = ["importlib-metadata"] name = "PyJWT" version = "2.6.0" description = "JSON Web Token implementation in Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1532,6 +1596,7 @@ tests = ["coverage[toml] (==5.0.4)", "pytest (>=6.0.0,<7.0.0)"] name = "pyodbc" version = "4.0.39" description = "DB API Module for ODBC" +category = "main" optional = true python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -1576,6 +1641,7 @@ files = [ name = "pyopenssl" version = "22.0.0" description = "Python wrapper module around the OpenSSL library" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1594,6 +1660,7 @@ test = ["flaky", "pretend", "pytest (>=3.0.1)"] name = "pyparsing" version = "3.0.9" description = "pyparsing module - Classes and methods to define and execute parsing grammars" +category = "main" optional = false python-versions = ">=3.6.8" files = [ @@ -1608,6 +1675,7 @@ diagrams = ["jinja2", "railroad-diagrams"] name = "pyrsistent" version = "0.19.3" description = "Persistent/Functional/Immutable data structures" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1644,6 +1712,7 @@ files = [ name = "python-dateutil" version = "2.8.2" description = "Extensions to the standard Python datetime module" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,>=2.7" files = [ @@ -1658,6 +1727,7 @@ six = ">=1.5" name = "python-slugify" version = "7.0.0" description = "A Python slugify application that also handles Unicode" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1675,6 +1745,7 @@ unidecode = ["Unidecode (>=1.1.1)"] name = "pytimeparse" version = "1.1.8" description = "Time expression parser" +category = "main" optional = false python-versions = "*" files = [ @@ -1686,6 +1757,7 @@ files = [ name = "pytz" version = "2022.6" description = "World timezone definitions, modern and historical" +category = "main" optional = false python-versions = "*" files = [ @@ -1697,6 +1769,7 @@ files = [ name = "pytz-deprecation-shim" version = "0.1.0.post0" description = "Shims to make deprecation of pytz easier" +category = "main" optional = false python-versions = "!=3.0.*,!=3.1.*,!=3.2.*,!=3.3.*,!=3.4.*,!=3.5.*,>=2.7" files = [ @@ -1712,6 +1785,7 @@ tzdata = {version = "*", markers = "python_version >= \"3.6\""} name = "pywin32-ctypes" version = "0.2.0" description = "" +category = "main" optional = false python-versions = "*" files = [ @@ -1723,6 +1797,7 @@ files = [ name = "pyyaml" version = "6.0" description = "YAML parser and emitter for Python" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1768,10 +1843,109 @@ files = [ {file = "PyYAML-6.0.tar.gz", hash = "sha256:68fb519c14306fec9720a2a5b45bc9f0c8d1b9c72adf45c37baedfcd949c35a2"}, ] +[[package]] +name = "regex" +version = "2023.10.3" +description = "Alternative regular expression module, to replace re." +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "regex-2023.10.3-cp310-cp310-macosx_10_9_x86_64.whl", hash = "sha256:4c34d4f73ea738223a094d8e0ffd6d2c1a1b4c175da34d6b0de3d8d69bee6bcc"}, + {file = "regex-2023.10.3-cp310-cp310-macosx_11_0_arm64.whl", hash = "sha256:a8f4e49fc3ce020f65411432183e6775f24e02dff617281094ba6ab079ef0915"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4cd1bccf99d3ef1ab6ba835308ad85be040e6a11b0977ef7ea8c8005f01a3c29"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:81dce2ddc9f6e8f543d94b05d56e70d03a0774d32f6cca53e978dc01e4fc75b8"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9c6b4d23c04831e3ab61717a707a5d763b300213db49ca680edf8bf13ab5d91b"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:c15ad0aee158a15e17e0495e1e18741573d04eb6da06d8b84af726cfc1ed02ee"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:6239d4e2e0b52c8bd38c51b760cd870069f0bdf99700a62cd509d7a031749a55"}, + {file = "regex-2023.10.3-cp310-cp310-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:4a8bf76e3182797c6b1afa5b822d1d5802ff30284abe4599e1247be4fd6b03be"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_aarch64.whl", hash = "sha256:d9c727bbcf0065cbb20f39d2b4f932f8fa1631c3e01fcedc979bd4f51fe051c5"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_i686.whl", hash = "sha256:3ccf2716add72f80714b9a63899b67fa711b654be3fcdd34fa391d2d274ce767"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_ppc64le.whl", hash = "sha256:107ac60d1bfdc3edb53be75e2a52aff7481b92817cfdddd9b4519ccf0e54a6ff"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_s390x.whl", hash = "sha256:00ba3c9818e33f1fa974693fb55d24cdc8ebafcb2e4207680669d8f8d7cca79a"}, + {file = "regex-2023.10.3-cp310-cp310-musllinux_1_1_x86_64.whl", hash = "sha256:f0a47efb1dbef13af9c9a54a94a0b814902e547b7f21acb29434504d18f36e3a"}, + {file = "regex-2023.10.3-cp310-cp310-win32.whl", hash = "sha256:36362386b813fa6c9146da6149a001b7bd063dabc4d49522a1f7aa65b725c7ec"}, + {file = "regex-2023.10.3-cp310-cp310-win_amd64.whl", hash = "sha256:c65a3b5330b54103e7d21cac3f6bf3900d46f6d50138d73343d9e5b2900b2353"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_10_9_x86_64.whl", hash = "sha256:90a79bce019c442604662d17bf69df99090e24cdc6ad95b18b6725c2988a490e"}, + {file = "regex-2023.10.3-cp311-cp311-macosx_11_0_arm64.whl", hash = "sha256:c7964c2183c3e6cce3f497e3a9f49d182e969f2dc3aeeadfa18945ff7bdd7051"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4ef80829117a8061f974b2fda8ec799717242353bff55f8a29411794d635d964"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:5addc9d0209a9afca5fc070f93b726bf7003bd63a427f65ef797a931782e7edc"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c148bec483cc4b421562b4bcedb8e28a3b84fcc8f0aa4418e10898f3c2c0eb9b"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:8d1f21af4c1539051049796a0f50aa342f9a27cde57318f2fc41ed50b0dbc4ac"}, + {file = "regex-2023.10.3-cp311-cp311-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:0b9ac09853b2a3e0d0082104036579809679e7715671cfbf89d83c1cb2a30f58"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_aarch64.whl", hash = "sha256:ebedc192abbc7fd13c5ee800e83a6df252bec691eb2c4bedc9f8b2e2903f5e2a"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_i686.whl", hash = "sha256:d8a993c0a0ffd5f2d3bda23d0cd75e7086736f8f8268de8a82fbc4bd0ac6791e"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_ppc64le.whl", hash = "sha256:be6b7b8d42d3090b6c80793524fa66c57ad7ee3fe9722b258aec6d0672543fd0"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_s390x.whl", hash = "sha256:4023e2efc35a30e66e938de5aef42b520c20e7eda7bb5fb12c35e5d09a4c43f6"}, + {file = "regex-2023.10.3-cp311-cp311-musllinux_1_1_x86_64.whl", hash = "sha256:0d47840dc05e0ba04fe2e26f15126de7c755496d5a8aae4a08bda4dd8d646c54"}, + {file = "regex-2023.10.3-cp311-cp311-win32.whl", hash = "sha256:9145f092b5d1977ec8c0ab46e7b3381b2fd069957b9862a43bd383e5c01d18c2"}, + {file = "regex-2023.10.3-cp311-cp311-win_amd64.whl", hash = "sha256:b6104f9a46bd8743e4f738afef69b153c4b8b592d35ae46db07fc28ae3d5fb7c"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_10_9_x86_64.whl", hash = "sha256:bff507ae210371d4b1fe316d03433ac099f184d570a1a611e541923f78f05037"}, + {file = "regex-2023.10.3-cp312-cp312-macosx_11_0_arm64.whl", hash = "sha256:be5e22bbb67924dea15039c3282fa4cc6cdfbe0cbbd1c0515f9223186fc2ec5f"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:4a992f702c9be9c72fa46f01ca6e18d131906a7180950958f766c2aa294d4b41"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7434a61b158be563c1362d9071358f8ab91b8d928728cd2882af060481244c9e"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:c2169b2dcabf4e608416f7f9468737583ce5f0a6e8677c4efbf795ce81109d7c"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:a9e908ef5889cda4de038892b9accc36d33d72fb3e12c747e2799a0e806ec841"}, + {file = "regex-2023.10.3-cp312-cp312-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:12bd4bc2c632742c7ce20db48e0d99afdc05e03f0b4c1af90542e05b809a03d9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_aarch64.whl", hash = "sha256:bc72c231f5449d86d6c7d9cc7cd819b6eb30134bb770b8cfdc0765e48ef9c420"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_i686.whl", hash = "sha256:bce8814b076f0ce5766dc87d5a056b0e9437b8e0cd351b9a6c4e1134a7dfbda9"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_ppc64le.whl", hash = "sha256:ba7cd6dc4d585ea544c1412019921570ebd8a597fabf475acc4528210d7c4a6f"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_s390x.whl", hash = "sha256:b0c7d2f698e83f15228ba41c135501cfe7d5740181d5903e250e47f617eb4292"}, + {file = "regex-2023.10.3-cp312-cp312-musllinux_1_1_x86_64.whl", hash = "sha256:5a8f91c64f390ecee09ff793319f30a0f32492e99f5dc1c72bc361f23ccd0a9a"}, + {file = "regex-2023.10.3-cp312-cp312-win32.whl", hash = "sha256:ad08a69728ff3c79866d729b095872afe1e0557251da4abb2c5faff15a91d19a"}, + {file = "regex-2023.10.3-cp312-cp312-win_amd64.whl", hash = "sha256:39cdf8d141d6d44e8d5a12a8569d5a227f645c87df4f92179bd06e2e2705e76b"}, + {file = "regex-2023.10.3-cp37-cp37m-macosx_10_9_x86_64.whl", hash = "sha256:4a3ee019a9befe84fa3e917a2dd378807e423d013377a884c1970a3c2792d293"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:76066d7ff61ba6bf3cb5efe2428fc82aac91802844c022d849a1f0f53820502d"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:bfe50b61bab1b1ec260fa7cd91106fa9fece57e6beba05630afe27c71259c59b"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:9fd88f373cb71e6b59b7fa597e47e518282455c2734fd4306a05ca219a1991b0"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:b3ab05a182c7937fb374f7e946f04fb23a0c0699c0450e9fb02ef567412d2fa3"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:dac37cf08fcf2094159922edc7a2784cfcc5c70f8354469f79ed085f0328ebdf"}, + {file = "regex-2023.10.3-cp37-cp37m-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:e54ddd0bb8fb626aa1f9ba7b36629564544954fff9669b15da3610c22b9a0991"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_aarch64.whl", hash = "sha256:3367007ad1951fde612bf65b0dffc8fd681a4ab98ac86957d16491400d661302"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_i686.whl", hash = "sha256:16f8740eb6dbacc7113e3097b0a36065a02e37b47c936b551805d40340fb9971"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_ppc64le.whl", hash = "sha256:f4f2ca6df64cbdd27f27b34f35adb640b5d2d77264228554e68deda54456eb11"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_s390x.whl", hash = "sha256:39807cbcbe406efca2a233884e169d056c35aa7e9f343d4e78665246a332f597"}, + {file = "regex-2023.10.3-cp37-cp37m-musllinux_1_1_x86_64.whl", hash = "sha256:7eece6fbd3eae4a92d7c748ae825cbc1ee41a89bb1c3db05b5578ed3cfcfd7cb"}, + {file = "regex-2023.10.3-cp37-cp37m-win32.whl", hash = "sha256:ce615c92d90df8373d9e13acddd154152645c0dc060871abf6bd43809673d20a"}, + {file = "regex-2023.10.3-cp37-cp37m-win_amd64.whl", hash = "sha256:0f649fa32fe734c4abdfd4edbb8381c74abf5f34bc0b3271ce687b23729299ed"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_10_9_x86_64.whl", hash = "sha256:9b98b7681a9437262947f41c7fac567c7e1f6eddd94b0483596d320092004533"}, + {file = "regex-2023.10.3-cp38-cp38-macosx_11_0_arm64.whl", hash = "sha256:91dc1d531f80c862441d7b66c4505cd6ea9d312f01fb2f4654f40c6fdf5cc37a"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:82fcc1f1cc3ff1ab8a57ba619b149b907072e750815c5ba63e7aa2e1163384a4"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:7979b834ec7a33aafae34a90aad9f914c41fd6eaa8474e66953f3f6f7cbd4368"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:ef71561f82a89af6cfcbee47f0fabfdb6e63788a9258e913955d89fdd96902ab"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:dd829712de97753367153ed84f2de752b86cd1f7a88b55a3a775eb52eafe8a94"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:00e871d83a45eee2f8688d7e6849609c2ca2a04a6d48fba3dff4deef35d14f07"}, + {file = "regex-2023.10.3-cp38-cp38-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:706e7b739fdd17cb89e1fbf712d9dc21311fc2333f6d435eac2d4ee81985098c"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_aarch64.whl", hash = "sha256:cc3f1c053b73f20c7ad88b0d1d23be7e7b3901229ce89f5000a8399746a6e039"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_i686.whl", hash = "sha256:6f85739e80d13644b981a88f529d79c5bdf646b460ba190bffcaf6d57b2a9863"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_ppc64le.whl", hash = "sha256:741ba2f511cc9626b7561a440f87d658aabb3d6b744a86a3c025f866b4d19e7f"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_s390x.whl", hash = "sha256:e77c90ab5997e85901da85131fd36acd0ed2221368199b65f0d11bca44549711"}, + {file = "regex-2023.10.3-cp38-cp38-musllinux_1_1_x86_64.whl", hash = "sha256:979c24cbefaf2420c4e377ecd1f165ea08cc3d1fbb44bdc51bccbbf7c66a2cb4"}, + {file = "regex-2023.10.3-cp38-cp38-win32.whl", hash = "sha256:58837f9d221744d4c92d2cf7201c6acd19623b50c643b56992cbd2b745485d3d"}, + {file = "regex-2023.10.3-cp38-cp38-win_amd64.whl", hash = "sha256:c55853684fe08d4897c37dfc5faeff70607a5f1806c8be148f1695be4a63414b"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_10_9_x86_64.whl", hash = "sha256:2c54e23836650bdf2c18222c87f6f840d4943944146ca479858404fedeb9f9af"}, + {file = "regex-2023.10.3-cp39-cp39-macosx_11_0_arm64.whl", hash = "sha256:69c0771ca5653c7d4b65203cbfc5e66db9375f1078689459fe196fe08b7b4930"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:6ac965a998e1388e6ff2e9781f499ad1eaa41e962a40d11c7823c9952c77123e"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:1c0e8fae5b27caa34177bdfa5a960c46ff2f78ee2d45c6db15ae3f64ecadde14"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:6c56c3d47da04f921b73ff9415fbaa939f684d47293f071aa9cbb13c94afc17d"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:7ef1e014eed78ab650bef9a6a9cbe50b052c0aebe553fb2881e0453717573f52"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_i686.manylinux1_i686.manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:d29338556a59423d9ff7b6eb0cb89ead2b0875e08fe522f3e068b955c3e7b59b"}, + {file = "regex-2023.10.3-cp39-cp39-manylinux_2_5_x86_64.manylinux1_x86_64.manylinux_2_12_x86_64.manylinux2010_x86_64.whl", hash = "sha256:9c6d0ced3c06d0f183b73d3c5920727268d2201aa0fe6d55c60d68c792ff3588"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_aarch64.whl", hash = "sha256:994645a46c6a740ee8ce8df7911d4aee458d9b1bc5639bc968226763d07f00fa"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_i686.whl", hash = "sha256:66e2fe786ef28da2b28e222c89502b2af984858091675044d93cb50e6f46d7af"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_ppc64le.whl", hash = "sha256:11175910f62b2b8c055f2b089e0fedd694fe2be3941b3e2633653bc51064c528"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_s390x.whl", hash = "sha256:06e9abc0e4c9ab4779c74ad99c3fc10d3967d03114449acc2c2762ad4472b8ca"}, + {file = "regex-2023.10.3-cp39-cp39-musllinux_1_1_x86_64.whl", hash = "sha256:fb02e4257376ae25c6dd95a5aec377f9b18c09be6ebdefa7ad209b9137b73d48"}, + {file = "regex-2023.10.3-cp39-cp39-win32.whl", hash = "sha256:3b2c3502603fab52d7619b882c25a6850b766ebd1b18de3df23b2f939360e1bd"}, + {file = "regex-2023.10.3-cp39-cp39-win_amd64.whl", hash = "sha256:adbccd17dcaff65704c856bd29951c58a1bd4b2b0f8ad6b826dbd543fe740988"}, + {file = "regex-2023.10.3.tar.gz", hash = "sha256:3fef4f844d2290ee0ba57addcec17eec9e3df73f10a2748485dfd6a3a188cc0f"}, +] + [[package]] name = "requests" version = "2.28.1" description = "Python HTTP for Humans." +category = "main" optional = false python-versions = ">=3.7, <4" files = [ @@ -1793,6 +1967,7 @@ use-chardet-on-py3 = ["chardet (>=3.0.2,<6)"] name = "rich" version = "12.0.1" description = "Render rich text, tables, progress bars, syntax highlighting, markdown and more to the terminal" +category = "main" optional = false python-versions = ">=3.6.2,<4.0.0" files = [ @@ -1811,6 +1986,7 @@ jupyter = ["ipywidgets (>=7.5.1,<8.0.0)"] name = "ruff" version = "0.1.4" description = "An extremely fast Python linter and code formatter, written in Rust." +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -1837,6 +2013,7 @@ files = [ name = "runtype" version = "0.2.7" description = "Type dispatch and validation for run-time Python" +category = "main" optional = false python-versions = ">=3.6,<4.0" files = [ @@ -1848,6 +2025,7 @@ files = [ name = "secretstorage" version = "3.3.3" description = "Python bindings to FreeDesktop.org Secret Service API" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -1863,6 +2041,7 @@ jeepney = ">=0.6" name = "setuptools" version = "65.6.3" description = "Easily download, build, install, upgrade, and uninstall Python packages" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1879,6 +2058,7 @@ testing-integration = ["build[virtualenv]", "filelock (>=3.4.0)", "jaraco.envs ( name = "six" version = "1.16.0" description = "Python 2 and 3 compatibility utilities" +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -1890,6 +2070,7 @@ files = [ name = "snowflake-connector-python" version = "3.0.4" description = "Snowflake Connector for Python" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1948,6 +2129,7 @@ secure-local-storage = ["keyring (!=16.1.0,<24.0.0)"] name = "sortedcontainers" version = "2.4.0" description = "Sorted Containers -- Sorted List, Sorted Dict, Sorted Set" +category = "main" optional = false python-versions = "*" files = [ @@ -1959,6 +2141,7 @@ files = [ name = "sqlparse" version = "0.4.3" description = "A non-validating SQL parser." +category = "main" optional = false python-versions = ">=3.5" files = [ @@ -1970,6 +2153,7 @@ files = [ name = "tabulate" version = "0.9.0" description = "Pretty-print tabular data" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -1984,6 +2168,7 @@ widechars = ["wcwidth"] name = "text-unidecode" version = "1.3" description = "The most basic Text::Unidecode port" +category = "main" optional = false python-versions = "*" files = [ @@ -1995,6 +2180,7 @@ files = [ name = "toml" version = "0.10.2" description = "Python Library for Tom's Obvious, Minimal Language" +category = "main" optional = false python-versions = ">=2.6, !=3.0.*, !=3.1.*, !=3.2.*" files = [ @@ -2002,10 +2188,40 @@ files = [ {file = "toml-0.10.2.tar.gz", hash = "sha256:b3bda1d108d5dd99f4a20d24d9c348e91c4db7ab1b749200bded2f839ccbe68f"}, ] +[[package]] +name = "toml-cli" +version = "0.3.1" +description = "Command line interface to read and write keys/values to/from toml files" +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "toml-cli-0.3.1.tar.gz", hash = "sha256:6f1d92a15d8bdab0423682b4ca13e694a320d1fb117a5a285c0f28ac4d668e91"}, + {file = "toml_cli-0.3.1-py3-none-any.whl", hash = "sha256:1ca062222d09deef1a92fa8ef464816e7a643206a77fcf48a6696d9174d48966"}, +] + +[package.dependencies] +regex = ">=2020.7.14" +tomlkit = ">=0.7.2" +typer = ">=0.3.2" + +[[package]] +name = "tomlkit" +version = "0.12.3" +description = "Style preserving TOML library" +category = "main" +optional = false +python-versions = ">=3.7" +files = [ + {file = "tomlkit-0.12.3-py3-none-any.whl", hash = "sha256:b0a645a9156dc7cb5d3a1f0d4bab66db287fcb8e0430bdd4664a095ea16414ba"}, + {file = "tomlkit-0.12.3.tar.gz", hash = "sha256:75baf5012d06501f07bee5bf8e801b9f343e7aac5a92581f20f80ce632e6b5a4"}, +] + [[package]] name = "trino" version = "0.314.0" description = "Client for the Trino distributed SQL Engine" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2024,10 +2240,33 @@ kerberos = ["requests-kerberos"] sqlalchemy = ["sqlalchemy (>=1.3,<2.0)"] tests = ["click", "httpretty (<1.1)", "pytest", "pytest-runner", "requests-kerberos", "sqlalchemy (>=1.3,<2.0)"] +[[package]] +name = "typer" +version = "0.9.0" +description = "Typer, build great CLIs. Easy to code. Based on Python type hints." +category = "main" +optional = false +python-versions = ">=3.6" +files = [ + {file = "typer-0.9.0-py3-none-any.whl", hash = "sha256:5d96d986a21493606a358cae4461bd8cdf83cbf33a5aa950ae629ca3b51467ee"}, + {file = "typer-0.9.0.tar.gz", hash = "sha256:50922fd79aea2f4751a8e0408ff10d2662bd0c8bbfa84755a699f3bada2978b2"}, +] + +[package.dependencies] +click = ">=7.1.1,<9.0.0" +typing-extensions = ">=3.7.4.3" + +[package.extras] +all = ["colorama (>=0.4.3,<0.5.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] +dev = ["autoflake (>=1.3.1,<2.0.0)", "flake8 (>=3.8.3,<4.0.0)", "pre-commit (>=2.17.0,<3.0.0)"] +doc = ["cairosvg (>=2.5.2,<3.0.0)", "mdx-include (>=1.4.1,<2.0.0)", "mkdocs (>=1.1.2,<2.0.0)", "mkdocs-material (>=8.1.4,<9.0.0)", "pillow (>=9.3.0,<10.0.0)"] +test = ["black (>=22.3.0,<23.0.0)", "coverage (>=6.2,<7.0)", "isort (>=5.0.6,<6.0.0)", "mypy (==0.910)", "pytest (>=4.4.0,<8.0.0)", "pytest-cov (>=2.10.0,<5.0.0)", "pytest-sugar (>=0.9.4,<0.10.0)", "pytest-xdist (>=1.32.0,<4.0.0)", "rich (>=10.11.0,<14.0.0)", "shellingham (>=1.3.0,<2.0.0)"] + [[package]] name = "typing-extensions" version = "4.7.1" description = "Backported and Experimental Type Hints for Python 3.7+" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2039,6 +2278,7 @@ files = [ name = "tzdata" version = "2022.7" description = "Provider of IANA time zone data" +category = "main" optional = false python-versions = ">=2" files = [ @@ -2050,6 +2290,7 @@ files = [ name = "tzlocal" version = "4.2" description = "tzinfo object for the local timezone" +category = "main" optional = false python-versions = ">=3.6" files = [ @@ -2070,6 +2311,7 @@ test = ["pytest (>=4.3)", "pytest-mock (>=3.3)"] name = "unittest-parallel" version = "1.5.3" description = "Parallel unit test runner with coverage support" +category = "dev" optional = false python-versions = "*" files = [ @@ -2084,6 +2326,7 @@ coverage = ">=5.1" name = "urllib3" version = "1.26.13" description = "HTTP library with thread-safe connection pooling, file post, and more." +category = "main" optional = false python-versions = ">=2.7, !=3.0.*, !=3.1.*, !=3.2.*, !=3.3.*, !=3.4.*, !=3.5.*" files = [ @@ -2100,6 +2343,7 @@ socks = ["PySocks (>=1.5.6,!=1.5.7,<2.0)"] name = "vertica-python" version = "1.3.2" description = "Official native Python client for the Vertica database." +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2115,6 +2359,7 @@ six = ">=1.10.0" name = "virtualenv" version = "20.24.6" description = "Virtual Python Environment builder" +category = "dev" optional = false python-versions = ">=3.7" files = [ @@ -2135,6 +2380,7 @@ test = ["covdefaults (>=2.3)", "coverage (>=7.2.7)", "coverage-enable-subprocess name = "wcwidth" version = "0.2.5" description = "Measures the displayed width of unicode strings in a terminal" +category = "main" optional = false python-versions = "*" files = [ @@ -2146,6 +2392,7 @@ files = [ name = "zipp" version = "3.11.0" description = "Backport of pathlib-compatible object wrapper for zip files" +category = "main" optional = false python-versions = ">=3.7" files = [ @@ -2174,4 +2421,4 @@ vertica = ["vertica-python"] [metadata] lock-version = "2.0" python-versions = "^3.8.0" -content-hash = "b7f8880be9658fa523ff4737d1fdefd09d124ab09fac22fa9ec6a83454940c1d" +content-hash = "a540c23a2b4d9e69f2a0a5b228dd5db0330669fdd4988f2e82c02887b8d44c1f" diff --git a/pyproject.toml b/pyproject.toml index 1b380dde..38a7a63a 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -46,6 +46,7 @@ pyodbc = {version="^4.0.39", optional=true} typing-extensions = ">=4.0.1" attrs = "^23.1.0" mashumaro = {version = ">=2.9,<3.11.0", extras = ["msgpack"]} +toml-cli = "0.3.*" [tool.poetry.dev-dependencies] parameterized = "*" From 167932c439833eabc948f514d257bafa10022fb1 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 15:41:52 -0800 Subject: [PATCH 17/26] supress errors --- .github/workflows/_publish-data-platform-data-diff.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index cd7289da..519a7198 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -55,7 +55,7 @@ jobs: env: AWS_REGION: ${{ secrets.AWS_REGION }} run: | - VERSION=$(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) || { echo "FAILED TO GET POETRY VERSION"; exit 1;} + VERSION=$(poetry run toml get --toml-path pyproject.toml tool.poetry.version 2>/dev/null) || { echo "FAILED TO GET POETRY VERSION"; exit 1;} echo $VERSION > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV @@ -67,13 +67,13 @@ jobs: run: | echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt echo "To install this wheel \`poetry add \"data-platform-data-diff==$(cat version.txt)[all]\" --source artifact\`" - - poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt) || { echo "Failed to set version in pyproject.toml"; exit 1; } - + + poetry run toml set --toml-path pyproject.toml tool.poetry.version $(cat version.txt) || { echo "Failed to set version in pyproject.toml"; exit 1; } + cd python poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) || { echo "Failed to publish the package"; exit 1; } - + - name: Publish new version if: ${{ github.ref_name == 'main' }} run: | From 5e5bd5129bde3e9b2685f0cb2a444adcc0198872 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 15:42:50 -0800 Subject: [PATCH 18/26] supress errors --- .github/workflows/_publish-data-platform-data-diff.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 519a7198..f43b2f9b 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -59,7 +59,7 @@ jobs: echo $VERSION > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV - VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) || { echo "Error querying AWS CodeArtifact, repository might not exist."; exit 1; } + VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME 2>/dev/null | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) || { echo "Error querying AWS CodeArtifact, repository might not exist."; exit 1; } echo "VERSION_ALREADY_PUBLISHED=$VERSION_ALREADY_PUBLISHED >> $GITHUB_ENV - name: Publish dev version @@ -72,15 +72,15 @@ jobs: cd python poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) || { echo "Failed to publish the package"; exit 1; } + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } - name: Publish new version if: ${{ github.ref_name == 'main' }} run: | if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then cd python - poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text) || { echo "Failed to publish the package"; exit 1; } + poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } + poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } else echo "Version already published, skipping publish step." fi From 2890c1373b127448c5ccf4b3a846111810681e38 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 17:10:42 -0800 Subject: [PATCH 19/26] add url for poetry and add more safe guards --- .../_publish-data-platform-data-diff.yml | 55 ++++++++++++------- pyproject.toml | 5 ++ 2 files changed, 41 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index f43b2f9b..8bf1835d 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -54,33 +54,50 @@ jobs: - name: Set env variables env: AWS_REGION: ${{ secrets.AWS_REGION }} + CODEARTIFACT_URL: ${{ secrets.CODEARTIFACT_URL }} # Add your CodeArtifact URL as a secret run: | - VERSION=$(poetry run toml get --toml-path pyproject.toml tool.poetry.version 2>/dev/null) || { echo "FAILED TO GET POETRY VERSION"; exit 1;} + # Replace placeholder URL with actual repository URL + sed -i "s|PLACEHOLDER_URL|$CODEARTIFACT_URL|" pyproject.toml + + VERSION=$(poetry run toml get --toml-path pyproject.toml tool.poetry.version 2>/dev/null) || { echo "FAILED TO GET POETRY VERSION"; exit 1; } echo $VERSION > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV - - VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region $AWS_REGION list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME 2>/dev/null | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count) || { echo "Error querying AWS CodeArtifact, repository might not exist."; exit 1; } - echo "VERSION_ALREADY_PUBLISHED=$VERSION_ALREADY_PUBLISHED >> $GITHUB_ENV - - name: Publish dev version - if: ${{ github.ref_name != 'main' }} + - name: Check AWS CodeArtifact repository existence + if: ${{ github.ref_name == 'master' }} + env: + AWS_REGION: ${{ secrets.AWS_REGION }} run: | - echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt - echo "To install this wheel \`poetry add \"data-platform-data-diff==$(cat version.txt)[all]\" --source artifact\`" + if ! aws codeartifact list-repositories --region $AWS_REGION | grep -q "data-platform-data-diff"; then + echo "AWS CodeArtifact repository 'data-platform-data-diff' does not exist." + exit 0 + fi - poetry run toml set --toml-path pyproject.toml tool.poetry.version $(cat version.txt) || { echo "Failed to set version in pyproject.toml"; exit 1; } + - name: Check if version needs to be published + if: ${{ github.ref_name == 'master' }} + env: + AWS_REGION: ${{ secrets.AWS_REGION }} + id: check_version + run: | + if ! aws codeartifact list-package-versions --region $AWS_REGION --domain coinlist --repository data-platform-data-diff --format pypi --package data_diff 2>/dev/null | grep -q "$(cat version.txt | sed 's/\./\\./g')"; then + echo "skip_publish=false" >> $GITHUB_ENV + else + echo "skip_publish=true" >> $GITHUB_ENV + fi + - name: Publish dev version + if: ${{ github.ref_name != 'master' }} + run: | cd python - poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } + DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" + echo $DEV_VERSION > version.txt + poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } + poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } + poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; } - name: Publish new version - if: ${{ github.ref_name == 'main' }} + if: ${{ github.ref_name == 'master' }} && ${{ env.skip_publish != 'true' }} run: | - if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then - cd python - poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } - poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } - else - echo "Version already published, skipping publish step." - fi + cd python + poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } + poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } diff --git a/pyproject.toml b/pyproject.toml index 38a7a63a..48f31d7d 100755 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,6 +20,11 @@ classifiers = [ "Typing :: Typed" ] packages = [{ include = "data_diff" }] + +[tools.poetry.source] +name = "data-platform-data-diff" +url = "PLACEHOLDER_URL" + [tool.poetry.dependencies] pydantic = "1.10.12" python = "^3.8.0" From 26608ffca2ea5a29ccf986036779f057eef5fbf1 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 17:21:03 -0800 Subject: [PATCH 20/26] remove python dir --- .github/workflows/_publish-data-platform-data-diff.yml | 2 -- 1 file changed, 2 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 8bf1835d..a904cfd3 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -88,7 +88,6 @@ jobs: - name: Publish dev version if: ${{ github.ref_name != 'master' }} run: | - cd python DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" echo $DEV_VERSION > version.txt poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } @@ -98,6 +97,5 @@ jobs: - name: Publish new version if: ${{ github.ref_name == 'master' }} && ${{ env.skip_publish != 'true' }} run: | - cd python poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; } From c592ad15ba3549dc702c9c081bbc0bad16e28461 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 21:33:28 -0800 Subject: [PATCH 21/26] check if queryable --- .../workflows/_publish-data-platform-data-diff.yml | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index a904cfd3..c161ffd8 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -51,10 +51,20 @@ jobs: path: .venv key: ${{ inputs.python_cache_key }} + - name: Install jq + run: sudo apt-get update && sudo apt-get install -y jq + + - name: Check if AWS CodeArtifact is queryable and count repositories + env: + AWS_REGION: ${{ secrets.AWS_REGION }} + run: | + repo_count=$(aws codeartifact list-repositories --region $AWS_REGION --output json | jq '.repositories | length' 2>/dev/null) || { echo "Failed to query AWS CodeArtifact or an error occurred."; exit 0; } + echo "Number of repositories in AWS CodeArtifact: $repo_count" + - name: Set env variables env: AWS_REGION: ${{ secrets.AWS_REGION }} - CODEARTIFACT_URL: ${{ secrets.CODEARTIFACT_URL }} # Add your CodeArtifact URL as a secret + CODEARTIFACT_URL: ${{ secrets.CODEARTIFACT_URL }} run: | # Replace placeholder URL with actual repository URL sed -i "s|PLACEHOLDER_URL|$CODEARTIFACT_URL|" pyproject.toml From 65882b88c4d42e2ef3344e67394c2f700476010d Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 22:04:23 -0800 Subject: [PATCH 22/26] not queryable --- .../_publish-data-platform-data-diff.yml | 21 ++----------------- 1 file changed, 2 insertions(+), 19 deletions(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index c161ffd8..6c60b1c7 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -36,12 +36,12 @@ jobs: id: setup-python uses: actions/setup-python@v4 with: - python-version: '${{ env.PYTHON_VERSION }}' + python-version: '${{ inputs.python_version }}' - name: Install and configure Poetry uses: snok/install-poetry@v1 with: - version: ${{ env.POETRY_VERSION }} + version: ${{ inputs.poetry_version }} virtualenvs-in-project: true - name: Restore cached key @@ -54,13 +54,6 @@ jobs: - name: Install jq run: sudo apt-get update && sudo apt-get install -y jq - - name: Check if AWS CodeArtifact is queryable and count repositories - env: - AWS_REGION: ${{ secrets.AWS_REGION }} - run: | - repo_count=$(aws codeartifact list-repositories --region $AWS_REGION --output json | jq '.repositories | length' 2>/dev/null) || { echo "Failed to query AWS CodeArtifact or an error occurred."; exit 0; } - echo "Number of repositories in AWS CodeArtifact: $repo_count" - - name: Set env variables env: AWS_REGION: ${{ secrets.AWS_REGION }} @@ -73,16 +66,6 @@ jobs: echo $VERSION > version.txt echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV - - name: Check AWS CodeArtifact repository existence - if: ${{ github.ref_name == 'master' }} - env: - AWS_REGION: ${{ secrets.AWS_REGION }} - run: | - if ! aws codeartifact list-repositories --region $AWS_REGION | grep -q "data-platform-data-diff"; then - echo "AWS CodeArtifact repository 'data-platform-data-diff' does not exist." - exit 0 - fi - - name: Check if version needs to be published if: ${{ github.ref_name == 'master' }} env: From 72c55bfd6b48a959e194701b458c31da9f30e2bb Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 23:07:57 -0800 Subject: [PATCH 23/26] explicit config --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 6c60b1c7..fbd796a0 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -84,7 +84,7 @@ jobs: DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" echo $DEV_VERSION > version.txt poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } - poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; } + poetry config repositories.data-platform-data-diff $CODEARTIFACT_URL poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; } - name: Publish new version From 4ccd9ab800775ffb2bf027558abab018f56671e5 Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 23:10:27 -0800 Subject: [PATCH 24/26] add secrets --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index fbd796a0..17998c60 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -84,7 +84,7 @@ jobs: DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" echo $DEV_VERSION > version.txt poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } - poetry config repositories.data-platform-data-diff $CODEARTIFACT_URL + poetry config repositories.data-platform-data-diff ${{ secrets.CODEARTIFACT_URL }} poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; } - name: Publish new version From 98c9230756b2c9542459d0ae4b209cc7fb38c0ff Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 23:27:42 -0800 Subject: [PATCH 25/26] add build --- .github/workflows/_publish-data-platform-data-diff.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 17998c60..7e3ace69 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -84,6 +84,7 @@ jobs: DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" echo $DEV_VERSION > version.txt poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } + poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } poetry config repositories.data-platform-data-diff ${{ secrets.CODEARTIFACT_URL }} poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; } From 35e2596dbb1a6e9164d6bf871941973ddedeb18e Mon Sep 17 00:00:00 2001 From: Michal Ondras Date: Thu, 14 Dec 2023 23:31:50 -0800 Subject: [PATCH 26/26] config build publish --- .github/workflows/_publish-data-platform-data-diff.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/_publish-data-platform-data-diff.yml b/.github/workflows/_publish-data-platform-data-diff.yml index 7e3ace69..acfa544c 100644 --- a/.github/workflows/_publish-data-platform-data-diff.yml +++ b/.github/workflows/_publish-data-platform-data-diff.yml @@ -84,8 +84,8 @@ jobs: DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" echo $DEV_VERSION > version.txt poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; } - poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } poetry config repositories.data-platform-data-diff ${{ secrets.CODEARTIFACT_URL }} + poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; } poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; } - name: Publish new version