add ci/cd

.github/workflows/CODEOWNERS

@@ -0,0 +1 @@
+* @coinlist/data

.github/workflows/_cache-dependencies.yml

@@ -0,0 +1,51 @@
+name: cache_dependencies
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+    outputs:
+      python_cache_key:
+        description: "The key of the primary cache of the python dependencies"
+        value: ${{ jobs.python-cache.outputs.key }}
+
+
+jobs:
+  python-cache:
+    runs-on: ubuntu-latest
+    outputs:
+      key: ${{ steps.define-cache-key.outputs.cache_key }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: '${{ inputs.python_version }}'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ inputs.poetry_version }}
+          virtualenvs-in-project: true
+
+      - name: Define Cache Key
+        id: define-cache-key
+        run: |
+          echo "cache_key=python-${{ runner.os }}--${{ inputs.python_version }}-${{ inputs.poetry_version }}-${{ hashFiles('**/poetry.lock') }}" >> $GITHUB_OUTPUT
+
+      - name: Cache venv
+        id: cached-python
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: ${{ steps.define-cache-key.outputs.cache_key }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root

.github/workflows/_publish-data-platform-data-diff.yml

@@ -0,0 +1,85 @@
+name: publish_data-platform-data-diff
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+      python_cache_key:
+        required: true
+        type: string
+      role_to_assume:
+        required: true
+        type: string
+      aws_region:
+        required: true
+        type: string
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    name: 'Publish python SDK cl-data'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ inputs.role_to_assume }}
+          aws-region: ${{ inputs.aws_region }}
+          mask-aws-account-id: 'yes'
+
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '${{ env.PYTHON_VERSION }}'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ env.POETRY_VERSION }}
+          virtualenvs-in-project: true
+
+      - name: Restore cached key
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: .venv
+          key: ${{ inputs.python_cache_key }}
+
+      - name: Set env variables
+        run: |
+          echo $(poetry run toml get --toml-path python/pyproject.toml tool.poetry.version) > version.txt
+          echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV
+
+          echo "VERSION_ALREADY_PUBLISHED=$(aws codeartifact --region ${{ inputs.aws_region }} list-package-versions --domain coinlist --repository data_platform_data_diff --format pypi --package data-platform-data-diff --query versions --output text --sort-by PUBLISHED_TIME | grep -e "$(cat version.txt | sed 's/\./\\./g')$" --count)" >> $GITHUB_ENV
+
+      - name: Publish dev version
+        if: ${{ github.ref_name != 'main' }}
+        run: |
+          echo "$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}" > version.txt
+          echo "To install this wheel \`poetry add \"data-platform-data-diff==$(cat version.txt)[all]\" --source artifact\`"
+
+          poetry run toml set --toml-path python/pyproject.toml tool.poetry.version $(cat version.txt)
+
+          cd python
+          poetry build --format wheel
+          poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text)
+
+      - name: Publish new version
+        if: ${{ github.ref_name == 'main' }}
+        run: |
+          if [[ $VERSION_ALREADY_PUBLISHED == 0 ]]; then
+              cd python
+              poetry build --format wheel
+              poetry publish --repository artifact --username aws --password $(aws codeartifact --region ${{ inputs.aws_region }} get-authorization-token --domain coinlist --query authorizationToken --output text)

.github/workflows/ci.yml

@@ -1,20 +1,13 @@
 name: CI-COVER-VERSIONS
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
-
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   unit_tests:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest]
         python-version:

.github/workflows/ci_full.yml

@@ -1,14 +1,8 @@
 name: CI-COVER-DATABASES
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
   workflow_dispatch:
+  workflow_call:
 
 permissions:
   id-token: write # This is required for requesting the JWT
@@ -17,7 +11,7 @@ permissions:
 jobs:
   unit_tests:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest]
         python-version:

.github/workflows/formatter.yml

@@ -1,9 +1,7 @@
 name: formatter
 on:
-    pull_request:
-        branches: [ master ]
-
-    workflow_dispatch:
+  workflow_dispatch:
+  worflow_call:
 
 jobs:
   linter_name:
@@ -19,4 +17,4 @@ jobs:
       - name: Auto commit ruff formatting
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: 'style fixes by ruff'
+          commit_message: 'style fixes by ruff'

.github/workflows/pull-request-checks.yml

@@ -0,0 +1,70 @@
+name: PR checks
+
+on:
+  pull_request: {}
+
+permissions:
+  id-token: write
+  contents: read
+  actions: write
+
+jobs:
+  cancel:
+    description: Cancel any in-progress runs of this workflow execution
+    runs-on: ubuntu-latest
+    steps:
+    - uses: styfle/cancel-workflow-action@0.12.0
+      with:
+        access_token: ${{ github.token }}
+
+  setup:
+    description: Setsup the necessary variables for Python and Poery Versions
+    runs-on: ubuntu-latest
+    needs: [cancel]
+    outputs:
+      python_version: ${{ steps.set_var.outputs.python_version }}
+      poetry_version: ${{ steps.set_var.outputs.poetry_version }}
+      role_to_assume: ${{ steps.set_var.outputs.role_to_assume }}
+      aws_region: ${{ steps.set_var.outputs.aws_region }}
+    steps:
+      - id: set_var
+        run: |
+          echo "python_version=3.8" >> $GITHUB_OUTPUT
+          echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT
+          echo "role_to_assume=arn:aws:iam::${{ secrets.AWS_ACCOUNT_ID}}:role/github-actions-data-platform-data-diff" >> $GITHUB_OUTPUT
+          echo "aws_region=${{ secrets.AWS_REGION }}" >> $GITHUB_OUTPUT
+
+  perform-formatting:
+    description: Formats the *.py files using ruff
+    needs: [setup]
+    uses: ./.github/workflows/formatter.yml
+
+  cache-dependencies:
+    description: Caches project dependencies for faster workflow execution
+    needs: [setup, preform-formatting]
+    uses: ./.github/workflows/_cache-dependencies.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+
+  run-unit-test-versions:
+    description: Runs the unit tests for the different versions of Python
+    needs: [setup]
+    uses: ./.github/workflows/ci_full.yml
+
+  run-unit-test-per-database:
+    description: Runs the unit tests for the different databases
+    needs: [setup]
+    uses: ./.github/workflows/ci.yml
+
+  publish-data-platform-data-diff-dev:
+    needs: [setup, cache-dependencies]
+    uses: ./.github/workflows/_publish-data-platform-data-diff.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+      python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }}
+      role_to_assume: ${{ needs.setup.outputs.role_to_assume }}
+      aws_region: ${{ needs.setup.outputs.aws_region }}