Merge pull request #2 from coinlist/add-ci-cd

add ci/cd

.github/workflows/CODEOWNERS

@@ -0,0 +1 @@
+* @coinlist/data

.github/workflows/_cache-dependencies.yml

@@ -0,0 +1,51 @@
+name: cache_dependencies
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+    outputs:
+      python_cache_key:
+        description: "The key of the primary cache of the python dependencies"
+        value: ${{ jobs.python-cache.outputs.key }}
+
+
+jobs:
+  python-cache:
+    runs-on: ubuntu-latest
+    outputs:
+      key: ${{ steps.define-cache-key.outputs.cache_key }}
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        uses: actions/setup-python@v4
+        id: setup-python
+        with:
+          python-version: '${{ inputs.python_version }}'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ inputs.poetry_version }}
+          virtualenvs-in-project: true
+
+      - name: Define Cache Key
+        id: define-cache-key
+        run: |
+          echo "cache_key=python-${{ runner.os }}--${{ inputs.python_version }}-${{ inputs.poetry_version }}-${{ hashFiles('**/poetry.lock') }}" >> $GITHUB_OUTPUT
+
+      - name: Cache venv
+        id: cached-python
+        uses: actions/cache@v3
+        with:
+          path: .venv
+          key: ${{ steps.define-cache-key.outputs.cache_key }}
+
+      - name: Install dependencies
+        run: poetry install --no-interaction --no-root

.github/workflows/_publish-data-platform-data-diff.yml

@@ -0,0 +1,95 @@
+name: publish_data-platform-data-diff
+
+on:
+  workflow_call:
+    inputs:
+      python_version:
+        required: true
+        type: string
+      poetry_version:
+        required: true
+        type: string
+      python_cache_key:
+        required: true
+        type: string
+
+permissions:
+  id-token: write
+  contents: read
+
+jobs:
+  build:
+    name: 'Publish python data-platform-data-diff'
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Setup AWS credentials
+        uses: aws-actions/configure-aws-credentials@v1
+        with:
+          role-to-assume: ${{ secrets.CROSS_ACCOUNT_ROLE_TO_ASSUME }}
+          aws-region: ${{ secrets.AWS_REGION }}
+          mask-aws-account-id: 'yes'
+
+      - uses: actions/checkout@v3
+
+      - name: Setup Python
+        id: setup-python
+        uses: actions/setup-python@v4
+        with:
+          python-version: '${{ inputs.python_version }}'
+
+      - name: Install and configure Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: ${{ inputs.poetry_version }}
+          virtualenvs-in-project: true
+
+      - name: Restore cached key
+        id: cache-restore
+        uses: actions/cache/restore@v3
+        with:
+          path: .venv
+          key: ${{ inputs.python_cache_key }}
+
+      - name: Install jq
+        run: sudo apt-get update && sudo apt-get install -y jq
+
+      - name: Set env variables
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+          CODEARTIFACT_URL: ${{ secrets.CODEARTIFACT_URL }}
+        run: |
+          # Replace placeholder URL with actual repository URL
+          sed -i "s|PLACEHOLDER_URL|$CODEARTIFACT_URL|" pyproject.toml
+
+          VERSION=$(poetry run toml get --toml-path pyproject.toml tool.poetry.version 2>/dev/null) || { echo "FAILED TO GET POETRY VERSION"; exit 1; }
+          echo $VERSION > version.txt
+          echo "CURRENT_VERSION=$(cat version.txt)" >> $GITHUB_ENV
+
+      - name: Check if version needs to be published
+        if: ${{ github.ref_name == 'master' }}
+        env:
+          AWS_REGION: ${{ secrets.AWS_REGION }}
+        id: check_version
+        run: |
+          if ! aws codeartifact list-package-versions --region $AWS_REGION --domain coinlist --repository data-platform-data-diff --format pypi --package data_diff 2>/dev/null | grep -q "$(cat version.txt | sed 's/\./\\./g')"; then
+            echo "skip_publish=false" >> $GITHUB_ENV
+          else
+            echo "skip_publish=true" >> $GITHUB_ENV
+          fi
+
+      - name: Publish dev version
+        if: ${{ github.ref_name != 'master' }}
+        run: |
+          DEV_VERSION="$CURRENT_VERSION-dev+${GITHUB_SHA:0:7}"
+          echo $DEV_VERSION > version.txt
+          poetry run toml set --toml-path pyproject.toml tool.poetry.version $DEV_VERSION || { echo "Failed to set dev version in pyproject.toml"; exit 1; }
+          poetry config repositories.data-platform-data-diff ${{ secrets.CODEARTIFACT_URL }}
+          poetry build --format wheel || { echo "Failed to build the wheel"; exit 1; }
+          poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the dev package"; exit 1; }
+
+      - name: Publish new version
+        if: ${{ github.ref_name == 'master' }} && ${{ env.skip_publish != 'true' }}
+        run: |
+          poetry build --format wheel 2>/dev/null || { echo "Failed to build the wheel"; exit 1; }
+          poetry publish --repository data-platform-data-diff --username aws --password $(aws codeartifact --region ${{ secrets.AWS_REGION }} get-authorization-token --domain coinlist --query authorizationToken --output text 2>/dev/null) || { echo "Failed to publish the package"; exit 1; }

.github/workflows/ci.yml

@@ -1,20 +1,13 @@
 name: CI-COVER-VERSIONS
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
-
   workflow_dispatch:
+  workflow_call:
 
 jobs:
   unit_tests:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest]
         python-version:

.github/workflows/ci_full.yml

@@ -1,14 +1,8 @@
 name: CI-COVER-DATABASES
 
 on:
-#   push:
-#     paths:
-#       - '**.py'
-#       - '.github/workflows/**'
-#       - '!dev/**'
-  pull_request:
-    branches: [ master ]
   workflow_dispatch:
+  workflow_call:
 
 permissions:
   id-token: write # This is required for requesting the JWT
@@ -17,7 +11,7 @@ permissions:
 jobs:
   unit_tests:
     strategy:
-      fail-fast: false
+      fail-fast: true
       matrix:
         os: [ubuntu-latest]
         python-version:

.github/workflows/formatter.yml

@@ -1,9 +1,7 @@
 name: formatter
 on:
-    pull_request:
-        branches: [ master ]
-
-    workflow_dispatch:
+  workflow_dispatch:
+  workflow_call:
 
 jobs:
   linter_name:
@@ -19,4 +17,4 @@ jobs:
       - name: Auto commit ruff formatting
         uses: stefanzweifel/git-auto-commit-action@v5
         with:
-          commit_message: 'style fixes by ruff'
+          commit_message: 'style fixes by ruff'

.github/workflows/pull-request-checks.yml

@@ -0,0 +1,58 @@
+name: PR checks
+
+on:
+  pull_request: {}
+
+permissions:
+  id-token: write
+  contents: read
+  actions: write
+
+jobs:
+  cancel:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: styfle/cancel-workflow-action@0.12.0
+      with:
+        access_token: ${{ github.token }}
+
+  setup:
+    runs-on: ubuntu-latest
+    needs: [cancel]
+    outputs:
+      python_version: ${{ steps.set_var.outputs.python_version }}
+      poetry_version: ${{ steps.set_var.outputs.poetry_version }}
+    steps:
+      - id: set_var
+        run: |
+          echo "python_version=3.8" >> $GITHUB_OUTPUT
+          echo "poetry_version=1.7.1" >> $GITHUB_OUTPUT
+
+  perform-ruff-formatting:
+    needs: [setup]
+    uses: ./.github/workflows/formatter.yml
+
+  cache-dependencies:
+    needs: [setup, perform-ruff-formatting]
+    uses: ./.github/workflows/_cache-dependencies.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+
+  run-unit-test-versions:
+    needs: [setup]
+    uses: ./.github/workflows/ci_full.yml
+
+  run-unit-test-per-database:
+    needs: [setup]
+    uses: ./.github/workflows/ci.yml
+
+  publish-data-platform-data-diff:
+    needs: [setup, run-unit-test-versions, run-unit-test-per-database, cache-dependencies]
+    uses: ./.github/workflows/_publish-data-platform-data-diff.yml
+    secrets: inherit
+    with:
+      python_version: ${{ needs.setup.outputs.python_version }}
+      poetry_version: ${{ needs.setup.outputs.poetry_version }}
+      python_cache_key: ${{ needs.cache-dependencies.outputs.python_cache_key }}