ci.yaml

name: recap

on:
  push:
    branches: [ main ]

  pull_request:
    branches: [ main ]

jobs:
  unit-tests:
    runs-on: ubuntu-latest

    strategy:
      matrix:
        python-version: ['3.10']

    steps:
    - uses: actions/checkout@v2

    - name: Build using Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        pip3 install pdm
        pdm self update
        pdm install

    - name: Run black
      run: |
        pdm run black --check --diff recap/ tests/

    - name: Run isort
      run: |
        pdm run isort recap/ tests/ --check-only --diff

    - name: Run autoflake
      run: |
        pdm run autoflake --check-diff --remove-unused-variables --remove-all-unused-imports --recursive recap/ tests/

    - name: Run pylint
      run: |
        pdm run pylint --fail-under=7.0 recap/ tests/

    - name: Run pyright
      run: |
        pdm run pyright

    - name: Test with pytest
      run: |
        pdm run unit

  integration-tests:
    runs-on: ubuntu-latest
    needs: unit-tests

    services:
      postgres:
        image: postgres:14
        env:
          POSTGRES_USER: postgres
          POSTGRES_PASSWORD: password
          POSTGRES_DB: testdb
        ports:
          - 5432:5432
        options: >-
          --health-cmd pg_isready
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      mysql:
        image: mysql:8.0
        env:
          MYSQL_ROOT_PASSWORD: password
          MYSQL_DATABASE: testdb
          MYSQL_USER: mysql
          MYSQL_PASSWORD: password
        ports:
          - 3306:3306
        options: >-
          --health-cmd "mysqladmin ping --silent"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      # TODO Waiting on https://github.com/goccy/bigquery-emulator/pull/208
      bigquery:
        image: ghcr.io/criccomini/bigquery-emulator:0.4.3-envvar
        env:
          BIGQUERY_EMULATOR_PROJECT: test_project
          BIGQUERY_EMULATOR_DATASET: test_dataset
        ports:
          - 9050:9050

      zookeeper:
        image: confluentinc/cp-zookeeper:latest
        ports:
          - 2181:2181
        env:
          ZOOKEEPER_CLIENT_PORT: 2181
        options: >-
          --health-cmd "echo ruok | nc localhost 2181"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      kafka:
        image: confluentinc/cp-kafka:latest
        ports:
          - 9092:9092
        env:
          KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
          KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092
          KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1
        options: >-
          --health-cmd "kafka-topics --bootstrap-server localhost:9092 --list"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      schema-registry:
        image: confluentinc/cp-schema-registry:latest
        ports:
          - 8081:8081
        env:
          SCHEMA_REGISTRY_HOST_NAME: schema-registry
          SCHEMA_REGISTRY_LISTENERS: http://schema-registry:8081
          SCHEMA_REGISTRY_KAFKASTORE_CONNECTION_URL: zookeeper:2181
          SCHEMA_REGISTRY_KAFKASTORE_BOOTSTRAP_SERVERS: PLAINTEXT://kafka:9092
        options: >-
          --health-cmd "curl -f http://schema-registry:8081/"
          --health-interval 10s
          --health-timeout 5s
          --health-retries 5

      hive-metastore:
        image: ghcr.io/criccomini/hive-metastore-standalone:latest
        ports:
          - 9083:9083

    strategy:
      matrix:
        python-version: ['3.10']

    steps:
    - uses: actions/checkout@v2

    - name: Build using Python ${{ matrix.python-version }}
      uses: actions/setup-python@v4
      with:
        python-version: ${{ matrix.python-version }}

    - name: Install dependencies
      run: |
        pip3 install pdm
        pdm self update
        pdm install

    - name: Test with pytest
      run: |
        pdm run integration

  spec-tests:
    runs-on: ubuntu-latest
    needs: integration-tests
    strategy:
      matrix:
        python-version: ['3.10']
    steps:
      - uses: actions/checkout@v2

      - name: Build using Python ${{ matrix.python-version }}
        uses: actions/setup-python@v4
        with:
          python-version: ${{ matrix.python-version }}

      - name: Install dependencies
        run: |
          pip3 install pdm
          pdm self update
          pdm install
      - name: Test spec with pytest
        run: |
          pdm run spec