-
Notifications
You must be signed in to change notification settings - Fork 197
117 lines (96 loc) · 4.35 KB
/
test_destination_clickhouse.yml
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
name: test | clickhouse
on:
pull_request:
branches:
- master
- devel
workflow_dispatch:
schedule:
- cron: '0 2 * * *'
concurrency:
group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
cancel-in-progress: true
env:
# RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752
RUNTIME__LOG_LEVEL: ERROR
DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
ACTIVE_DESTINATIONS: "[\"clickhouse\"]"
ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]"
jobs:
get_docs_changes:
name: docs changes
uses: ./.github/workflows/get_docs_changes.yml
if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
run_loader:
name: test | clickhouse tests
needs: get_docs_changes
if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
defaults:
run:
shell: bash
runs-on: "ubuntu-latest"
steps:
- name: Check out
uses: actions/checkout@master
- name: Setup Python
uses: actions/setup-python@v4
with:
python-version: "3.10.x"
- name: Install Poetry
uses: snok/install-poetry@v1.3.2
with:
virtualenvs-create: true
virtualenvs-in-project: true
installer-parallel: true
version: 1.8.5
- name: Load cached venv
id: cached-poetry-dependencies
uses: actions/cache@v3
with:
path: .venv
key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp
- name: Install dependencies
run: poetry install --no-interaction -E clickhouse --with providers -E parquet --with sentry-sdk --with pipeline,ibis
- name: create secrets.toml
run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
# OSS ClickHouse
- run: |
docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" up -d
echo "Waiting for ClickHouse to be healthy..."
timeout 30s bash -c 'until docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" ps | grep -q "healthy"; do sleep 1; done'
echo "ClickHouse is up and running"
name: Start ClickHouse OSS
- run: poetry run pytest tests/load --ignore tests/load/sources -m "essential"
name: Run essential tests Linux (ClickHouse OSS)
if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}}
env:
DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
- run: poetry run pytest tests/load --ignore tests/load/sources
name: Run all tests Linux (ClickHouse OSS)
if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
env:
DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost
DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data
DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader
DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader
DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000
DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123
DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0
- name: Stop ClickHouse OSS
if: always()
run: docker compose -f "tests/load/clickhouse/clickhouse-compose.yml" down -v
# ClickHouse Cloud
- run: |
poetry run pytest tests/load --ignore tests/load/sources -m "essential"
name: Run essential tests Linux (ClickHouse Cloud)
if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}}
- run: |
poetry run pytest tests/load --ignore tests/load/sources
name: Run all tests Linux (ClickHouse Cloud)
if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}