diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 9a1d142..9e74249 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -25,7 +25,8 @@ jobs: # - Windows # - MacOs py: - - "3.12.0" + - "3.13" + - "3.12" - "3.11" - "3.10" - "3.9" @@ -43,7 +44,7 @@ jobs: version: '2.14.2' # the full version of `docker-compose` command - name: Docker compose up - run: make docker-compose-up + run: make docker-compose-up-deamon - name: Setup run: make setup diff --git a/Makefile b/Makefile index 8bdffdc..b1b4ee2 100644 --- a/Makefile +++ b/Makefile @@ -9,8 +9,9 @@ setup-dev: pyenv install -s 3.9.9 pyenv install -s 3.10.7 pyenv install -s 3.11.0 - pyenv install -s 3.12-dev - pyenv local 3.7.12 3.8.6 3.9.9 3.10.7 3.11.0 3.12-dev + pyenv install -s 3.12.0 + pyenv install -s 3.13.0 + pyenv local 3.7.12 3.8.6 3.9.9 3.10.7 3.11.0 3.12.0 3.13.0 python3 -m pip install tox==4.8.0 .PHONY: test @@ -38,6 +39,10 @@ open-coverage: docker-compose-up: cd dev && make configure up +.PHONY: docker-compose-up-deamon +docker-compose-up-deamon: + cd dev && make configure up-deamon + .PHONY: docker-compose-down docker-compose-down: cd dev && make down clean diff --git a/dev/CH-TEMPLATE/config.d/database_atomic_delay_before_drop_table.xml b/dev/CH-TEMPLATE/config.d/database_atomic_delay_before_drop_table.xml new file mode 100644 index 0000000..58fba97 --- /dev/null +++ b/dev/CH-TEMPLATE/config.d/database_atomic_delay_before_drop_table.xml @@ -0,0 +1,3 @@ + + 1 + \ No newline at end of file diff --git a/dev/config.xml b/dev/CH-TEMPLATE/config.xml similarity index 93% rename from dev/config.xml rename to dev/CH-TEMPLATE/config.xml index ae49b5c..27b98a2 100644 --- a/dev/config.xml +++ b/dev/CH-TEMPLATE/config.xml @@ -28,12 +28,17 @@ /var/lib/clickhouse/tmp/ /var/lib/clickhouse/user_files/ - users.xml default default Europe/Moscow false + + + /etc/clickhouse-server/default-users.xml + + + diff --git a/dev/users.xml b/dev/CH-TEMPLATE/default-users.xml similarity index 100% rename from dev/users.xml rename to dev/CH-TEMPLATE/default-users.xml diff --git a/dev/CH-TEMPLATE/users.d/default-user.xml b/dev/CH-TEMPLATE/users.d/default-user.xml new file mode 100644 index 0000000..1da237e --- /dev/null +++ b/dev/CH-TEMPLATE/users.d/default-user.xml @@ -0,0 +1,12 @@ + + + + + + + ::1 + 127.0.0.1 + + + + diff --git a/dev/Makefile b/dev/Makefile index 2ee67b3..5e59fa9 100644 --- a/dev/Makefile +++ b/dev/Makefile @@ -1,23 +1,26 @@ configure: make clean - mkdir -p clickhouse01 clickhouse02 clickhouse03 clickhouse04 - REPLICA=01 SHARD=01 envsubst < config.xml > clickhouse01/config.xml - REPLICA=02 SHARD=01 envsubst < config.xml > clickhouse02/config.xml - REPLICA=03 SHARD=02 envsubst < config.xml > clickhouse03/config.xml - REPLICA=04 SHARD=02 envsubst < config.xml > clickhouse04/config.xml - cp users.xml clickhouse01/users.xml - cp users.xml clickhouse02/users.xml - cp users.xml clickhouse03/users.xml - cp users.xml clickhouse04/users.xml + cp -r CH-TEMPLATE clickhouse01 + cp -r CH-TEMPLATE clickhouse02 + cp -r CH-TEMPLATE clickhouse03 + cp -r CH-TEMPLATE clickhouse04 + REPLICA=01 SHARD=01 envsubst < CH-TEMPLATE/config.xml > clickhouse01/config.xml + REPLICA=02 SHARD=01 envsubst < CH-TEMPLATE/config.xml > clickhouse02/config.xml + REPLICA=03 SHARD=02 envsubst < CH-TEMPLATE/config.xml > clickhouse03/config.xml + REPLICA=04 SHARD=02 envsubst < CH-TEMPLATE/config.xml > clickhouse04/config.xml up: + docker-compose up + +up-deamon: docker-compose up -d start: docker-compose start down: - docker-compose down + docker-compose down -v + docker volume prune -f clean: rm -rf clickhouse01 clickhouse02 clickhouse03 clickhouse04 \ No newline at end of file diff --git a/dev/docker-compose.yaml b/dev/docker-compose.yaml index 222e593..5966316 100644 --- a/dev/docker-compose.yaml +++ b/dev/docker-compose.yaml @@ -1,5 +1,3 @@ -version: '3.5' - networks: clickhouse-network: name: clickhouse-network @@ -17,8 +15,7 @@ services: ipv4_address: 172.23.0.10 clickhouse01: - image: yandex/clickhouse-server:20.4 - # image: altinity/clickhouse-server:23.8.11.29.altinitystable + image: clickhouse/clickhouse-server:25.7.4 container_name: clickhouse01 hostname: clickhouse01 networks: @@ -33,8 +30,7 @@ services: - zookeeper clickhouse02: - image: yandex/clickhouse-server:20.4 - # image: altinity/clickhouse-server:23.8.11.29.altinitystable + image: clickhouse/clickhouse-server:25.7.4 container_name: clickhouse02 hostname: clickhouse02 networks: @@ -46,8 +42,7 @@ services: - zookeeper clickhouse03: - image: yandex/clickhouse-server:20.4 - # image: altinity/clickhouse-server:23.8.11.29.altinitystable + image: clickhouse/clickhouse-server:25.7.4 container_name: clickhouse03 hostname: clickhouse03 networks: @@ -59,8 +54,7 @@ services: - zookeeper clickhouse04: - image: yandex/clickhouse-server:20.4 - # image: altinity/clickhouse-server:23.8.11.29.altinitystable + image: clickhouse/clickhouse-server:25.7.4 container_name: clickhouse04 hostname: clickhouse04 networks: diff --git a/pyproject.toml b/pyproject.toml index b754b59..2f4c69b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -23,14 +23,14 @@ classifiers = [ "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", + "Programming Language :: Python :: 3.12", ] dependencies = [ "clickhouse-driver>=0.2.2", ] optional-dependencies.testing = [ - "pytest==7.4.1", - "pytest-cov==4.1.0", - "parameterized==0.9.0", + "pytest==8.4.1", + "pytest-cov==6.2.1", ] dynamic =["version", "readme"] diff --git a/src/clickhouse_migrations/__init__.py b/src/clickhouse_migrations/__init__.py index 82bb8d2..3f38fcd 100644 --- a/src/clickhouse_migrations/__init__.py +++ b/src/clickhouse_migrations/__init__.py @@ -1,4 +1,5 @@ """ Simple file-based migrations for clickhouse """ -__version__ = "0.8.1" + +__version__ = "0.9.0" diff --git a/src/tests/conftest.py b/src/tests/conftest.py index 43313ec..4348f15 100644 --- a/src/tests/conftest.py +++ b/src/tests/conftest.py @@ -1,15 +1,38 @@ import pytest from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster +from clickhouse_migrations.migrator import Migrator @pytest.fixture def cluster() -> ClickhouseCluster: - return ClickhouseCluster(db_host="localhost", db_user="default", db_password="") + return ClickhouseCluster( + db_host="localhost", + db_user="default", + db_password="", + db_name="pytest", + ) @pytest.fixture(autouse=True) def before(cluster: ClickhouseCluster): with cluster.connection("") as conn: - conn.execute("DROP DATABASE IF EXISTS pytest") + conn.execute("DROP DATABASE IF EXISTS pytest SYNC") conn.execute("CREATE DATABASE pytest") + + +@pytest.fixture(name="_clean_slate") +def clean_slate(cluster): + with cluster.connection("") as conn: + conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster SYNC") + + +@pytest.fixture(name="_schema") +def schema(cluster, _clean_slate): + conn = cluster.connection("") + conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster") + conn = cluster.connection("pytest") + migrator = Migrator(conn) + migrator.init_schema("company_cluster") + + return conn diff --git a/src/tests/test_cluster_support.py b/src/tests/test_cluster_support.py index 3829672..cf3d1ba 100644 --- a/src/tests/test_cluster_support.py +++ b/src/tests/test_cluster_support.py @@ -1,9 +1,6 @@ from pathlib import Path -import pytest - from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster -from clickhouse_migrations.migrator import Migrator TESTS_DIR = Path(__file__).parent @@ -15,28 +12,6 @@ ) -@pytest.fixture -def cluster(): - return ClickhouseCluster(db_host="localhost", db_user="default", db_password="") - - -@pytest.fixture(name="_clean_slate") -def clean_slate(cluster): - with cluster.connection("") as conn: - conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster") - - -@pytest.fixture(name="_schema") -def schema(cluster, _clean_slate): - conn = cluster.connection("") - conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster") - conn = cluster.connection("pytest") - migrator = Migrator(conn) - migrator.init_schema("company_cluster") - - return conn - - def test_replicated_schema(_schema): with _schema: for server in CLICKHOUSE_SERVERS: @@ -45,7 +20,7 @@ def test_replicated_schema(_schema): )[0][0] assert ( table_engine - == "ReplicatedMergeTree('/clickhouse/tables/{database}/{table}', '{replica}') ORDER BY tuple(created_at) SETTINGS index_granularity = 8192" # pylint: disable=C0301 # noqa: E501 + == "ReplicatedMergeTree('/clickhouse/tables/pytest/schema_versions', '{replica}') ORDER BY tuple(created_at) SETTINGS index_granularity = 8192" # pylint: disable=C0301 # noqa: E501 ) diff --git a/src/tests/test_dryrun.py b/src/tests/test_dryrun.py index fa47efd..bf1c15e 100644 --- a/src/tests/test_dryrun.py +++ b/src/tests/test_dryrun.py @@ -1,36 +1,12 @@ import logging from pathlib import Path -import pytest - -from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster from clickhouse_migrations.migration import MigrationStorage TESTS_DIR = Path(__file__).parent MIGRATIONS = MigrationStorage(TESTS_DIR / "migrations").migrations() -@pytest.fixture -def cluster(): - return ClickhouseCluster(db_host="localhost", db_user="default", db_password="") - - -@pytest.fixture(name="_clean_slate") -def clean_slate(cluster): - with cluster.connection("") as conn: - conn.execute("DROP DATABASE IF EXISTS pytest ON CLUSTER company_cluster") - - -@pytest.fixture(name="_schema") -def schema(cluster, _clean_slate): - conn = cluster.connection("") - conn.execute("CREATE DATABASE pytest ON CLUSTER company_cluster") - conn = cluster.connection("pytest") - cluster.init_schema("pytest") - - return conn - - def test_dryrun(_schema, cluster, caplog): with caplog.at_level(logging.INFO): cluster.migrate("pytest", TESTS_DIR / "migrations", dryrun=True) diff --git a/src/tests/test_init_clickhouse_cluster.py b/src/tests/test_init_clickhouse_cluster.py index 7bfa1d9..a9b63b6 100644 --- a/src/tests/test_init_clickhouse_cluster.py +++ b/src/tests/test_init_clickhouse_cluster.py @@ -1,19 +1,11 @@ from pathlib import Path -import pytest - -from clickhouse_migrations.clickhouse_cluster import ClickhouseCluster from clickhouse_migrations.migration import Migration, MigrationStorage TESTS_DIR = Path(__file__).parent MIGRATIONS = MigrationStorage(TESTS_DIR / "migrations").migrations() -@pytest.fixture -def cluster(): - return ClickhouseCluster(db_url="clickhouse://default:@localhost:9000/pytest") - - def test_apply_new_migration_ok(cluster): cluster.init_schema() diff --git a/tox.ini b/tox.ini index c828cc9..38e3884 100644 --- a/tox.ini +++ b/tox.ini @@ -6,11 +6,11 @@ envlist = black-check flake8-check pylint-check - py{39,310,311,312} + py{39,310,311,312,313} coverage-report clean skip_missing_interpreters = True -basepython=py311 +basepython=py313 [testenv] deps = .[testing] @@ -20,39 +20,39 @@ commands = py.test src \ --cov-config="{toxinidir}/tox.ini" [testenv:flake8-check] -deps = flake8==6.1.0 +deps = flake8==7.3.0 commands = flake8 --config={toxinidir}/tox.ini src/ [testenv:pylint-check] deps = .[testing] - pylint==3.0.3 + pylint==3.3.8 commands = pylint src/tests/ --rcfile={toxinidir}/tox.ini pylint src/clickhouse_migrations/ --rcfile={toxinidir}/tox.ini [testenv:isort] changedir = {toxinidir}/src -deps = isort==5.10.1 +deps = isort==6.0.1 commands = isort . -sp={toxinidir}/tox.ini [testenv:isort-check] changedir = {toxinidir}/src -deps = isort==5.10.1 +deps = isort==6.0.1 commands = isort . --check-only -sp={toxinidir}/tox.ini [testenv:black] changedir = {toxinidir}/src -deps = black==22.6.0 +deps = black==25.1.0 commands = black . [testenv:black-check] changedir = {toxinidir}/src -deps = black==22.6.0 +deps = black==25.1.0 commands = black --check . [testenv:coverage-report] deps = - coverage==6.4.3 + coverage==7.10.3 commands = coverage html coverage report --include="src/clickhouse_migrations/*" --fail-under=99 -m @@ -107,7 +107,8 @@ commands = coverage erase max-line-length=120 # C0114: Missing module docstring # C0116: Missing function or method docstring -disable=fixme, invalid-name, R0801, W0621, C0116, C0114, R0913, C0115 +# R0917: Too many positional arguments +disable=fixme, invalid-name, R0801, R0917, W0621, C0116, C0114, R0913, C0115 [flake8]