Merge branch 'main' into cloud-tests-against-smt

.github/workflows/lint.yml

@@ -16,12 +16,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
-      - name: Setup Python 3.9
-        uses: actions/setup-python@v4
+      - name: Setup Python 3.11
+        uses: actions/setup-python@v5
         with:
-          python-version: 3.9
+          python-version: 3.11
 
       - name: Upgrade Setuptools
         run: pip install --upgrade setuptools wheel

.github/workflows/pypi.yml

@@ -16,12 +16,12 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
-          python-version: "3.10"
+          python-version: "3.11"
 
       - name: Upgrade Setuptools
         run: pip install --upgrade setuptools wheel

.github/workflows/test_cloud.yml

@@ -21,10 +21,10 @@ jobs:
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Setup Python 3.11
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: '3.11'
 

.github/workflows/test_matrix.yml

@@ -26,16 +26,17 @@ jobs:
           - '3.9'
           - '3.10'
           - '3.11'
+          - '3.12'
         clickhouse-version:
-          - '23.3'
           - '23.8'
-          - '23.9'
-          - '23.10'
+          - '24.1'
+          - '24.2'
+          - '24.3'
           - latest
 
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
 
       - name: Set environment variables
         if: ${{ matrix.clickhouse-version == '22.3' }}
@@ -49,7 +50,7 @@ jobs:
         run: REPLICA_NUM=1 docker-compose -f ${{ github.workspace }}/tests/integration/docker-compose.yml up -d
 
       - name: Setup Python ${{ matrix.python-version }}
-        uses: actions/setup-python@v4
+        uses: actions/setup-python@v5
         with:
           python-version: ${{ matrix.python-version }}
 

CHANGELOG.md

@@ -1,3 +1,32 @@
+### Release [1.7.6], 2024-04-12
+#### Bug Fix
+- A bug in (experimental) Distributed Table model creation could lead to errors when there was a change in the model definition (see, e.g.,
+https://github.com/ClickHouse/dbt-clickhouse/issues/226).  Thanks to [Thomas Schmidt](https://github.com/Somtom) for the Fix!
+- A comment at the end of a model would break the query used to retrieve the result column datatypes.  Thanks to [triou](https://github.com/tevariou)
+for the bug report and the fix.  Closes https://github.com/ClickHouse/dbt-clickhouse/issues/256
+
+#### Improvements
+- The new materialization for ClickHouse dictionaries now takes an optional "credentials dictionary" argument that overrides the
+global credentials values for user, password, database, host, and port (including removing any of those values by adding empty values if not needed).
+This allows better control over creating dictionaries on different server.  Thanks to [Cristhian Garcia](https://github.com/Ian2012)
+for the PR!
+- A new `ttl` setting has been added to model configuration that will insert the provided ClickHouse TTL expression in the appropriate place.
+Thanks to [Evan Rusackas](https://github.com/rusackas) for the contribution!
+- The Agate library should now be lazy loaded.  This should modestly improve dbt startup times (after dbt-clickhouse is upgraded to dbt 1.8.x).
+Thanks to [Daniel Reeves](https://github.com/dwreeves) for PR.
+
+### Release [1.7.5], 2024-04-02
+#### Bug Fixes
+- Requirements and tests upgraded to include Python 3.12.  Closes https://github.com/ClickHouse/dbt-clickhouse/issues/264
+- Model settings were not working correctly for custom materializations.  Thanks to original dbt-clickhouse [silentsokolov](https://github.com/silentsokolov)
+for the PR!
+
+### Release [1.7.4], 2024-03-23
+#### Improvement
+- Adds support for materializing ClickHouse dictionaries.  Thanks to [Rory Sawyer](https://github.com/SoryRawyer) for the contribution!
+See his excellent [tests](https://github.com/ClickHouse/dbt-clickhouse/blob/main/tests/integration/adapter/dictionary/test_dictionary.py) 
+for example usage.
+
 ### Release [1.7.3], 2024-03-11
 #### Bug Fixes
 - Fixed an [issue](https://github.com/ClickHouse/dbt-clickhouse/issues/231) where passing settings to on view creation didn't work.

Makefile

@@ -1,5 +1,5 @@
 # Check style and linting
-.PHONY: check-black check-isort check-flake8 check-mypy lint
+.PHONY: check-black check-isort check-mypy lint
 
 check-black:
 	@echo "--> Running black checks"
@@ -9,10 +9,6 @@ check-isort:
 	@echo "--> Running isort checks"
 	@isort --check-only .
 
-check-flake8:
-	@echo "--> Running flake8 checks"
-	@flake8 .
-
 check-mypy:
 	@echo "--> Running mypy checks"
 	@mypy --exclude dbt/adapters/clickhouse/__init__.py --exclude conftest.py .
@@ -21,7 +17,7 @@ check-yamllint:
 	@echo "--> Running yamllint checks"
 	@yamllint dbt tests .github 
 
-lint: check-black check-isort check-flake8 check-mypy check-yamllint
+lint: check-black check-isort check-mypy check-yamllint
 
 # Format code
 .PHONY: fmt

README.md

@@ -102,6 +102,7 @@ your_profile_name:
 | incremental_predicates | Additional conditions to be applied to the incremental materialization (only applied to `delete+insert` strategy                                                                                                                                       |                |
 | settings               | A map/dictionary of "TABLE" settings to be used to DDL statements like 'CREATE TABLE' with this model                                                                                                                                                  |                |
 | query_settings         | A map/dictionary of ClickHouse user level settings to be used with `INSERT` or `DELETE` statements in conjunction with this model                                                                                                                      |                |
+| ttl                    | A TTL expression to be used with the table.  The TTL expression is a string that can be used to specify the TTL for the table.                                                                                                                         |                |
 
 ## ClickHouse Cluster 
 
@@ -179,6 +180,7 @@ The following macros are included to facilitate creating ClickHouse specific tab
 - `order_cols` -- Uses the `order_by` model configuration to assign a ClickHouse order by/sorting key.  If not specified ClickHouse will use an empty tuple() and the table will be unsorted
 - `primary_key_clause` -- Uses the `primary_key` model configuration property to assign a ClickHouse primary key.  By default, primary key is set and ClickHouse will use the order by clause as the primary key. 
 - `on_cluster_clause` -- Uses the `cluster` profile property to add an `ON CLUSTER` clause to certain dbt-operations: distributed materializations, views creation, database creation.
+- `ttl_config` -- Uses the `ttl` model configuration property to assign a ClickHouse table TTL expression.  No TTL is assigned by default.
 
 ### s3Source Helper Macro
 
@@ -213,6 +215,10 @@ no corresponding REFRESH operation).  Instead, it acts as an "insert trigger", a
 (https://github.com/ClickHouse/dbt-clickhouse/blob/main/tests/integration/adapter/materialized_view/test_materialized_view.py)  for an introductory example
 of how to use this functionality.
 
+# Dictionary materializations (experimental)
+See the tests in https://github.com/ClickHouse/dbt-clickhouse/blob/main/tests/integration/adapter/dictionary/test_dictionary.py for examples of how to
+implement materializations for ClickHouse dictionaries 
+
 # Distributed materializations
 
 Notes:

dbt/adapters/clickhouse/__version__.py

@@ -1 +1 @@
-version = '1.7.3'
+version = '1.7.6'

dbt/adapters/clickhouse/connections.py

@@ -1,16 +1,18 @@
 import re
 import time
 from contextlib import contextmanager
-from typing import Any, Optional, Tuple, Union
+from typing import TYPE_CHECKING, Any, Optional, Tuple, Union
 
-import agate
 import dbt.exceptions
 from dbt.adapters.sql import SQLConnectionManager
 from dbt.contracts.connection import AdapterResponse, Connection
 
 from dbt.adapters.clickhouse.dbclient import ChRetryableException, get_db_client
 from dbt.adapters.clickhouse.logger import logger
 
+if TYPE_CHECKING:
+    import agate
+
 retryable_exceptions = [ChRetryableException]
 ddl_re = re.compile(r'^\s*(CREATE|DROP|ALTER)\s', re.IGNORECASE)
 
@@ -60,21 +62,23 @@ def release(self):
         pass  # There is no "release" type functionality in the existing ClickHouse connectors
 
     @classmethod
-    def get_table_from_response(cls, response, column_names) -> agate.Table:
+    def get_table_from_response(cls, response, column_names) -> "agate.Table":
         """
         Build agate table from response.
         :param response: ClickHouse query result
         :param column_names: Table column names
         """
+        from dbt.clients.agate_helper import table_from_data_flat
+
         data = []
         for row in response:
             data.append(dict(zip(column_names, row)))
 
-        return dbt.clients.agate_helper.table_from_data_flat(data, column_names)
+        return table_from_data_flat(data, column_names)
 
     def execute(
         self, sql: str, auto_begin: bool = False, fetch: bool = False, limit: Optional[int] = None
-    ) -> Tuple[AdapterResponse, agate.Table]:
+    ) -> Tuple[AdapterResponse, "agate.Table"]:
         # Don't try to fetch result of clustered DDL responses, we don't know what to do with them
         if fetch and ddl_re.match(sql):
             fetch = False
@@ -97,7 +101,9 @@ def execute(
                     query_result.result_set, query_result.column_names
                 )
             else:
-                table = dbt.clients.agate_helper.empty_table()
+                from dbt.clients.agate_helper import empty_table
+
+                table = empty_table()
             return AdapterResponse(_message=status), table
 
     def add_query(

dbt/adapters/clickhouse/dbclient.py

@@ -53,20 +53,20 @@ def get_db_client(credentials: ClickHouseCredentials):
             from dbt.adapters.clickhouse.nativeclient import ChNativeClient
 
             return ChNativeClient(credentials)
-        except ImportError:
+        except ImportError as ex:
             raise FailedToConnectError(
                 'Native adapter required but package clickhouse-driver is not installed'
-            )
+            ) from ex
     try:
         import clickhouse_connect  # noqa
 
         from dbt.adapters.clickhouse.httpclient import ChHttpClient
 
         return ChHttpClient(credentials)
-    except ImportError:
+    except ImportError as ex:
         raise FailedToConnectError(
             'HTTP adapter required but package clickhouse-connect is not installed'
-        )
+        ) from ex
 
 
 class ChRetryableException(Exception):
@@ -150,7 +150,8 @@ def _server_version(self):
         pass
 
     def update_model_settings(self, model_settings: Dict[str, str], materialization_type: str):
-        model_settings_to_add = copy.deepcopy(self._model_settings[materialization_type])
+        settings = self._model_settings.get(materialization_type, {})
+        model_settings_to_add = copy.deepcopy(settings)
         model_settings_to_add.update(self._model_settings['general'])
         for key, value in model_settings_to_add.items():
             if key not in model_settings:

dbt/adapters/clickhouse/httpclient.py

@@ -25,7 +25,10 @@ def command(self, sql, **kwargs):
 
     def columns_in_query(self, sql: str, **kwargs) -> List[ClickHouseColumn]:
         try:
-            query_result = self._client.query(f"SELECT * FROM ({sql}) LIMIT 0", **kwargs)
+            query_result = self._client.query(
+                f"SELECT * FROM ( \n" f"{sql} \n" f") LIMIT 0",
+                **kwargs,
+            )
             return [
                 ClickHouseColumn.create(name, ch_type.name)
                 for name, ch_type in zip(query_result.column_names, query_result.column_types)