179 upgrade to support dbt core v150

CHANGELOG.md

@@ -1,14 +1,25 @@
-# dbt-dremio 1.X.X - release TBD
+# dbt-dremio 1.5.0 - release TBD
 
 ## Features
 
 ## Fixes
 
 ## Under the Hood
 
+-   [#179](https://github.com/dremio/dbt-dremio/issues/179) Upgrade to support dbt-core v1.5.0.
+    -   Add support for Python 3.11.
+    -   Add support for relevant Tests:
+        -   caching
+        -   hooks
+        -   simple_copy
+-   Add support for model contracts (Stub the feature to let users know the feature is not supported).
+
 ## Dependency
 
 -   Upgrade sqlparse to 0.4.4 [#180](https://github.com/dremio/dbt-dremio/issues/180).
+-   Upgrade dbt-core to 1.5.0.
+-   Upgrade dbt-tests-adapter to 1.5.0.
+-   Upgrade Requests to 2.31.0. [#183](https://github.com/dremio/dbt-dremio/issues/183).
 
 # dbt-dremio 1.4.5 - release March 23, 2023
 

README.md

@@ -6,20 +6,20 @@
 
 dbt is the T in ELT. Organize, cleanse, denormalize, filter, rename, and pre-aggregate the raw data in your warehouse so that it's ready for analysis.
 
-## dbt-dremio version 1.4.5
+## dbt-dremio version 1.5.0
 
 The `dbt-dremio` package contains all of the code enabling dbt to work with [Dremio](https://www.dremio.com/). For more information on using dbt with Dremio, consult [the docs](https://docs.getdbt.com/reference/warehouse-profiles/dremio-profile).
 
 The dbt-dremio package supports both Dremio Cloud and Dremio Software (versions 22.0 and later).
 
-Version 1.4.5 of the dbt-dremio adapter is compatible with dbt-core versions 1.2.0 to 1.4.5.
+Version 1.5.0 of the dbt-dremio adapter is compatible with dbt-core versions 1.2.0 to 1.5.0.
 
 > Prior to version 1.1.0b, dbt-dremio was created and maintained by [Fabrice Etanchaud](https://github.com/fabrice-etanchaud) on [their GitHub repo](https://github.com/fabrice-etanchaud/dbt-dremio). Code for using Dremio REST APIs was originally authored by [Ryan Murray](https://github.com/rymurr). Contributors in this repo are credited for laying the groundwork and maintaining the adapter till version 1.0.6.5. The dbt-dremio adapter is maintained and distributed by Dremio starting with version 1.1.0b.
 
 ## Getting started
 
 -   [Install dbt-dremio](https://docs.getdbt.com/reference/warehouse-setups/dremio-setup)
-    -   Version 1.4.5 of dbt-dremio requires dbt-core >= 1.2.0 and <= 1.4.5. Installing dbt-dremio will automatically upgrade existing dbt-core versions earlier than 1.2.0 to 1.4.5, or install dbt-core v1.4.5 if no version of dbt-core is found.
+    -   Version 1.5.0 of dbt-dremio requires dbt-core >= 1.2.0 and <= 1.5.0. Installing dbt-dremio will automatically upgrade existing dbt-core versions earlier than 1.2.0 to 1.5.0, or install dbt-core v1.5.0 if no version of dbt-core is found.
 -   Read the [introduction](https://docs.getdbt.com/docs/introduction/) and [viewpoint](https://docs.getdbt.com/docs/about/viewpoint/)
 
 ## Join the dbt Community

THIRD_PARTY_LICENSES.txt

@@ -13,10 +13,10 @@
  charset-normalizer        3.1.0      MIT License
  click                     8.1.3      BSD License
  colorama                  0.4.6      BSD License
- dbt-core                  1.4.5      Apache Software License
- dbt-dremio                1.4.5      Apache Software License
+ dbt-core                  1.5.0      Apache Software License
+ dbt-dremio                1.5.0      Apache Software License
  dbt-extractor             0.4.1      Apache Software License
- dbt-tests-adapter         1.4.5      Apache Software License
+ dbt-tests-adapter         1.5.0      Apache Software License
  exceptiongroup            1.1.1      MIT License
  future                    0.18.3     MIT License
  grpclib                   0.4.3      BSD License
@@ -45,7 +45,7 @@
  python-slugify            8.0.1      MIT License
  pytimeparse               1.1.8      MIT License
  pytz                      2022.7.1   MIT License
- requests                  2.28.2     Apache Software License
+ requests                  2.31.0     Apache Software License
  six                       1.16.0     MIT License
  sqlparse                  0.4.4      BSD License
  stringcase                1.2.0      MIT

dbt/adapters/dremio/__version__.py

@@ -9,4 +9,4 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-version = "1.4.5"
+version = "1.5.0"

dbt/adapters/dremio/credentials.py

@@ -53,6 +53,14 @@ class DremioCredentials(Credentials):
     def type(self):
         return "dremio"
 
+    @property
+    def unique_field(self):
+        """
+        Hashed and included in anonymous telemetry to track adapter adoption.
+        Pick a field that can uniquely identify one team/organization building with this adapter
+        """
+        return self.software_host if self.cloud_host is None else self.cloud_host
+
     @property
     def aliases(self):
         return self._ALIASES

dbt/include/dremio/macros/materializations/table/create_table_as.sql

@@ -26,6 +26,11 @@ limitations under the License.*/
 #}
 
 {% macro dremio__create_table_as(temporary, relation, sql) -%}
+  {% set contract_config = config.get('contract') %}
+  {% if contract_config.enforced %}
+     {{exceptions.warn("Model contracts are not enforced by dbt-dremio!")}}
+  {% endif %}
+
   {%- set sql_header = config.get('sql_header', none) -%}
 
   {{ sql_header if sql_header is not none }}

dbt/include/dremio/macros/materializations/view/create_view_as.sql

@@ -13,6 +13,11 @@ See the License for the specific language governing permissions and
 limitations under the License.*/
 
 {% macro dremio__create_view_as(relation, sql) -%}
+  {% set contract_config = config.get('contract') %}
+  {% if contract_config.enforced %}
+     {{exceptions.warn("Model contracts are not enforced by dbt-dremio!")}}
+  {% endif %}
+
   {%- set sql_header = config.get('sql_header', none) -%}
 
   {{ sql_header if sql_header is not none }}

dev_requirements.txt

@@ -1,39 +1,22 @@
-agate==1.7.0
 attrs==22.2.0
 Babel==2.12.1
 betterproto==1.2.5
 certifi==2022.12.7
-cffi==1.15.1
 charset-normalizer==3.1.0
-click==8.1.3
-colorama==0.4.6
-dbt-core==1.4.5
-dbt-dremio==1.4.5
-dbt-extractor==0.4.1
-dbt-tests-adapter==1.4.5
+dbt-tests-adapter==1.5.0
 exceptiongroup==1.1.1
 future==0.18.3
 grpclib==0.4.3
 h2==4.1.0
-hologram==0.0.15
 hpack==4.0.0
 hyperframe==6.0.1
-idna==3.4
 iniconfig==2.0.0
-isodate==0.6.1
-Jinja2==3.1.2
 jsonschema==4.17.3
 leather==0.3.4
-Logbook==1.5.3
 MarkupSafe==2.1.2
-mashumaro==3.5
-minimal-snowplow-tracker==0.0.2
 msgpack==1.0.5
 multidict==6.0.4
-networkx==3.0
-packaging==23.0
-parsedatetime==2.6
-pathspec==0.11.1
+parsedatetime==2.4
 pip-licenses==4.1.0
 pluggy==1.0.0
 prettytable==3.6.0
@@ -43,15 +26,9 @@ pytest==7.2.2
 python-dateutil==2.8.2
 python-slugify==8.0.1
 pytimeparse==1.1.8
-pytz==2022.7.1
-PyYAML==6.0
-requests==2.28.2
 six==1.16.0
-sqlparse==0.4.4
 stringcase==1.2.0
 text-unidecode==1.3
 tomli==2.0.1
-typing_extensions==4.5.0
 urllib3==1.26.15
 wcwidth==0.2.6
-Werkzeug==2.2.3

setup.py

@@ -13,7 +13,7 @@
 from setuptools import find_namespace_packages, setup
 
 package_name = "dbt-dremio"
-package_version = "1.4.5"
+package_version = "1.5.0"
 description = """The Dremio adapter plugin for dbt"""
 
 setup(
@@ -27,13 +27,14 @@
     packages=find_namespace_packages(include=["dbt", "dbt.*"]),
     include_package_data=True,
     install_requires=[
-        "dbt-core>=1.2, <=1.4.5",
+        "dbt-core>=1.2, <1.6.0",
+        "requests>=2.31.0",
     ],
     classifiers=[
         "License :: OSI Approved :: Apache Software License",
         "Operating System :: Microsoft :: Windows",
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.10",
+        "Programming Language :: Python :: 3.11",
     ],
 )

tests/caching/test_caching.py

@@ -0,0 +1,70 @@
+import pytest
+from tests.utils.util import BUCKET
+from dbt.tests.adapter.caching.test_caching import (
+    BaseCachingTest,
+    BaseCachingSelectedSchemaOnly,
+    BaseCachingLowercaseModel,
+    BaseCachingUppercaseModel,
+    model_sql,
+)
+
+
+# This ensures the schema works with our datalake
+@pytest.fixture(scope="class")
+def unique_schema(request, prefix) -> str:
+    test_file = request.module.__name__
+    # We only want the last part of the name
+    test_file = test_file.split(".")[-1]
+    unique_schema = f"{BUCKET}.{prefix}_{test_file}"
+    return unique_schema
+
+
+# Override this fixture to set root_path=schema
+@pytest.fixture(scope="class")
+def dbt_profile_data(unique_schema, dbt_profile_target, profiles_config_update):
+    profile = {
+        "config": {"send_anonymous_usage_stats": False},
+        "test": {
+            "outputs": {
+                "default": {},
+            },
+            "target": "default",
+        },
+    }
+    target = dbt_profile_target
+    target["schema"] = unique_schema
+    target["root_path"] = f"{unique_schema}"
+    profile["test"]["outputs"]["default"] = target
+
+    if profiles_config_update:
+        profile.update(profiles_config_update)
+    return profile
+
+
+class TestNoPopulateCache(BaseCachingTest):
+    @pytest.fixture(scope="class")
+    def models(self):
+        return {
+            "model.sql": model_sql,
+        }
+
+    def test_cache(self, project):
+        # --no-populate-cache still allows the cache to populate all relations
+        # under a schema, so the behavior here remains the same as other tests
+        run_args = ["--no-populate-cache", "run"]
+        self.run_and_inspect_cache(project, run_args)
+
+
+class TestCachingLowerCaseModelDremio(BaseCachingLowercaseModel):
+    pass
+
+
+@pytest.mark.skip(
+    reason="Dremio does not support case-sensitive data file names or table names."
+)
+class TestCachingUppercaseModelDremio(BaseCachingUppercaseModel):
+    pass
+
+
+class TestCachingSelectedSchemaOnlyDremio(BaseCachingSelectedSchemaOnly):
+    pass

tests/component/test_profile_template.py

@@ -60,7 +60,6 @@ class TestProfileTemplate:
         "dremio_space": "@user",
         "dremio_space_folder": "no_schema",
         "threads": 1,
-        "verify_ssl": True,
     }
     _DREMIO_CLOUD_PROFILE_SPECIFIC_OPTIONS_WITH_DEFAULTS = {
         "cloud_host": "api.dremio.cloud",
@@ -137,7 +136,6 @@ def _get_dbt_test_project_dict(self, dbt_test_project_name: str) -> Dict[str, an
     def _test_project_profile_options(
         self, test_project: Dict[str, any], test_options: Dict[str, any]
     ) -> None:
-
         assert test_project is not None
 
         for option in test_options:

tests/hooks/data/seed_model.sql

@@ -0,0 +1,16 @@
+
+drop table if exists dbt_test_source.{schema}.on_model_hook;
+
+create table dbt_test_source.{schema}.on_model_hook (
+    test_state       VARCHAR, -- start|end
+    target_dbname    VARCHAR,
+    target_host      VARCHAR,
+    target_name      VARCHAR,
+    target_schema    VARCHAR,
+    target_type      VARCHAR,
+    target_user      VARCHAR,
+    target_pass      VARCHAR,
+    target_threads   INT,
+    run_started_at   VARCHAR,
+    invocation_id    VARCHAR
+);

tests/hooks/data/seed_run.sql

@@ -0,0 +1,16 @@
+
+drop table if exists dbt_test_source.{schema}.on_run_hook;
+
+create table dbt_test_source.{schema}.on_run_hook (
+    test_state       VARCHAR, -- start|end
+    target_dbname    VARCHAR,
+    target_host      VARCHAR,
+    target_name      VARCHAR,
+    target_schema    VARCHAR,
+    target_type      VARCHAR,
+    target_user      VARCHAR,
+    target_pass      VARCHAR,
+    target_threads   INT,
+    run_started_at   VARCHAR,
+    invocation_id    VARCHAR
+);