Merge pull request #173 from dbt-msft/v0.21.0

v0.21.0

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+### v0.21.0
+
+Please see [dbt-core v0.21.0 release notes](https://github.com/dbt-labs/dbt-core/releases/tag/v0.21.0) for upstream changes
+
+#### fixes
+
+- in dbt-sqlserver v0.20.0, users couldn't use some out of the box tests, such as accepted_values. users can now also use CTEs in their ~bespoke~ custom data tests
+- fixes issue with changing column types in incremental table column type [#152](https://github.com/dbt-msft/dbt-sqlserver/issue/152) [#169](https://github.com/dbt-msft/dbt-sqlserver/pull/169)
+- workaround for Azure CLI token expires after one hour. Now we get new tokens for every transaction. [#156](https://github.com/dbt-msft/dbt-sqlserver/issue/156) [#158](https://github.com/dbt-msft/dbt-sqlserver/pull/158)
+
 ### v0.20.1
 
 #### fixes:

dbt/adapters/sqlserver/__version__.py

@@ -1 +1 @@
-version = '0.20.1'
+version = '0.21.0'

dbt/adapters/sqlserver/connections.py

@@ -79,6 +79,10 @@ def _connection_keys(self):
  "trust_cert"
  )
 
+ @property
+ def unique_field(self):
+ return self.host
+
 
 def convert_bytes_to_mswindows_byte_string(value: bytes) -> bytes:
  """

dbt/include/sqlserver/macros/adapters.sql

@@ -166,7 +166,7 @@
  {{ sqlserver__create_clustered_columnstore_index(relation) }}
  {% endif %}
 
-{% endmacro %}_
+{% endmacro %}
 
 {% macro sqlserver__insert_into_from(to_relation, from_relation) -%}
  SELECT * INTO {{ to_relation }} FROM {{ from_relation }}

dbt/include/sqlserver/macros/materializations/seed/seed.sql

@@ -13,43 +13,54 @@
  {{ return(batch_size) }}
 {% endmacro %}
 
-{% macro sqlserver__basic_load_csv_rows(model, max_batch_size, agate_table) %}
- {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
-
- {% set batch_size = calc_batch_size(cols_sql|length, max_batch_size) %}
- {% set bindings = [] %}
- {% set statements = [] %}
-
- {% for chunk in agate_table.rows | batch(batch_size) %}
- {% set bindings = [] %}
-
- {% for row in chunk %}
- {% set _ = bindings.extend(row) %}
- {% endfor %}
-
- {% set sql %}
- insert into {{ this.render() }} ({{ cols_sql }}) values
- {% for row in chunk -%}
- ({%- for column in agate_table.column_names -%}
- ?
- {%- if not loop.last%},{%- endif %}
- {%- endfor -%})
- {%- if not loop.last%},{%- endif %}
- {%- endfor %}
- {% endset %}
-
- {% set _ = adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
-
- {% if loop.index0 == 0 %}
- {% set _ = statements.append(sql) %}
- {% endif %}
- {% endfor %}
-
- {# Return SQL so we can render it out into the compiled files #}
- {{ return(statements[0]) }}
+{% macro sqlserver__get_binding_char() %}
+ {{ return('?') }}
+{% endmacro %}
+
+{% macro sqlserver__get_batch_size() %}
+ {{ return(400) }}
+{% endmacro %}
+
+{% macro basic_load_csv_rows(model, batch_size, agate_table) %}
+
+ {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+ {% set bindings = [] %}
+
+ {% set statements = [] %}
+
+ {% for chunk in agate_table.rows | batch(batch_size) %}
+ {% set bindings = [] %}
+
+ {% for row in chunk %}
+ {% do bindings.extend(row) %}
+ {% endfor %}
+
+ {% set sql %}
+ insert into {{ this.render() }} ({{ cols_sql }}) values
+ {% for row in chunk -%}
+ ({%- for column in agate_table.column_names -%}
+ {{ get_binding_char() }}
+ {%- if not loop.last%},{%- endif %}
+ {%- endfor -%})
+ {%- if not loop.last%},{%- endif %}
+ {%- endfor %}
+ {% endset %}
+
+ {% do adapter.add_query(sql, bindings=bindings, abridge_sql_log=True) %}
+
+ {% if loop.index0 == 0 %}
+ {% do statements.append(sql) %}
+ {% endif %}
+ {% endfor %}
+
+ {# Return SQL so we can render it out into the compiled files #}
+ {{ return(statements[0]) }}
 {% endmacro %}
 
 {% macro sqlserver__load_csv_rows(model, agate_table) %}
- {% set max_batch_size = var("max_batch_size", 400) %}
- {{ return(sqlserver__basic_load_csv_rows(model, max_batch_size, agate_table) )}}
-{% endmacro %}
+ {% set max_batch_size = get_batch_size() %}
+ {% set cols_sql = get_seed_column_quoted_csv(model, agate_table.column_names) %}
+ {% set batch_size = calc_batch_size(cols_sql|length, max_batch_size) %}
+
+ {{ return(basic_load_csv_rows(model, batch_size, agate_table) )}}
+{% endmacro %}

dbt/include/sqlserver/macros/materializations/test.sql

@@ -1,12 +1,61 @@
 {% macro sqlserver__get_test_sql(main_sql, fail_calc, warn_if, error_if, limit) -%}
  select
+ {{ "top (" ~ limit ~ ')' if limit != none }}
  {{ fail_calc }} as failures,
  case when {{ fail_calc }} {{ warn_if }}
  then 'true' else 'false' end as should_warn,
  case when {{ fail_calc }} {{ error_if }}
  then 'true' else 'false' end as should_error
  from (
  {{ main_sql }}
- {{ "limit " ~ limit if limit != none }}
  ) dbt_internal_test
-{%- endmacro %}
+{%- endmacro %}
+
+{%- materialization test, adapter='sqlserver' -%}
+
+ {% set relations = [] %}
+
+ {% set identifier = model['alias'] %}
+ {% set old_relation = adapter.get_relation(database=database, schema=schema, identifier=identifier) %}
+ {% set target_relation = api.Relation.create(
+ identifier=identifier, schema=schema, database=database, type='table') -%} %}
+
+
+ {% if old_relation %}
+ {% do adapter.drop_relation(old_relation) %}
+ {% elif not old_relation %}
+ {% do adapter.create_schema(target_relation) %}
+ {% endif %}
+
+ {% call statement(auto_begin=True) %}
+ {{ create_table_as(False, target_relation, sql) }}
+ {% endcall %}
+
+ {% set main_sql %}
+ select *
+ from {{ target_relation }}
+ {% endset %}
+
+ {{ adapter.commit() }}
+
+
+ {% set limit = config.get('limit') %}
+ {% set fail_calc = config.get('fail_calc') %}
+ {% set warn_if = config.get('warn_if') %}
+ {% set error_if = config.get('error_if') %}
+
+ {% call statement('main', fetch_result=True) -%}
+
+ {{ get_test_sql(main_sql, fail_calc, warn_if, error_if, limit)}}
+
+ {%- endcall %}
+
+ {% if should_store_failures() %}
+ {% do relations.append(target_relation) %}
+ {% elif not should_store_failures() %}
+ {% do adapter.drop_relation(target_relation) %}
+ {% endif %}
+
+ {{ return({'relations': relations}) }}
+
+{%- endmaterialization -%}

dbt/include/sqlserver/macros/merge.sql

@@ -0,0 +1,18 @@
+ {# global project no longer includes semi-colons in merge statements, so
+ default macro are invoked below w/ a semi-colons after it.
+ more context:
+ https://github.com/dbt-labs/dbt-core/pull/3510
+ https://getdbt.slack.com/archives/C50NEBJGG/p1636045535056600
+ #}
+
+{% macro sqlserver__get_merge_sql(target, source, unique_key, dest_columns, predicates) %}
+ {{ default__get_merge_sql(target, source, unique_key, dest_columns, predicates) }};
+{% endmacro %}
+
+{% macro sqlserver__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) %}
+ {{ default__get_insert_overwrite_merge_sql(target, source, dest_columns, predicates, include_sql_header) }};
+{% endmacro %}
+
+{% macro sqlserver__snapshot_merge_sql(target, source, insert_cols) %}
+ {{ default__snapshot_merge_sql(target, source, insert_cols) }};
+{% endmacro %}

setup.py

@@ -1,6 +1,5 @@
 #!/usr/bin/env python
-from setuptools import find_packages
-from distutils.core import setup
+from setuptools import find_namespace_packages, setup
 import os
 import re
 
@@ -28,7 +27,7 @@ def _dbt_sqlserver_version():
 package_version = _dbt_sqlserver_version()
 description = """A sqlserver adapter plugin for dbt (data build tool)"""
 
-dbt_version = '0.20'
+dbt_version = '0.21'
 # the package version should be the dbt version, with maybe some things on the
 # ends of it. (0.18.1 vs 0.18.1a1, 0.18.1.1, ...)
 if not package_version.startswith(dbt_version):
@@ -47,11 +46,11 @@ def _dbt_sqlserver_version():
  author="Mikael Ene",
  author_email="mikael.ene@eneanalytics.com",
  url="https://github.com/mikaelene/dbt-sqlserver",
- packages=find_packages(),
+ packages=find_namespace_packages(include=['dbt', 'dbt.*']),
  include_package_data=True,
  install_requires=[
- "dbt-core~=0.20.0",
+ "dbt-core~=0.21.1",
  "pyodbc~=4.0.31",
- "azure-identity>=1.6.0",
+ "azure-identity>=1.7.0",
  ]
 )