Add SQL Endpoint specific integration tests. (#45)

Adds a new test profile and env variable DBT_DATABRICKS_ENDPOINT_HTTP_PATH

### Description

Some features behave differently on sql endpoints vs spark clusters.

tests/integration/base.py

@@ -85,7 +85,7 @@ def __init__(self, kwargs):
 
 
 def _profile_from_test_name(test_name):
-    adapter_names = 'databricks_sql_connector',
+    adapter_names = ('databricks_sql_connector', 'databricks_sql_endpoint_connector')
     adapters_in_name = sum(x in test_name for x in adapter_names)
     if adapters_in_name != 1:
         raise ValueError(
@@ -162,6 +162,25 @@ def databricks_sql_connector_profile(self):
             }
         }
 
+    def databricks_sql_endpoint_connector_profile(self):
+        return {
+            'config': {
+                'send_anonymous_usage_stats': False
+            },
+            'test': {
+                'outputs': {
+                    'endpoint': {
+                        'type': 'databricks',
+                        'host': os.getenv('DBT_DATABRICKS_HOST_NAME'),
+                        'http_path': os.getenv('DBT_DATABRICKS_ENDPOINT_HTTP_PATH'),
+                        'token': os.getenv('DBT_DATABRICKS_TOKEN'),
+                        'schema': self.unique_schema()
+                    },
+                },
+                'target': 'endpoint'
+            }
+        }
+
     @property
     def packages_config(self):
         return None
@@ -189,6 +208,8 @@ def alternative_database(self):
     def get_profile(self, adapter_type):
         if adapter_type == 'databricks_sql_connector':
             return self.databricks_sql_connector_profile()
+        elif adapter_type == 'databricks_sql_endpoint_connector':
+            return self.databricks_sql_endpoint_connector_profile()
         else:
             raise ValueError('invalid adapter type {}'.format(adapter_type))
 

tests/integration/conftest.py

@@ -2,3 +2,6 @@ def pytest_configure(config):
     config.addinivalue_line(
         "markers", "profile_databricks_sql_connector"
     )
+    config.addinivalue_line(
+        "markers", "profile_databricks_sql_endpoint_connector"
+    )

tests/specs/spark-databricks-sql-endpoint-connector.dbtspec

@@ -0,0 +1,32 @@
+target:
+  type: databricks
+  host: "{{ env_var('DBT_DATABRICKS_HOST_NAME') }}"
+  http_path: "{{ env_var('DBT_DATABRICKS_ENDPOINT_HTTP_PATH') }}"
+  token: "{{ env_var('DBT_DATABRICKS_TOKEN') }}"
+  port: 443
+  schema: "analytics_{{ var('_dbt_random_suffix') }}"
+  connect_retries: 5
+  connect_timeout: 60
+projects:
+  - overrides: snapshot_strategy_check_cols
+    dbt_project_yml: &file_format_delta
+      # we're going to UPDATE the seed tables as part of testing, so we must make them delta format
+      seeds:
+        dbt_test_project:
+          file_format: delta
+      snapshots:
+        dbt_test_project:
+          file_format: delta
+  - overrides: snapshot_strategy_timestamp
+    dbt_project_yml: *file_format_delta
+sequences:
+  test_dbt_empty: empty
+  # Skip for now as not all `SET` commands work on endpoints
+  # test_dbt_base: base
+  test_dbt_ephemeral: ephemeral
+  test_dbt_incremental: incremental
+  test_dbt_snapshot_strategy_timestamp: snapshot_strategy_timestamp
+  test_dbt_snapshot_strategy_check_cols: snapshot_strategy_check_cols
+  test_dbt_data_test: data_test
+  test_dbt_ephemeral_data_tests: data_test_ephemeral_models
+  test_dbt_schema_test: schema_test

tox.ini

@@ -27,3 +27,13 @@ deps =
     -r{toxinidir}/requirements.txt
     -r{toxinidir}/dev_requirements.txt
     -e.
+
+[testenv:integration-spark-databricks-sql-endpoint-connector]
+basepython = python3.8
+commands = /bin/bash -c '{envpython} -m pytest -v tests/specs/spark-databricks-sql-endpoint-connector.dbtspec'
+           /bin/bash -c '{envpython} -m pytest -v -m profile_databricks_sql_connector {posargs} -n4 tests/integration/*'
+passenv = DBT_* PYTEST_ADDOPTS
+deps =
+    -r{toxinidir}/requirements.txt
+    -r{toxinidir}/dev_requirements.txt
+    -e.