Test various snapshot configs

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 1.9.0a1
+current_version = 1.9.0b1
 parse = (?P<major>[\d]+) # major version number
 	\.(?P<minor>[\d]+) # minor version number
 	\.(?P<patch>[\d]+) # patch version number

.changes/1.9.0-b1.md

@@ -0,0 +1,31 @@
+## dbt-spark 1.9.0-b1 - October 01, 2024
+
+### Features
+
+- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028))
+- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031))
+- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096))
+- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109))
+
+### Fixes
+
+- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033))
+
+### Under the Hood
+
+- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049))
+
+### Dependencies
+
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966))
+- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010))
+- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011))
+- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012))
+- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035))
+- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073))
+- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074))
+
+### Contributors
+- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))

.changes/unreleased/Breaking Changes-20241016-184157.yaml

@@ -0,0 +1,6 @@
+kind: Breaking Changes
+body: Drop support for Python 3.8
+time: 2024-10-16T18:41:57.721002-04:00
+custom:
+  Author: mikealfare
+  Issue: "1121"

.changes/unreleased/Features-20240927-133927.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Enable setting current value of dbt_valid_to
+time: 2024-09-27T13:39:27.268886-04:00
+custom:
+  Author: gshank
+  Issue: "1112"

.changes/unreleased/Under the Hood-20240911-192845.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Isolating distribution testing
+time: 2024-09-11T19:28:45.653064-04:00
+custom:
+    Author: leahwicz
+    Issue: "1069"

.github/workflows/main.yml

@@ -49,7 +49,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -75,7 +75,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository
@@ -126,7 +126,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.8'
+          python-version: '3.9'
 
       - name: Install python dependencies
         run: |
@@ -161,7 +161,7 @@ jobs:
           overwrite: true
 
   test-build:
-    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }}
+    name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }}
 
     if: needs.build.outputs.is_alpha == 0
 
@@ -172,8 +172,9 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        os: [ubuntu-latest, macos-12, windows-latest]
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        os: [ubuntu-latest, macos-14, windows-latest]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
+        dist-type: ["whl", "gz"]
 
     steps:
       - name: Set up Python ${{ matrix.python-version }}
@@ -186,6 +187,7 @@ jobs:
           python -m pip install --user --upgrade pip
           python -m pip install --upgrade wheel
           python -m pip --version
+
       - uses: actions/download-artifact@v4
         with:
           name: dist
@@ -194,15 +196,10 @@ jobs:
       - name: Show distributions
         run: ls -lh dist/
 
-      - name: Install wheel distributions
-        run: |
-          find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check wheel distributions
+      - name: Install ${{ matrix.dist-type }} distributions
         run: |
-          python -c "import dbt.adapters.spark"
-      - name: Install source distributions
-        run: |
-          find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
-      - name: Check source distributions
+          find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/
+
+      - name: Check ${{ matrix.dist-type }} distributions
         run: |
           python -c "import dbt.adapters.spark"

.github/workflows/release-internal.yml

@@ -37,7 +37,7 @@ defaults:
     shell: "bash"
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
 
 jobs:
   run-unit-tests:

.github/workflows/release-prep.yml

@@ -84,7 +84,7 @@ defaults:
     shell: bash
 
 env:
-  PYTHON_TARGET_VERSION: 3.8
+  PYTHON_TARGET_VERSION: 3.9
   NOTIFICATION_PREFIX: "[Release Preparation]"
 
 jobs:
@@ -448,7 +448,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"]
+        python-version: ["3.9", "3.10", "3.11", "3.12"]
 
     steps:
       - name: Check out the repository

.pre-commit-config.yaml

@@ -24,7 +24,6 @@ repos:
     -   id: black
         args:
         -   --line-length=99
-        -   --target-version=py38
         -   --target-version=py39
         -   --target-version=py310
         -   --target-version=py311

CHANGELOG.md

@@ -5,10 +5,41 @@
 - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version.
 - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry)
 
+## dbt-spark 1.9.0-b1 - October 01, 2024
+
+### Features
+
+- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028))
+- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031))
+- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096))
+- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109))
+
+### Fixes
+
+- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033))
+
+### Under the Hood
+
+- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049))
+
+### Dependencies
+
+- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966))
+- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010))
+- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011))
+- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012))
+- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035))
+- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073))
+- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074))
+
+### Contributors
+- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092))
+
+
 ## Previous Releases
 For information on prior major and minor releases, see their changelogs:
-- [1.8](https://github.com/dbt-labs/dbt-spark/blob/1.8.latest/CHANGELOG.md)
-- [1.7](https://github.com/dbt-labs/dbt-spark/blob/1.7.latest/CHANGELOG.md)
 - [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md)
 - [1.5](https://github.com/dbt-labs/dbt-spark/blob/1.5.latest/CHANGELOG.md)
 - [1.4](https://github.com/dbt-labs/dbt-spark/blob/1.4.latest/CHANGELOG.md)

Makefile

@@ -17,12 +17,12 @@ lint: ## Runs flake8 and mypy code checks against staged changes.
 	pre-commit run --all-files
 
 .PHONY: unit
-unit: ## Runs unit tests with py38.
+unit: ## Runs unit tests with py39.
 	@\
 	python -m pytest tests/unit
 
 .PHONY: test
-test: ## Runs unit tests with py38 and code checks against staged changes.
+test: ## Runs unit tests with py39 and code checks against staged changes.
 	@\
 	python -m pytest tests/unit; \
 	python dagger/run_dbt_spark_tests.py --profile spark_session \

dagger/run_dbt_spark_tests.py

@@ -104,7 +104,7 @@ async def test_spark(test_args):
         platform = dagger.Platform("linux/amd64")
         tst_container = (
             client.container(platform=platform)
-            .from_("python:3.8-slim")
+            .from_("python:3.9-slim")
             .with_mounted_cache("/var/cache/apt/archives", os_reqs_cache)
             .with_mounted_cache("/root/.cache/pip", pip_cache)
             # install OS deps first so any local changes don't invalidate the cache

dbt/adapters/spark/__version__.py

@@ -1 +1 @@
-version = "1.9.0a1"
+version = "1.9.0b1"

dbt/include/spark/macros/materializations/snapshot.sql

@@ -24,7 +24,12 @@
     {% endif %}
     on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }}
     when matched
-     and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% if config.get("dbt_valid_to_current") %}
+       and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or
+             DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null )
+     {% else %}
+       and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null
+     {% endif %}
      and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete')
         then update
         set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }}

dev-requirements.txt

@@ -1,14 +1,13 @@
 # install latest changes in dbt-core
 # TODO: how to automate switching from develop to version branches?
-git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
+git+https://github.com/dbt-labs/dbt-core.git@snapshot_dbt_valid_to_current#egg=dbt-core&subdirectory=core
 git+https://github.com/dbt-labs/dbt-common.git
-git+https://github.com/dbt-labs/dbt-adapters.git
-git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter
+git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs
+git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs#subdirectory=dbt-tests-adapter
 
 # dev
 ipdb~=0.13.13
-pre-commit~=3.7.0;python_version>="3.9"
-pre-commit~=3.5.0;python_version<"3.9"
+pre-commit~=3.7.0
 pytest~=7.4
 pytest-csv~=3.0
 pytest-dotenv~=0.5.2

docker/Dockerfile

@@ -1,7 +1,7 @@
 # this image gets published to GHCR for production use
 ARG py_version=3.11.2
 
-FROM python:$py_version-slim-bullseye as base
+FROM python:$py_version-slim-bullseye AS base
 
 RUN apt-get update \
   && apt-get dist-upgrade -y \
@@ -29,7 +29,7 @@ ENV LANG=C.UTF-8
 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir
 
 
-FROM base as dbt-spark
+FROM base AS dbt-spark
 
 ARG commit_ref=main
 ARG extras=all

setup.py

@@ -4,9 +4,9 @@
 import re
 
 # require python 3.8 or newer
-if sys.version_info < (3, 8):
+if sys.version_info < (3, 9):
     print("Error: dbt does not support this version of Python.")
-    print("Please upgrade to Python 3.8 or higher.")
+    print("Please upgrade to Python 3.9 or higher.")
     sys.exit(1)
 
 # require version of setuptools that supports find_namespace_packages
@@ -41,7 +41,7 @@ def _get_plugin_version_dict():
 
 
 package_name = "dbt-spark"
-package_version = "1.9.0a1"
+package_version = "1.9.0b1"
 description = """The Apache Spark adapter plugin for dbt"""
 
 odbc_extras = ["pyodbc~=5.1.0"]
@@ -83,11 +83,10 @@ def _get_plugin_version_dict():
         "Operating System :: Microsoft :: Windows",
         "Operating System :: MacOS :: MacOS X",
         "Operating System :: POSIX :: Linux",
-        "Programming Language :: Python :: 3.8",
         "Programming Language :: Python :: 3.9",
         "Programming Language :: Python :: 3.10",
         "Programming Language :: Python :: 3.11",
         "Programming Language :: Python :: 3.12",
     ],
-    python_requires=">=3.8",
+    python_requires=">=3.9",
 )

tests/functional/adapter/test_python_model.py

@@ -85,7 +85,11 @@ def model(dbt, spark):
 
 
 @pytest.mark.skip_profile(
-    "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc"
+    "apache_spark",
+    "spark_session",
+    "databricks_sql_endpoint",
+    "spark_http_odbc",
+    "databricks_http_cluster",
 )
 class TestChangingSchemaSpark:
     """

tests/functional/adapter/test_simple_snapshot.py

@@ -0,0 +1,32 @@
+from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot
+
+from dbt.tests.adapter.simple_snapshot.test_various_configs import (
+    BaseSnapshotColumnNames,
+    BaseSnapshotColumnNamesFromDbtProject,
+    BaseSnapshotInvalidColumnNames,
+    BaseSnapshotDbtValidToCurrent,
+)
+
+
+class TestSnapshot(BaseSimpleSnapshot):
+    pass
+
+
+class TestSnapshotCheck(BaseSnapshotCheck):
+    pass
+
+
+class TestSnapshotColumnNames(BaseSnapshotColumnNames):
+    pass
+
+
+class TestSnapshotColumnNamesFromDbtProject(BaseSnapshotColumnNamesFromDbtProject):
+    pass
+
+
+class TestSnapshotInvalidColumnNames(BaseSnapshotInvalidColumnNames):
+    pass
+
+
+class TestSnapshotDbtValidToCurrent(BaseSnapshotDbtValidToCurrent):
+    pass