From a0064fc3f038b519961db8caab48fe29696db38b Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Wed, 25 Sep 2024 18:25:39 -0400 Subject: [PATCH 01/10] Use dbt_valid_to_current config --- dbt/include/spark/macros/materializations/snapshot.sql | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/snapshot.sql b/dbt/include/spark/macros/materializations/snapshot.sql index 43c4750f6..005a2e3d3 100644 --- a/dbt/include/spark/macros/materializations/snapshot.sql +++ b/dbt/include/spark/macros/materializations/snapshot.sql @@ -24,7 +24,11 @@ {% endif %} on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% if config.get("dbt_valid_to_current") %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} + {% else %} + and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null + {% endif %} and DBT_INTERNAL_SOURCE.dbt_change_type in ('update', 'delete') then update set {{ columns.dbt_valid_to }} = DBT_INTERNAL_SOURCE.{{ columns.dbt_valid_to }} From a5a25abadc6df5e55787ef7d042819a95503ff64 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 27 Sep 2024 13:29:27 -0400 Subject: [PATCH 02/10] Check both dbt_valid_to_current and null --- dbt/include/spark/macros/materializations/snapshot.sql | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/dbt/include/spark/macros/materializations/snapshot.sql b/dbt/include/spark/macros/materializations/snapshot.sql index 005a2e3d3..b4ef6e5d4 100644 --- a/dbt/include/spark/macros/materializations/snapshot.sql +++ b/dbt/include/spark/macros/materializations/snapshot.sql @@ -25,7 +25,8 @@ on DBT_INTERNAL_SOURCE.{{ columns.dbt_scd_id }} = DBT_INTERNAL_DEST.{{ columns.dbt_scd_id }} when matched {% if config.get("dbt_valid_to_current") %} - and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} + and ( DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} = {{ config.get('dbt_valid_to_current') }} or + DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null ) {% else %} and DBT_INTERNAL_DEST.{{ columns.dbt_valid_to }} is null {% endif %} From ff3fc504c7b50ad54d20f07bce6d7479620ad48f Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 27 Sep 2024 13:39:33 -0400 Subject: [PATCH 03/10] Changie --- .changes/unreleased/Features-20240927-133927.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 .changes/unreleased/Features-20240927-133927.yaml diff --git a/.changes/unreleased/Features-20240927-133927.yaml b/.changes/unreleased/Features-20240927-133927.yaml new file mode 100644 index 000000000..ce04ac073 --- /dev/null +++ b/.changes/unreleased/Features-20240927-133927.yaml @@ -0,0 +1,6 @@ +kind: Features +body: Enable setting current value of dbt_valid_to +time: 2024-09-27T13:39:27.268886-04:00 +custom: + Author: gshank + Issue: "1112" From 61b573171a8b7b06a5a9bd6bef8551bbd11dfa66 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Thu, 10 Oct 2024 11:15:01 -0400 Subject: [PATCH 04/10] Add test_simple_snapshot.py --- dev-requirements.txt | 6 ++-- .../adapter/test_simple_snapshot.py | 32 +++++++++++++++++++ 2 files changed, 35 insertions(+), 3 deletions(-) create mode 100644 tests/functional/adapter/test_simple_snapshot.py diff --git a/dev-requirements.txt b/dev-requirements.txt index 055cb92f7..f44ff4a85 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -1,9 +1,9 @@ # install latest changes in dbt-core # TODO: how to automate switching from develop to version branches? -git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core +git+https://github.com/dbt-labs/dbt-core.git@snapshot_dbt_valid_to_current#egg=dbt-core&subdirectory=core git+https://github.com/dbt-labs/dbt-common.git -git+https://github.com/dbt-labs/dbt-adapters.git -git+https://github.com/dbt-labs/dbt-adapters.git#subdirectory=dbt-tests-adapter +git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs +git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs#subdirectory=dbt-tests-adapter # dev ipdb~=0.13.13 diff --git a/tests/functional/adapter/test_simple_snapshot.py b/tests/functional/adapter/test_simple_snapshot.py new file mode 100644 index 000000000..660678a99 --- /dev/null +++ b/tests/functional/adapter/test_simple_snapshot.py @@ -0,0 +1,32 @@ +from dbt.tests.adapter.simple_snapshot.test_snapshot import BaseSnapshotCheck, BaseSimpleSnapshot + +from dbt.tests.adapter.simple_snapshot.test_various_configs import ( + BaseSnapshotColumnNames, + BaseSnapshotColumnNamesFromDbtProject, + BaseSnapshotInvalidColumnNames, + BaseSnapshotDbtValidToCurrent, +) + + +class TestSnapshot(BaseSimpleSnapshot): + pass + + +class TestSnapshotCheck(BaseSnapshotCheck): + pass + + +class TestSnapshotColumnNames(BaseSnapshotColumnNames): + pass + + +class TestSnapshotColumnNamesFromDbtProject(BaseSnapshotColumnNamesFromDbtProject): + pass + + +class TestSnapshotInvalidColumnNames(BaseSnapshotInvalidColumnNames): + pass + + +class TestSnapshotDbtValidToCurrent(BaseSnapshotDbtValidToCurrent): + pass From 9853bf705830d3d66c378c4637f0f2864be75660 Mon Sep 17 00:00:00 2001 From: Gerda Shank Date: Fri, 11 Oct 2024 14:37:16 -0400 Subject: [PATCH 05/10] Try skipping databricks_http_cluster profile for test --- tests/functional/adapter/test_python_model.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/functional/adapter/test_python_model.py b/tests/functional/adapter/test_python_model.py index 50132b883..c95c29270 100644 --- a/tests/functional/adapter/test_python_model.py +++ b/tests/functional/adapter/test_python_model.py @@ -85,7 +85,11 @@ def model(dbt, spark): @pytest.mark.skip_profile( - "apache_spark", "spark_session", "databricks_sql_endpoint", "spark_http_odbc" + "apache_spark", + "spark_session", + "databricks_sql_endpoint", + "spark_http_odbc", + "databricks_http_cluster", ) class TestChangingSchemaSpark: """ From 278b475087416b08fbed786a52340e28e09623f9 Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Thu, 17 Oct 2024 15:17:07 -0400 Subject: [PATCH 06/10] Drop support for Python 3.8 (#1121) * drop support for python 3.8 * drop support for python 3.8 --- .changes/unreleased/Breaking Changes-20241016-184157.yaml | 6 ++++++ .github/workflows/main.yml | 8 ++++---- .github/workflows/release-internal.yml | 2 +- .github/workflows/release-prep.yml | 4 ++-- .pre-commit-config.yaml | 1 - Makefile | 4 ++-- dagger/run_dbt_spark_tests.py | 2 +- dev-requirements.txt | 3 +-- docker/Dockerfile | 4 ++-- setup.py | 7 +++---- 10 files changed, 22 insertions(+), 19 deletions(-) create mode 100644 .changes/unreleased/Breaking Changes-20241016-184157.yaml diff --git a/.changes/unreleased/Breaking Changes-20241016-184157.yaml b/.changes/unreleased/Breaking Changes-20241016-184157.yaml new file mode 100644 index 000000000..0a2104e60 --- /dev/null +++ b/.changes/unreleased/Breaking Changes-20241016-184157.yaml @@ -0,0 +1,6 @@ +kind: Breaking Changes +body: Drop support for Python 3.8 +time: 2024-10-16T18:41:57.721002-04:00 +custom: + Author: mikealfare + Issue: "1121" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e4fc66ccc..a4e2c7a45 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -49,7 +49,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -75,7 +75,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out the repository @@ -126,7 +126,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.8' + python-version: '3.9' - name: Install python dependencies run: | @@ -173,7 +173,7 @@ jobs: fail-fast: false matrix: os: [ubuntu-latest, macos-12, windows-latest] - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Set up Python ${{ matrix.python-version }} diff --git a/.github/workflows/release-internal.yml b/.github/workflows/release-internal.yml index 1a5090312..702ef9aea 100644 --- a/.github/workflows/release-internal.yml +++ b/.github/workflows/release-internal.yml @@ -37,7 +37,7 @@ defaults: shell: "bash" env: - PYTHON_TARGET_VERSION: 3.8 + PYTHON_TARGET_VERSION: 3.9 jobs: run-unit-tests: diff --git a/.github/workflows/release-prep.yml b/.github/workflows/release-prep.yml index 9937463d3..d5878ec1e 100644 --- a/.github/workflows/release-prep.yml +++ b/.github/workflows/release-prep.yml @@ -84,7 +84,7 @@ defaults: shell: bash env: - PYTHON_TARGET_VERSION: 3.8 + PYTHON_TARGET_VERSION: 3.9 NOTIFICATION_PREFIX: "[Release Preparation]" jobs: @@ -448,7 +448,7 @@ jobs: strategy: fail-fast: false matrix: - python-version: ["3.8", "3.9", "3.10", "3.11", "3.12"] + python-version: ["3.9", "3.10", "3.11", "3.12"] steps: - name: Check out the repository diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index fdb195262..6697bbeb5 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -24,7 +24,6 @@ repos: - id: black args: - --line-length=99 - - --target-version=py38 - --target-version=py39 - --target-version=py310 - --target-version=py311 diff --git a/Makefile b/Makefile index ff4c0fc1b..46b9af294 100644 --- a/Makefile +++ b/Makefile @@ -17,12 +17,12 @@ lint: ## Runs flake8 and mypy code checks against staged changes. pre-commit run --all-files .PHONY: unit -unit: ## Runs unit tests with py38. +unit: ## Runs unit tests with py39. @\ python -m pytest tests/unit .PHONY: test -test: ## Runs unit tests with py38 and code checks against staged changes. +test: ## Runs unit tests with py39 and code checks against staged changes. @\ python -m pytest tests/unit; \ python dagger/run_dbt_spark_tests.py --profile spark_session \ diff --git a/dagger/run_dbt_spark_tests.py b/dagger/run_dbt_spark_tests.py index 67fa56587..6c310a6f8 100644 --- a/dagger/run_dbt_spark_tests.py +++ b/dagger/run_dbt_spark_tests.py @@ -104,7 +104,7 @@ async def test_spark(test_args): platform = dagger.Platform("linux/amd64") tst_container = ( client.container(platform=platform) - .from_("python:3.8-slim") + .from_("python:3.9-slim") .with_mounted_cache("/var/cache/apt/archives", os_reqs_cache) .with_mounted_cache("/root/.cache/pip", pip_cache) # install OS deps first so any local changes don't invalidate the cache diff --git a/dev-requirements.txt b/dev-requirements.txt index f44ff4a85..1fbead472 100644 --- a/dev-requirements.txt +++ b/dev-requirements.txt @@ -7,8 +7,7 @@ git+https://github.com/dbt-labs/dbt-adapters.git@test_various_snapshot_configs#s # dev ipdb~=0.13.13 -pre-commit~=3.7.0;python_version>="3.9" -pre-commit~=3.5.0;python_version<"3.9" +pre-commit~=3.7.0 pytest~=7.4 pytest-csv~=3.0 pytest-dotenv~=0.5.2 diff --git a/docker/Dockerfile b/docker/Dockerfile index ef4574ddd..81e5e28f3 100644 --- a/docker/Dockerfile +++ b/docker/Dockerfile @@ -1,7 +1,7 @@ # this image gets published to GHCR for production use ARG py_version=3.11.2 -FROM python:$py_version-slim-bullseye as base +FROM python:$py_version-slim-bullseye AS base RUN apt-get update \ && apt-get dist-upgrade -y \ @@ -29,7 +29,7 @@ ENV LANG=C.UTF-8 RUN python -m pip install --upgrade "pip==24.0" "setuptools==69.2.0" "wheel==0.43.0" --no-cache-dir -FROM base as dbt-spark +FROM base AS dbt-spark ARG commit_ref=main ARG extras=all diff --git a/setup.py b/setup.py index 00aeba60d..cf8ff5691 100644 --- a/setup.py +++ b/setup.py @@ -4,9 +4,9 @@ import re # require python 3.8 or newer -if sys.version_info < (3, 8): +if sys.version_info < (3, 9): print("Error: dbt does not support this version of Python.") - print("Please upgrade to Python 3.8 or higher.") + print("Please upgrade to Python 3.9 or higher.") sys.exit(1) # require version of setuptools that supports find_namespace_packages @@ -83,11 +83,10 @@ def _get_plugin_version_dict(): "Operating System :: Microsoft :: Windows", "Operating System :: MacOS :: MacOS X", "Operating System :: POSIX :: Linux", - "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", "Programming Language :: Python :: 3.10", "Programming Language :: Python :: 3.11", "Programming Language :: Python :: 3.12", ], - python_requires=">=3.8", + python_requires=">=3.9", ) From 4463bd0a4afa033d255f47a296dd670ef689bb6e Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Thu, 17 Oct 2024 20:33:12 +0000 Subject: [PATCH 07/10] [create-pull-request] automated change (#1116) Co-authored-by: Github Build Bot --- .bumpversion.cfg | 2 +- .changes/1.9.0-b1.md | 31 ++++++++++++++++ .../Dependencies-20231219-222211.yaml | 0 .../Dependencies-20240419-024814.yaml | 0 .../Dependencies-20240419-024818.yaml | 0 .../Dependencies-20240419-024820.yaml | 0 .../Dependencies-20240503-224157.yaml | 0 .../Dependencies-20240718-223145.yaml | 0 .../Dependencies-20240718-223238.yaml | 0 .../Features-20240430-185723.yaml | 0 .../Features-20240501-151904.yaml | 0 .../Features-20240903-161003.yaml | 0 .../Features-20240910-175846.yaml | 0 .../Features-20240925-125242.yaml | 0 .../Fixes-20240513-160121.yaml | 0 .../Under the Hood-20240612-195629.yaml | 0 CHANGELOG.md | 35 +++++++++++++++++-- dbt/adapters/spark/__version__.py | 2 +- setup.py | 2 +- 19 files changed, 67 insertions(+), 5 deletions(-) create mode 100644 .changes/1.9.0-b1.md rename .changes/{unreleased => 1.9.0}/Dependencies-20231219-222211.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240419-024814.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240419-024818.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240419-024820.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240503-224157.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-223145.yaml (100%) rename .changes/{unreleased => 1.9.0}/Dependencies-20240718-223238.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240430-185723.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240501-151904.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240903-161003.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240910-175846.yaml (100%) rename .changes/{unreleased => 1.9.0}/Features-20240925-125242.yaml (100%) rename .changes/{unreleased => 1.9.0}/Fixes-20240513-160121.yaml (100%) rename .changes/{unreleased => 1.9.0}/Under the Hood-20240612-195629.yaml (100%) diff --git a/.bumpversion.cfg b/.bumpversion.cfg index 77a3f463f..40074e83c 100644 --- a/.bumpversion.cfg +++ b/.bumpversion.cfg @@ -1,5 +1,5 @@ [bumpversion] -current_version = 1.9.0a1 +current_version = 1.9.0b1 parse = (?P[\d]+) # major version number \.(?P[\d]+) # minor version number \.(?P[\d]+) # patch version number diff --git a/.changes/1.9.0-b1.md b/.changes/1.9.0-b1.md new file mode 100644 index 000000000..b6daee41e --- /dev/null +++ b/.changes/1.9.0-b1.md @@ -0,0 +1,31 @@ +## dbt-spark 1.9.0-b1 - October 01, 2024 + +### Features + +- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028)) +- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031)) +- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096)) +- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109)) + +### Fixes + +- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033)) + +### Under the Hood + +- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966)) +- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010)) +- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073)) +- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074)) + +### Contributors +- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) diff --git a/.changes/unreleased/Dependencies-20231219-222211.yaml b/.changes/1.9.0/Dependencies-20231219-222211.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20231219-222211.yaml rename to .changes/1.9.0/Dependencies-20231219-222211.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024814.yaml b/.changes/1.9.0/Dependencies-20240419-024814.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024814.yaml rename to .changes/1.9.0/Dependencies-20240419-024814.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024818.yaml b/.changes/1.9.0/Dependencies-20240419-024818.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024818.yaml rename to .changes/1.9.0/Dependencies-20240419-024818.yaml diff --git a/.changes/unreleased/Dependencies-20240419-024820.yaml b/.changes/1.9.0/Dependencies-20240419-024820.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240419-024820.yaml rename to .changes/1.9.0/Dependencies-20240419-024820.yaml diff --git a/.changes/unreleased/Dependencies-20240503-224157.yaml b/.changes/1.9.0/Dependencies-20240503-224157.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240503-224157.yaml rename to .changes/1.9.0/Dependencies-20240503-224157.yaml diff --git a/.changes/unreleased/Dependencies-20240718-223145.yaml b/.changes/1.9.0/Dependencies-20240718-223145.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-223145.yaml rename to .changes/1.9.0/Dependencies-20240718-223145.yaml diff --git a/.changes/unreleased/Dependencies-20240718-223238.yaml b/.changes/1.9.0/Dependencies-20240718-223238.yaml similarity index 100% rename from .changes/unreleased/Dependencies-20240718-223238.yaml rename to .changes/1.9.0/Dependencies-20240718-223238.yaml diff --git a/.changes/unreleased/Features-20240430-185723.yaml b/.changes/1.9.0/Features-20240430-185723.yaml similarity index 100% rename from .changes/unreleased/Features-20240430-185723.yaml rename to .changes/1.9.0/Features-20240430-185723.yaml diff --git a/.changes/unreleased/Features-20240501-151904.yaml b/.changes/1.9.0/Features-20240501-151904.yaml similarity index 100% rename from .changes/unreleased/Features-20240501-151904.yaml rename to .changes/1.9.0/Features-20240501-151904.yaml diff --git a/.changes/unreleased/Features-20240903-161003.yaml b/.changes/1.9.0/Features-20240903-161003.yaml similarity index 100% rename from .changes/unreleased/Features-20240903-161003.yaml rename to .changes/1.9.0/Features-20240903-161003.yaml diff --git a/.changes/unreleased/Features-20240910-175846.yaml b/.changes/1.9.0/Features-20240910-175846.yaml similarity index 100% rename from .changes/unreleased/Features-20240910-175846.yaml rename to .changes/1.9.0/Features-20240910-175846.yaml diff --git a/.changes/unreleased/Features-20240925-125242.yaml b/.changes/1.9.0/Features-20240925-125242.yaml similarity index 100% rename from .changes/unreleased/Features-20240925-125242.yaml rename to .changes/1.9.0/Features-20240925-125242.yaml diff --git a/.changes/unreleased/Fixes-20240513-160121.yaml b/.changes/1.9.0/Fixes-20240513-160121.yaml similarity index 100% rename from .changes/unreleased/Fixes-20240513-160121.yaml rename to .changes/1.9.0/Fixes-20240513-160121.yaml diff --git a/.changes/unreleased/Under the Hood-20240612-195629.yaml b/.changes/1.9.0/Under the Hood-20240612-195629.yaml similarity index 100% rename from .changes/unreleased/Under the Hood-20240612-195629.yaml rename to .changes/1.9.0/Under the Hood-20240612-195629.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 36a3ea69a..7679e37bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,10 +5,41 @@ - "Breaking changes" listed under a version may require action from end users or external maintainers when upgrading to that version. - Do not edit this file directly. This file is auto-generated using [changie](https://github.com/miniscruff/changie). For details on how to document a change, see [the contributing guide](https://github.com/dbt-labs/dbt-spark/blob/main/CONTRIBUTING.md#adding-changelog-entry) +## dbt-spark 1.9.0-b1 - October 01, 2024 + +### Features + +- Add tests for cross-database `cast` macro ([#1028](https://github.com/dbt-labs/dbt-spark/issues/1028)) +- Cross-database `date` macro ([#1031](https://github.com/dbt-labs/dbt-spark/issues/1031)) +- Allow configuring snapshot column names ([#1096](https://github.com/dbt-labs/dbt-spark/issues/1096)) +- Support custom ODBC connection parameters via `connection_string_suffix` config ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- Add Microbatch Strategy to dbt-spark ([#1109](https://github.com/dbt-labs/dbt-spark/issues/1109)) + +### Fixes + +- Fix incremental python models error where Databricks could not find the temp table transaction logs ([#1033](https://github.com/dbt-labs/dbt-spark/issues/1033)) + +### Under the Hood + +- Lazy load agate to improve performance ([#1049](https://github.com/dbt-labs/dbt-spark/issues/1049)) + +### Dependencies + +- Update freezegun requirement from ~=1.3 to ~=1.4 ([#966](https://github.com/dbt-labs/dbt-spark/pull/966)) +- Bump actions/download-artifact from 3 to 4 ([#1010](https://github.com/dbt-labs/dbt-spark/pull/1010)) +- Bump actions/upload-artifact from 3 to 4 ([#1011](https://github.com/dbt-labs/dbt-spark/pull/1011)) +- Bump dbt-labs/actions from 1.1.0 to 1.1.1 ([#1012](https://github.com/dbt-labs/dbt-spark/pull/1012)) +- Update wheel requirement from ~=0.42 to ~=0.43 ([#1035](https://github.com/dbt-labs/dbt-spark/pull/1035)) +- Update pytest-xdist requirement from ~=3.5 to ~=3.6 ([#1073](https://github.com/dbt-labs/dbt-spark/pull/1073)) +- Loosen pin on pre-commit from 3.7.0 to 3.7 ([#1074](https://github.com/dbt-labs/dbt-spark/pull/1074)) + +### Contributors +- [@jpoley](https://github.com/jpoley) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) +- [@nilan3](https://github.com/nilan3) ([#1092](https://github.com/dbt-labs/dbt-spark/issues/1092)) + + ## Previous Releases For information on prior major and minor releases, see their changelogs: -- [1.8](https://github.com/dbt-labs/dbt-spark/blob/1.8.latest/CHANGELOG.md) -- [1.7](https://github.com/dbt-labs/dbt-spark/blob/1.7.latest/CHANGELOG.md) - [1.6](https://github.com/dbt-labs/dbt-spark/blob/1.6.latest/CHANGELOG.md) - [1.5](https://github.com/dbt-labs/dbt-spark/blob/1.5.latest/CHANGELOG.md) - [1.4](https://github.com/dbt-labs/dbt-spark/blob/1.4.latest/CHANGELOG.md) diff --git a/dbt/adapters/spark/__version__.py b/dbt/adapters/spark/__version__.py index 6698ed64c..a4077fff2 100644 --- a/dbt/adapters/spark/__version__.py +++ b/dbt/adapters/spark/__version__.py @@ -1 +1 @@ -version = "1.9.0a1" +version = "1.9.0b1" diff --git a/setup.py b/setup.py index cf8ff5691..406c181d5 100644 --- a/setup.py +++ b/setup.py @@ -41,7 +41,7 @@ def _get_plugin_version_dict(): package_name = "dbt-spark" -package_version = "1.9.0a1" +package_version = "1.9.0b1" description = """The Apache Spark adapter plugin for dbt""" odbc_extras = ["pyodbc~=5.1.0"] From 42d7f78ffa4820d64e7a7e7b84b6a4761ed52a4a Mon Sep 17 00:00:00 2001 From: leahwicz <60146280+leahwicz@users.noreply.github.com> Date: Thu, 17 Oct 2024 23:16:58 -0400 Subject: [PATCH 08/10] Isolating distribution testing (#1105) Co-authored-by: Mike Alfare <13974384+mikealfare@users.noreply.github.com> --- .../Under the Hood-20240911-192845.yaml | 6 ++++++ .github/workflows/main.yml | 17 +++++++---------- 2 files changed, 13 insertions(+), 10 deletions(-) create mode 100644 .changes/unreleased/Under the Hood-20240911-192845.yaml diff --git a/.changes/unreleased/Under the Hood-20240911-192845.yaml b/.changes/unreleased/Under the Hood-20240911-192845.yaml new file mode 100644 index 000000000..0c878f7d8 --- /dev/null +++ b/.changes/unreleased/Under the Hood-20240911-192845.yaml @@ -0,0 +1,6 @@ +kind: Under the Hood +body: Isolating distribution testing +time: 2024-09-11T19:28:45.653064-04:00 +custom: + Author: leahwicz + Issue: "1069" diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a4e2c7a45..9a9d66125 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -161,7 +161,7 @@ jobs: overwrite: true test-build: - name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} + name: verify packages / python ${{ matrix.python-version }} / ${{ matrix.os }} / ${{ matrix.dist-type }} if: needs.build.outputs.is_alpha == 0 @@ -174,6 +174,7 @@ jobs: matrix: os: [ubuntu-latest, macos-12, windows-latest] python-version: ["3.9", "3.10", "3.11", "3.12"] + dist-type: ["whl", "gz"] steps: - name: Set up Python ${{ matrix.python-version }} @@ -186,6 +187,7 @@ jobs: python -m pip install --user --upgrade pip python -m pip install --upgrade wheel python -m pip --version + - uses: actions/download-artifact@v4 with: name: dist @@ -194,15 +196,10 @@ jobs: - name: Show distributions run: ls -lh dist/ - - name: Install wheel distributions - run: | - find ./dist/*.whl -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check wheel distributions + - name: Install ${{ matrix.dist-type }} distributions run: | - python -c "import dbt.adapters.spark" - - name: Install source distributions - run: | - find ./dist/*.gz -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ - - name: Check source distributions + find ./dist/*.${{ matrix.dist-type }} -maxdepth 1 -type f | xargs python -m pip install --force-reinstall --find-links=dist/ + + - name: Check ${{ matrix.dist-type }} distributions run: | python -c "import dbt.adapters.spark" From 0a694bdd11a4b1b48daec8207d7e4713846c478e Mon Sep 17 00:00:00 2001 From: Mike Alfare <13974384+mikealfare@users.noreply.github.com> Date: Fri, 8 Nov 2024 16:17:08 -0500 Subject: [PATCH 09/10] move from macos-12 to macos-14 (#1136) --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 9a9d66125..287e5acb7 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -172,7 +172,7 @@ jobs: strategy: fail-fast: false matrix: - os: [ubuntu-latest, macos-12, windows-latest] + os: [ubuntu-latest, macos-14, windows-latest] python-version: ["3.9", "3.10", "3.11", "3.12"] dist-type: ["whl", "gz"] From d2d99672604f1533f7fb8d170098dd1b5c1993ca Mon Sep 17 00:00:00 2001 From: Colin Rogers <111200756+colin-rogers-dbt@users.noreply.github.com> Date: Fri, 8 Nov 2024 13:55:19 -0800 Subject: [PATCH 10/10] move github runner from macos-12 to macos-14 (#1135)