Merge main to feature (#6817)

merge main to feature/click-cli

.bumpversion.cfg

@@ -1,15 +1,21 @@
 [bumpversion]
 current_version = 1.5.0a1
-parse = (?P<major>\d+)
-	\.(?P<minor>\d+)
-	\.(?P<patch>\d+)
-	((?P<prekind>a|b|rc)
-	(?P<pre>\d+)  # pre-release version num
-	)(\.(?P<nightly>[a-z..0-9]+)
+
+# `parse` allows parsing the version into the parts we need to check.  There are some
+# unnamed groups and that's okay because they do not need to be audited.  If any part
+# of the version passed and does not match the regex, it will fail.
+# expected matches: `1.5.0`, `1.5.0a1`, `1.5.0a1.dev123457+nightly`
+# excepted failures: `1`, `1.5`, `1.5.2-a1`, `text1.5.0`
+parse = (?P<major>[\d]+) # major version number
+	\.(?P<minor>[\d]+) # minor version number
+	\.(?P<patch>[\d]+) # patch version number
+	(((?P<prekind>a|b|rc) # optional pre-release type
+	?(?P<num>[\d]+?)) # optional pre-release version number
+	\.?(?P<nightly>[a-z0-9]+\+[a-z]+)? # optional nightly release indicator
 	)?
 serialize =
-	{major}.{minor}.{patch}{prekind}{pre}.{nightly}
-	{major}.{minor}.{patch}{prekind}{pre}
+	{major}.{minor}.{patch}{prekind}{num}.{nightly}
+	{major}.{minor}.{patch}{prekind}{num}
 	{major}.{minor}.{patch}
 commit = False
 tag = False
@@ -23,7 +29,7 @@ values =
 	rc
 	final
 
-[bumpversion:part:pre]
+[bumpversion:part:num]
 first_value = 1
 
 [bumpversion:part:nightly]

.changes/unreleased/Features-20230126-154716.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Adjust makefile to have clearer instructions for CI env var changes.
+time: 2023-01-26T15:47:16.887327-08:00
+custom:
+  Author: versusfacit
+  Issue: "6689"

.changes/unreleased/Features-20230127-162812.yaml

@@ -0,0 +1,6 @@
+kind: Features
+body: Stand-alone Python module for PostgresColumn
+time: 2023-01-27T16:28:12.212427-08:00
+custom:
+  Author: nssalian
+  Issue: "6772"

.changes/unreleased/Fixes-20230201-154418.yaml

@@ -0,0 +1,6 @@
+kind: Fixes
+body: Remove pin on packaging and stop using it for prerelease comparisons
+time: 2023-02-01T15:44:18.279158-05:00
+custom:
+  Author: gshank
+  Issue: "6834"

.changes/unreleased/Under the Hood-20230126-164741.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: '[CT-1841] Convert custom target test to Pytest'
+time: 2023-01-26T16:47:41.198714-08:00
+custom:
+  Author: aranke
+  Issue: "6638"

.changes/unreleased/Under the Hood-20230203-143551.yaml

@@ -0,0 +1,6 @@
+kind: Under the Hood
+body: Moving simple_seed to adapter zone to help adapter test conversions
+time: 2023-02-03T14:35:51.481856-08:00
+custom:
+  Author: nssalian
+  Issue: CT-1959

.changie.yaml

@@ -97,22 +97,28 @@ footerFormat: |
       {{- /* we only want to include non-core team contributors */}}
       {{- if not (has $authorLower $core_team)}}
         {{- $changeList := splitList " " $change.Custom.Author }}
-          {{- /* Docs kind link back to dbt-docs instead of dbt-core issues */}}
+          {{- $IssueList := list }}
           {{- $changeLink := $change.Kind }}
           {{- if or (eq $change.Kind "Dependencies") (eq $change.Kind "Security") }}
-            {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $change.Custom.PR }}
-          {{- else if eq $change.Kind "Docs"}}
-            {{- $changeLink = "[dbt-docs/#nbr](https://github.com/dbt-labs/dbt-docs/issues/nbr)" | replace "nbr" $change.Custom.Issue }}
+            {{- $changes := splitList " " $change.Custom.PR }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/pull/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
           {{- else }}
-            {{- $changeLink = "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $change.Custom.Issue }}
+            {{- $changes := splitList " " $change.Custom.Issue }}
+            {{- range $issueNbr := $changes }}
+              {{- $changeLink := "[#nbr](https://github.com/dbt-labs/dbt-core/issues/nbr)" | replace "nbr" $issueNbr }}
+              {{- $IssueList = append $IssueList $changeLink  }}
+            {{- end -}}
           {{- end }}
           {{- /* check if this contributor has other changes associated with them already */}}
           {{- if hasKey $contributorDict $author }}
             {{- $contributionList := get $contributorDict $author }}
-            {{- $contributionList = append $contributionList $changeLink  }}
+            {{- $contributionList = concat $contributionList $IssueList  }}
             {{- $contributorDict := set $contributorDict $author $contributionList }}
           {{- else }}
-            {{- $contributionList := list $changeLink }}
+            {{- $contributionList := $IssueList }}
             {{- $contributorDict := set $contributorDict $author $contributionList }}
           {{- end }}
         {{- end}}

.github/workflows/release.yml

@@ -178,8 +178,8 @@ jobs:
       nightly_release: ${{ inputs.nightly_release }}
 
     secrets:
-      AWS_ACCESS_KEY_ID: ${{ secrets.PRODUCTION_AWS_ACCESS_KEY_ID }}
-      AWS_SECRET_ACCESS_KEY: ${{ secrets.PRODUCTION_AWS_SECRET_ACCESS_KEY }}
+      AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }}
 
   github-release:
     name: GitHub Release
@@ -212,7 +212,7 @@ jobs:
 
   slack-notification:
     name: Slack Notification
-    if: ${{ failure() }}
+    if: ${{ failure() && (!inputs.test_run || inputs.nightly_release) }}
 
     needs:
       [

.gitignore

@@ -51,6 +51,7 @@ coverage.xml
 *,cover
 .hypothesis/
 test.env
+makefile.test.env
 *.pytest_cache/
 
 

Makefile

@@ -6,18 +6,26 @@ ifeq ($(USE_DOCKER),true)
 	DOCKER_CMD := docker-compose run --rm test
 endif
 
-LOGS_DIR := ./logs
+#
+# To override CI_flags, create a file at this repo's root dir named `makefile.test.env`. Fill it
+# with any ENV_VAR overrides required by your test environment, e.g.
+#    DBT_TEST_USER_1=user
+#    LOG_DIR="dir with a space in it"
+#
+# Warn: Restrict each line to one variable only.
+#
+ifeq (./makefile.test.env,$(wildcard ./makefile.test.env))
+	include ./makefile.test.env
+endif
 
-# Optional flag to invoke tests using our CI env.
-# But we always want these active for structured
-# log testing.
 CI_FLAGS =\
-	DBT_TEST_USER_1=dbt_test_user_1\
-	DBT_TEST_USER_2=dbt_test_user_2\
-	DBT_TEST_USER_3=dbt_test_user_3\
-	RUSTFLAGS="-D warnings"\
-	LOG_DIR=./logs\
-	DBT_LOG_FORMAT=json
+	DBT_TEST_USER_1=$(if $(DBT_TEST_USER_1),$(DBT_TEST_USER_1),dbt_test_user_1)\
+	DBT_TEST_USER_2=$(if $(DBT_TEST_USER_2),$(DBT_TEST_USER_2),dbt_test_user_2)\
+	DBT_TEST_USER_3=$(if $(DBT_TEST_USER_3),$(DBT_TEST_USER_3),dbt_test_user_3)\
+	RUSTFLAGS=$(if $(RUSTFLAGS),$(RUSTFLAGS),"-D warnings")\
+	LOG_DIR=$(if $(LOG_DIR),$(LOG_DIR),./logs)\
+	DBT_LOG_FORMAT=$(if $(DBT_LOG_FORMAT),$(DBT_LOG_FORMAT),json)
+
 
 .PHONY: dev_req
 dev_req: ## Installs dbt-* packages in develop mode along with only development dependencies.
@@ -66,7 +74,7 @@ test: .env ## Runs unit tests with py and code checks against staged changes.
 .PHONY: integration
 integration: .env ## Runs postgres integration tests with py-integration
 	@\
-	$(if $(USE_CI_FLAGS), $(CI_FLAGS)) $(DOCKER_CMD) tox -e py-integration -- -nauto
+	$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto
 
 .PHONY: integration-fail-fast
 integration-fail-fast: .env ## Runs postgres integration tests with py-integration in "fail fast" mode.
@@ -76,9 +84,9 @@ integration-fail-fast: .env ## Runs postgres integration tests with py-integrati
 .PHONY: interop
 interop: clean
 	@\
-	mkdir $(LOGS_DIR) && \
+	mkdir $(LOG_DIR) && \
 	$(CI_FLAGS) $(DOCKER_CMD) tox -e py-integration -- -nauto && \
-	LOG_DIR=$(LOGS_DIR) cargo run --manifest-path test/interop/log_parsing/Cargo.toml
+	LOG_DIR=$(LOG_DIR) cargo run --manifest-path test/interop/log_parsing/Cargo.toml
 
 .PHONY: setup-db
 setup-db: ## Setup Postgres database with docker-compose for system testing.

core/dbt/config/project.py

@@ -378,8 +378,8 @@ def create_project(self, rendered: RenderComponents) -> "Project":
 
         flag_target_path = str(flags.TARGET_PATH) if flags.TARGET_PATH else None
         target_path: str = flag_or(flag_target_path, cfg.target_path, "target")
-
         log_path: str = str(flags.LOG_PATH)
+
         clean_targets: List[str] = value_or(cfg.clean_targets, [target_path])
         packages_install_path: str = value_or(cfg.packages_install_path, "dbt_packages")
         # in the default case we'll populate this once we know the adapter type

core/dbt/semver.py

@@ -1,10 +1,7 @@
 from dataclasses import dataclass
 import re
-import warnings
 from typing import List
 
-from packaging import version as packaging_version
-
 from dbt.exceptions import VersionsNotCompatibleError
 import dbt.utils
 
@@ -70,6 +67,11 @@ class VersionSpecification(dbtClassMixin):
 _VERSION_REGEX = re.compile(_VERSION_REGEX_PAT_STR, re.VERBOSE)
 
 
+def _cmp(a, b):
+    """Return negative if a<b, zero if a==b, positive if a>b."""
+    return (a > b) - (a < b)
+
+
 @dataclass
 class VersionSpecifier(VersionSpecification):
     def to_version_string(self, skip_matcher=False):
@@ -142,13 +144,19 @@ def compare(self, other):
                     return 1
                 if b is None:
                     return -1
-            # This suppresses the LegacyVersion deprecation warning
-            with warnings.catch_warnings():
-                warnings.simplefilter("ignore", category=DeprecationWarning)
-                if packaging_version.parse(a) > packaging_version.parse(b):
+
+                # Check the prerelease component only
+                prcmp = self._nat_cmp(a, b)
+                if prcmp != 0:  # either -1 or 1
+                    return prcmp
+                # else is equal and will fall through
+
+            else:  # major/minor/patch, should all be numbers
+                if a > b:
                     return 1
-                elif packaging_version.parse(a) < packaging_version.parse(b):
+                elif a < b:
                     return -1
+                # else is equal and will fall through
 
         equal = (
             self.matcher == Matchers.GREATER_THAN_OR_EQUAL
@@ -212,6 +220,29 @@ def is_upper_bound(self):
     def is_exact(self):
         return self.matcher == Matchers.EXACT
 
+    @classmethod
+    def _nat_cmp(cls, a, b):
+        def cmp_prerelease_tag(a, b):
+            if isinstance(a, int) and isinstance(b, int):
+                return _cmp(a, b)
+            elif isinstance(a, int):
+                return -1
+            elif isinstance(b, int):
+                return 1
+            else:
+                return _cmp(a, b)
+
+        a, b = a or "", b or ""
+        a_parts, b_parts = a.split("."), b.split(".")
+        a_parts = [int(x) if re.match(r"^\d+$", x) else x for x in a_parts]
+        b_parts = [int(x) if re.match(r"^\d+$", x) else x for x in b_parts]
+        for sub_a, sub_b in zip(a_parts, b_parts):
+            cmp_result = cmp_prerelease_tag(sub_a, sub_b)
+            if cmp_result != 0:
+                return cmp_result
+        else:
+            return _cmp(len(a), len(b))
+
 
 @dataclass
 class VersionRange:

core/dbt/tests/util.py

@@ -29,6 +29,8 @@
 #   rm_file
 #   write_file
 #   read_file
+#   mkdir
+#   rm_dir
 #   get_artifact
 #   update_config_file
 #   write_config_file
@@ -166,6 +168,22 @@ def read_file(*paths):
     return contents
 
 
+# To create a directory
+def mkdir(directory_path):
+    try:
+        os.makedirs(directory_path)
+    except FileExistsError:
+        raise FileExistsError(f"{directory_path} already exists.")
+
+
+# To remove a directory
+def rm_dir(directory_path):
+    try:
+        shutil.rmtree(directory_path)
+    except FileNotFoundError:
+        raise FileNotFoundError(f"{directory_path} does not exist.")
+
+
 # Get an artifact (usually from the target directory) such as
 # manifest.json or catalog.json to use in a test
 def get_artifact(*paths):

core/setup.py

@@ -58,7 +58,7 @@
         "minimal-snowplow-tracker==0.0.2",
         "networkx>=2.3,<2.8.1;python_version<'3.8'",
         "networkx>=2.3,<3;python_version>='3.8'",
-        "packaging>=20.9,<22.0",
+        "packaging>20.9",
         "sqlparse>=0.2.3,<0.5",
         "dbt-extractor~=0.4.1",
         "typing-extensions>=3.7.4",

plugins/postgres/dbt/adapters/postgres/__init__.py

@@ -1,7 +1,7 @@
 # these are mostly just exports, #noqa them so flake8 will be happy
 from dbt.adapters.postgres.connections import PostgresConnectionManager  # noqa
 from dbt.adapters.postgres.connections import PostgresCredentials
-from dbt.adapters.postgres.relation import PostgresColumn  # noqa
+from dbt.adapters.postgres.column import PostgresColumn  # noqa
 from dbt.adapters.postgres.relation import PostgresRelation  # noqa: F401
 from dbt.adapters.postgres.impl import PostgresAdapter
 

plugins/postgres/dbt/adapters/postgres/column.py

@@ -0,0 +1,12 @@
+from dbt.adapters.base import Column
+
+
+class PostgresColumn(Column):
+    @property
+    def data_type(self):
+        # on postgres, do not convert 'text' or 'varchar' to 'varchar()'
+        if self.dtype.lower() == "text" or (
+            self.dtype.lower() == "character varying" and self.char_size is None
+        ):
+            return self.dtype
+        return super().data_type

plugins/postgres/dbt/adapters/postgres/impl.py

@@ -5,7 +5,7 @@
 from dbt.adapters.base.impl import AdapterConfig
 from dbt.adapters.sql import SQLAdapter
 from dbt.adapters.postgres import PostgresConnectionManager
-from dbt.adapters.postgres import PostgresColumn
+from dbt.adapters.postgres.column import PostgresColumn
 from dbt.adapters.postgres import PostgresRelation
 from dbt.dataclass_schema import dbtClassMixin, ValidationError
 from dbt.exceptions import (

plugins/postgres/dbt/adapters/postgres/relation.py

@@ -1,4 +1,3 @@
-from dbt.adapters.base import Column
 from dataclasses import dataclass
 from dbt.adapters.base.relation import BaseRelation
 from dbt.exceptions import DbtRuntimeError
@@ -21,14 +20,3 @@ def __post_init__(self):
 
     def relation_max_name_length(self):
         return 63
-
-
-class PostgresColumn(Column):
-    @property
-    def data_type(self):
-        # on postgres, do not convert 'text' or 'varchar' to 'varchar()'
-        if self.dtype.lower() == "text" or (
-            self.dtype.lower() == "character varying" and self.char_size is None
-        ):
-            return self.dtype
-        return super().data_type

test/unit/test_config.py

@@ -28,7 +28,6 @@
 from .utils import normalize
 
 INITIAL_ROOT = os.getcwd()
-# Skip due to interface for config updated
 
 
 @contextmanager