Merge branch 'main' into fix/multiline-query-comments-result

.github/scripts/integration-test-matrix.js

@@ -1,6 +1,6 @@
 module.exports = ({ context }) => {
  const defaultPythonVersion = "3.8";
- const supportedPythonVersions = ["3.7", "3.8", "3.9"];
+ const supportedPythonVersions = ["3.7", "3.8", "3.9", "3.10"];
  const supportedAdapters = ["snowflake"];
 
  // if PR, generate matrix based on files changed and PR labels

.github/workflows/main.yml

@@ -70,7 +70,7 @@ jobs:
  strategy:
  fail-fast: false
  matrix:
- python-version: [3.7, 3.8] # TODO: support unit testing for python 3.9 (https://github.com/dbt-labs/dbt/issues/3689)
+ python-version: ['3.7', '3.8', '3.9', '3.10']
 
  env:
  TOXENV: "unit"
@@ -172,7 +172,7 @@ jobs:
  fail-fast: false
  matrix:
  os: [ubuntu-latest, macos-latest, windows-latest]
- python-version: [3.7, 3.8, 3.9]
+ python-version: ['3.7', '3.8', '3.9', '3.10']
 
  steps:
  - name: Set up Python ${{ matrix.python-version }}

.pre-commit-config.yaml

@@ -42,7 +42,7 @@ repos:
  alias: flake8-check
  stages: [manual]
 - repo: https://github.com/pre-commit/mirrors-mypy
- rev: v0.782
+ rev: v0.942
  hooks:
  - id: mypy
  # N.B.: Mypy is... a bit fragile.

CHANGELOG.md

@@ -1,9 +1,16 @@
 ## dbt-snowflake 1.2.0 (tbd)
+- Add set_query_tag and unset_query_tag to the dbt macro namespace ([#133](https://github.com/dbt-labs/dbt-snowflake/issues/133), [#132](https://github.com/dbt-labs/dbt-snowflake/pull/132))
 
 ### Under the hood
 - Add precommits for this repo ([#107](https://github.com/dbt-labs/dbt-snowflake/pull/107))
 - Cleanup redundant precommit hook command ([#145](https://github.com/dbt-labs/dbt-snowflake/pull/145))
 - File rename to match reference to core ([#152](https://github.com/dbt-labs/dbt-snowflake/pull/152))
+- Bump cryptography restraint upper bound to <37.0.0 ([#171](https://github.com/dbt-labs/dbt-snowflake/pull/171))
+- migrate oauth refresh script from core ([#175](https://github.com/dbt-labs/dbt-snowflake/pull/175))
+
+### Contributors
+- [@hhobson](https://github.com/hhobson) ([#171](https://github.com/dbt-labs/dbt-snowflake/pull/171))
+- [@robscriva](https://github.com/robscriva) ([#132](https://github.com/dbt-labs/dbt-snowflake/pull/132))
 
 ## dbt-snowflake 1.1.0b1 (March 23, 2022)
 

CONTRIBUTING.MD

@@ -79,7 +79,7 @@ WARNING: The parameters in your `test.env` file must link to a valid Snowflake a
 There are a few methods for running tests locally.
 
 #### `tox`
-`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.7, Python 3.8, Python 3.9, and `flake8` in parallel. Run `tox -e py37` to invoke tests on Python version 3.7 only (use py37, py38, or py39). Tox recipes are found in `tox.ini`.
+`tox` automatically runs unit tests against several Python versions using its own virtualenvs. Run `tox -p` to run unit tests for Python 3.7, Python 3.8, Python 3.9, Python 3.10, and `flake8` in parallel. Run `tox -e py37` to invoke tests on Python version 3.7 only (use py37, py38, py39, or py310). Tox recipes are found in `tox.ini`.
 
 #### `pytest`
 You may run a specific test or group of tests using `pytest` directly. Activate a Python virtualenv active with dev dependencies installed. Then, run tests like so:

dbt/__init__.py

@@ -0,0 +1,3 @@
+from pkgutil import extend_path
+
+__path__ = extend_path(__path__, __name__)

dbt/include/snowflake/macros/adapters.sql

@@ -193,6 +193,11 @@
 
 
 {% macro set_query_tag() -%}
+ {{ return(adapter.dispatch('set_query_tag', 'dbt')()) }}
+{% endmacro %}
+
+
+{% macro snowflake__set_query_tag() -%}
  {% set new_query_tag = config.get('query_tag') %}
  {% if new_query_tag %}
  {% set original_query_tag = get_current_query_tag() %}
@@ -203,7 +208,13 @@
  {{ return(none)}}
 {% endmacro %}
 
+
 {% macro unset_query_tag(original_query_tag) -%}
+ {{ return(adapter.dispatch('unset_query_tag', 'dbt')(original_query_tag)) }}
+{% endmacro %}
+
+
+{% macro snowflake__unset_query_tag(original_query_tag) -%}
  {% set new_query_tag = config.get('query_tag') %}
  {% if new_query_tag %}
  {% if original_query_tag %}

dev-requirements.txt

@@ -3,14 +3,14 @@
 git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-core&subdirectory=core
 git+https://github.com/dbt-labs/dbt-core.git#egg=dbt-tests-adapter&subdirectory=tests/adapter
 
-black==21.12b0
+black==22.3.0
 click~=8.0.4
 bumpversion
 flake8
 flaky
 freezegun==0.3.12
 ipdb
-mypy==0.782
+mypy==0.942
 pip-tools
 pre-commit
 pytest
@@ -20,5 +20,7 @@ pytest-csv
 pytest-xdist
 pytz
 tox>=3.13
+types-pytz
+types-requests
 twine
 wheel

mypy.ini

@@ -0,0 +1,2 @@
+[mypy]
+namespace_packages = True

scripts/werkzeug-refresh-token.py

@@ -0,0 +1,144 @@
+import argparse
+import json
+import secrets
+import textwrap
+from base64 import b64encode
+
+import requests
+from werkzeug.utils import redirect
+from werkzeug.middleware.dispatcher import DispatcherMiddleware
+from werkzeug.wrappers import Request, Response
+from werkzeug.serving import run_simple
+from urllib.parse import urlencode
+
+
+def _make_rfp_claim_value():
+ # from https://tools.ietf.org/id/draft-bradley-oauth-jwt-encoded-state-08.html#rfc.section.4 # noqa
+ # we can do whatever we want really, so just token.urlsafe?
+ return secrets.token_urlsafe(112)
+
+
+def _make_response(client_id, client_secret, refresh_token):
+ return Response(
+ textwrap.dedent(
+ f'''\
+ SNOWFLAKE_TEST_OAUTH_REFRESH_TOKEN="{refresh_token}"
+ SNOWFLAKE_TEST_OAUTH_CLIENT_ID="{client_id}"
+ SNOWFLAKE_TEST_OAUTH_CLIENT_SECRET="{client_secret}"'''
+ )
+ )
+
+
+class TokenManager:
+ def __init__(self, account_name, client_id, client_secret):
+ self.account_name = account_name
+ self.client_id = client_id
+ self.client_secret = client_secret
+ self.token = None
+ self.rfp_claim = _make_rfp_claim_value()
+ self.port = 8080
+
+ @property
+ def account_url(self):
+ return f"https://{self.account_name}.snowflakecomputing.com"
+
+ @property
+ def auth_url(self):
+ return f"{self.account_url}/oauth/authorize"
+
+ @property
+ def token_url(self):
+ return f"{self.account_url}/oauth/token-request"
+
+ @property
+ def redirect_uri(self):
+ return f"http://localhost:{self.port}"
+
+ @property
+ def headers(self):
+ auth = f"{self.client_id}:{self.client_secret}".encode("ascii")
+ encoded_auth = b64encode(auth).decode("ascii")
+ return {
+ "Authorization": f"Basic {encoded_auth}",
+ "Content-type": "application/x-www-form-urlencoded; charset=utf-8",
+ }
+
+ def _code_to_token(self, code):
+ data = {
+ "grant_type": "authorization_code",
+ "code": code,
+ "redirect_uri": self.redirect_uri,
+ }
+ # data = urlencode(data)
+ resp = requests.post(
+ url=self.token_url,
+ headers=self.headers,
+ data=data,
+ )
+ try:
+ refresh_token = resp.json()["refresh_token"]
+ except KeyError:
+ print(resp.json())
+ raise
+ return refresh_token
+
+ @Request.application
+ def auth(self, request):
+ code = request.args.get("code")
+ if code:
+ # we got 303'ed here with a code
+ state_received = request.args.get("state")
+ if state_received != self.rfp_claim:
+ return Response("Invalid RFP claim: MITM?", status=401)
+ refresh_token = self._code_to_token(code)
+ return _make_response(
+ self.client_id,
+ self.client_secret,
+ refresh_token,
+ )
+ else:
+ return redirect("/login")
+
+ @Request.application
+ def login(self, request):
+ # take the auth URL and add the query string to it
+ query = {
+ "response_type": "code",
+ "client_id": self.client_id,
+ "redirect_uri": self.redirect_uri,
+ "state": self.rfp_claim,
+ }
+ query = urlencode(query)
+ return redirect(f"{self.auth_url}?{query}")
+
+
+def parse_args():
+ parser = argparse.ArgumentParser()
+ parser.add_argument("account_name", help="The account name")
+ parser.add_argument("json_blob", help="The json auth blob")
+
+ return parser.parse_args()
+
+
+def main():
+ args = parse_args()
+ data = json.loads(args.json_blob)
+ client_id = data["OAUTH_CLIENT_ID"]
+ client_secret = data["OAUTH_CLIENT_SECRET"]
+ token_manager = TokenManager(
+ account_name=args.account_name,
+ client_id=client_id,
+ client_secret=client_secret,
+ )
+ app = DispatcherMiddleware(
+ token_manager.auth,
+ {
+ "/login": token_manager.login,
+ },
+ )
+
+ run_simple("localhost", token_manager.port, app)
+
+
+if __name__ == "__main__":
+ main()

setup.py

@@ -69,7 +69,7 @@ def _get_dbt_core_version():
  "dbt-core~={}".format(dbt_core_version),
  "snowflake-connector-python[secure-local-storage]>=2.4.1,<2.8.0",
  "requests<3.0.0",
- "cryptography>=3.2,<4",
+ "cryptography>=3.2,<37.0.0",
  ],
  zip_safe=False,
  classifiers=[
@@ -81,6 +81,7 @@ def _get_dbt_core_version():
  "Programming Language :: Python :: 3.7",
  "Programming Language :: Python :: 3.8",
  "Programming Language :: Python :: 3.9",
+ "Programming Language :: Python :: 3.10",
  ],
  python_requires=">=3.7",
 )

tests/integration/base.py

@@ -7,6 +7,7 @@
 import tempfile
 import traceback
 import unittest
+import warnings
 from contextlib import contextmanager
 from datetime import datetime
 from functools import wraps
@@ -251,6 +252,13 @@ def _generate_test_root_dir(self):
  return normalize(tempfile.mkdtemp(prefix='dbt-int-test-'))
 
  def setUp(self):
+ # Logbook warnings are ignored so we don't have to fork logbook to support python 3.10. 
+ # This _only_ works for tests in `test/integration`.
+ warnings.filterwarnings(
+ "ignore",
+ category=DeprecationWarning,
+ module="logbook"
+ )
  self.dbt_core_install_root = os.path.dirname(dbt.__file__)
  log_manager.reset_handlers()
  self.initial_dir = INITIAL_ROOT

tox.ini

@@ -1,8 +1,8 @@
 [tox]
 skipsdist = True
-envlist = py37,py38,py39
+envlist = py37,py38,py39,py310
 
-[testenv:{unit,py37,py38,py39,py}]
+[testenv:{unit,py37,py38,py39,py310,py}]
 description = unit testing
 skip_install = true
 passenv = DBT_* PYTEST_ADDOPTS
@@ -11,7 +11,7 @@ deps =
  -rdev-requirements.txt
  -e.
 
-[testenv:{integration,py37,py38,py39,py}-{snowflake}]
+[testenv:{integration,py37,py38,py39,py310,py}-{snowflake}]
 description = adapter plugin integration testing
 skip_install = true
 passenv = DBT_* SNOWFLAKE_TEST_* PYTEST_ADDOPTS