From 5bc8c89cf9ba572f57ccb58cbbdc63815ac81b75 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 15:52:26 -0700 Subject: [PATCH 01/20] update sslmode change --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 358ca5c689b..9656f7c7c48 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -132,6 +132,9 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes +### `sslmode` change + + ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). From a1a06d79a35e0b1102a68887caca80408983ce0f Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:14:35 -0700 Subject: [PATCH 02/20] Update redshift-setup.md --- .../docs/docs/core/connect-data-platform/redshift-setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 9656f7c7c48..50d6b297bb6 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -69,7 +69,7 @@ company-name: keepalives_idle: 240 # default 240 seconds connect_timeout: 10 # default 10 seconds # search_path: public # optional, not recommended - sslmode: [optional, set the sslmode used to connect to the database (in case this parameter is set, will look for ca in ~/.postgresql/root.crt)] + sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. Accepted inputs are "disable", "prefer", "allow", "require", "verify-ca", "verify-full". ra3_node: true # enables cross-database sources region: [optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com)] ``` @@ -117,7 +117,7 @@ my-redshift-db: connect_timeout: 10 # default 10 seconds [retries](#retries): 1 # default 1 retry on error/timeout when opening connections # search_path: public # optional, but not recommended - sslmode: [optional, set the sslmode used to connect to the database (in case this parameter is set, will look for ca in ~/.postgresql/root.crt)] + sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect and look for ca in ~/.postgresql/root.crt] ra3_node: true # enables cross-database sources ``` From 9fa40164daefea97b39e476fbc65eae4e307ec6f Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:39:53 -0700 Subject: [PATCH 03/20] Update redshift-setup.md --- .../connect-data-platform/redshift-setup.md | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 50d6b297bb6..6e2506b1837 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -69,7 +69,7 @@ company-name: keepalives_idle: 240 # default 240 seconds connect_timeout: 10 # default 10 seconds # search_path: public # optional, not recommended - sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. Accepted inputs are "disable", "prefer", "allow", "require", "verify-ca", "verify-full". + sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect.] ra3_node: true # enables cross-database sources region: [optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com)] ``` @@ -117,7 +117,7 @@ my-redshift-db: connect_timeout: 10 # default 10 seconds [retries](#retries): 1 # default 1 retry on error/timeout when opening connections # search_path: public # optional, but not recommended - sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect and look for ca in ~/.postgresql/root.crt] + sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect.] ra3_node: true # enables cross-database sources ``` @@ -133,7 +133,20 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes ### `sslmode` change +Prior to dbt-redshift 1.5, `psycopg2` was used as the driver.`psycopg2` accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). In dbt-redshift 1.5, `redshift_connector` is used and `redshift_connector` accepted inputs are "verify-ca", and "verify-full", see redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. +Table below details accepted `sslmode` parameters and how connection will be made according to each option: + +accepted `sslmode` parameter | Expected behavior +-- | -- +disable | Connection will be made without using ssl +allow | Connection will be made using verify-ca +prefer | Connection will be made using verify-ca +require | Connection will be made using verify-ca +verify-ca | Connection will be made using verify-ca +verify-full | Connection will be made using verify-full + +For more details on changes of sslmode, our design choices and reasoning, please refer to [PR#439](https://github.com/dbt-labs/dbt-redshift/pull/439). ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). From 263ba686e6a66789ef26508c696e00008a4c6198 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:43:44 -0700 Subject: [PATCH 04/20] Update redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 6e2506b1837..30d4c861971 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -133,7 +133,7 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes ### `sslmode` change -Prior to dbt-redshift 1.5, `psycopg2` was used as the driver.`psycopg2` accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). In dbt-redshift 1.5, `redshift_connector` is used and `redshift_connector` accepted inputs are "verify-ca", and "verify-full", see redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. +Prior to dbt-redshift 1.5, `psycopg2` was used as the driver.`psycopg2` accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). In dbt-redshift 1.5, `redshift_connector` is used and `redshift_connector` accepted inputs are "verify-ca", and "verify-full", according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. Table below details accepted `sslmode` parameters and how connection will be made according to each option: From 8400a8b3d019fa75e47837a94b017a17ac61c23e Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 16:50:20 -0700 Subject: [PATCH 05/20] Update redshift-setup.md --- .../docs/docs/core/connect-data-platform/redshift-setup.md | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 30d4c861971..b4c1978cdde 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -133,7 +133,8 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes ### `sslmode` change -Prior to dbt-redshift 1.5, `psycopg2` was used as the driver.`psycopg2` accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). In dbt-redshift 1.5, `redshift_connector` is used and `redshift_connector` accepted inputs are "verify-ca", and "verify-full", according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. +Prior to dbt-redshift 1.5, psycopg2 was used as the driver.psycopg2 accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). +In dbt-redshift 1.5, redshift_connector is used and redshift_connector accepted inputs are "verify-ca", and "verify-full", according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. Table below details accepted `sslmode` parameters and how connection will be made according to each option: @@ -146,7 +147,7 @@ require | Connection will be made using verify-ca verify-ca | Connection will be made using verify-ca verify-full | Connection will be made using verify-full -For more details on changes of sslmode, our design choices and reasoning, please refer to [PR#439](https://github.com/dbt-labs/dbt-redshift/pull/439). +For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). From 29572f920ec6fd365fc1aec650b9ea9bdb7b7719 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Thu, 1 Jun 2023 17:09:08 -0700 Subject: [PATCH 06/20] Update redshift-setup.md --- .../connect-data-platform/redshift-setup.md | 27 ++++++++++++------- 1 file changed, 17 insertions(+), 10 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index b4c1978cdde..c1d47123bab 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -133,19 +133,26 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes ### `sslmode` change -Prior to dbt-redshift 1.5, psycopg2 was used as the driver.psycopg2 accepts "disable", "prefer", "allow", "require", "verify-ca", "verify-full" as valid inputs, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). -In dbt-redshift 1.5, redshift_connector is used and redshift_connector accepted inputs are "verify-ca", and "verify-full", according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). For backwards compatibility, dbt-redshift now extended support for all valid inputs for sslmode in psycopg2. +Prior to dbt-redshift 1.5, `psycopg2` was used as the driver. `psycopg2` accepts `disable`, `prefer`, `allow`, `require`, `verify-ca`, `verify-full` as valid inputs of `sslmode`, and does not have a `ssl` paremeter, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). + +In dbt-redshift 1.5, we switched to using `redshift_connector`, which accepts `verify-ca`, and `verify-full` as valid `sslmode` inputs, and has a `ssl` parameter of `True` or `False`, according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). + +For backwards compatibility, dbt-redshift now supports for valid inputs for `sslmode` in `psycopg2`. We've added conversion logic mapping each of `psycopg2`'s accepted `sslmode` values to the corresponding `ssl` and `sslmode` parameters in `redshift_connector`. Table below details accepted `sslmode` parameters and how connection will be made according to each option: -accepted `sslmode` parameter | Expected behavior --- | -- -disable | Connection will be made without using ssl -allow | Connection will be made using verify-ca -prefer | Connection will be made using verify-ca -require | Connection will be made using verify-ca -verify-ca | Connection will be made using verify-ca -verify-full | Connection will be made using verify-full + + +`sslmode` parameter | Expected behavior in dbt-redshift | Actions behind the scenes +-- | -- | -- +disable | Connection will be made without using ssl | Set `ssl` = False +allow | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +prefer | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full + + For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). From b0b17726603c14711043b1a058a67d74c4a8a0a9 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 09:05:45 -0700 Subject: [PATCH 07/20] Update redshift-setup.md --- .../connect-data-platform/redshift-setup.md | 24 +++++++++++-------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index c1d47123bab..1307357e1ce 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -67,11 +67,12 @@ company-name: schema: analytics threads: 4 keepalives_idle: 240 # default 240 seconds - connect_timeout: 10 # default 10 seconds + connect_timeout: None # optional, number of seconds before connection times out # search_path: public # optional, not recommended - sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect.] + sslmode: prefer # optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. ra3_node: true # enables cross-database sources - region: [optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com)] + autocommit: true # enables autocommit after each statement + region: # optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com) ``` @@ -114,11 +115,14 @@ my-redshift-db: schema: analytics threads: 4 [keepalives_idle](#keepalives_idle): 240 # default 240 seconds - connect_timeout: 10 # default 10 seconds + connect_timeout: None # optional, number of seconds before connection times out [retries](#retries): 1 # default 1 retry on error/timeout when opening connections # search_path: public # optional, but not recommended - sslmode: prefer # [optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect.] + sslmode: prefer # optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. ra3_node: true # enables cross-database sources + autocommit: true # optional, enables autocommit after each statement + region: # optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com) + ``` @@ -146,11 +150,11 @@ Table below details accepted `sslmode` parameters and how connection will be mad `sslmode` parameter | Expected behavior in dbt-redshift | Actions behind the scenes -- | -- | -- disable | Connection will be made without using ssl | Set `ssl` = False -allow | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca -prefer | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca -require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca -verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca -verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full +allow | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +prefer | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca +verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full From 2daee2cad36d4194bb556f4df49dd89d8c95f83e Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 11:59:59 -0700 Subject: [PATCH 08/20] Update redshift-setup.md --- .../connect-data-platform/redshift-setup.md | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 1307357e1ce..3038c4fb7e3 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -160,6 +160,36 @@ verify-full | Connection will be made using verify-full | Set `ssl` = True & `s For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). +### `autocommit` parameter + +The[ autocommit mode](https://www.psycopg.org/docs/connection.html#connection.autocommit) is useful to execute commands that run outside a transaction. Connection objects used in Python must have `autocommit = True` to run operations such as `CREATE DATABASE`, and `VACUUM`. `autocommit` is off by default in `redshift_connector`, but we've changed this default to `True` to ensure certain macros run successfully in your dbt project. + +If desired, you can define a seperate target with `autocommit=True` as such: + + + +```yaml +profile-to-my-RS-target: + target: dev + outputs: + dev: + type: redshift + ... + autocommit: False + + + profile-to-my-RS-target-with-autocommit-enabled: + target: dev + outputs: + dev: + type: redshift + ... + autocommit: True + ``` + + +To run certain macros with autocommit, load the autocommit profile with the `--profile` flag. For more context, please refer to this [PR](https://github.com/dbt-labs/dbt-redshift/pull/475/files). + ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). From 7895688dec45e464a06961d6c3477d1a05236769 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:11:41 -0700 Subject: [PATCH 09/20] Update redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 3038c4fb7e3..95df76c19f3 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -156,7 +156,7 @@ require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full - +When connection is made using `verify-ca`, will look for CA certificate in ~/redshift-ca-bundle.crt. For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). From be1b7501dfd861ea603d06d57890cc46f19f10e8 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:13:39 -0700 Subject: [PATCH 10/20] Update redshift-setup.md --- .../docs/docs/core/connect-data-platform/redshift-setup.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 95df76c19f3..0c6260760b2 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -156,7 +156,7 @@ require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full -When connection is made using `verify-ca`, will look for CA certificate in ~/redshift-ca-bundle.crt. +When connection is made using `verify-ca`, will look for CA certificate in `~/redshift-ca-bundle.crt`. For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). @@ -188,7 +188,7 @@ profile-to-my-RS-target: ``` -To run certain macros with autocommit, load the autocommit profile with the `--profile` flag. For more context, please refer to this [PR](https://github.com/dbt-labs/dbt-redshift/pull/475/files). +To run certain macros with autocommit, load the profile with autocommit using the `--profile` flag. For more context, please refer to this [PR](https://github.com/dbt-labs/dbt-redshift/pull/475/files). ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). From be8add7472c0eb4f4ceb91dd0e3c76c6f1eabd4c Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:33:38 -0700 Subject: [PATCH 11/20] remove iam_duration_seconds, keepalives_idle, search_path parameters, added role parameter in profiles.yml --- .../docs/docs/core/connect-data-platform/redshift-setup.md | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 0c6260760b2..a8c83df3c1e 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -66,10 +66,10 @@ company-name: dbname: analytics schema: analytics threads: 4 - keepalives_idle: 240 # default 240 seconds connect_timeout: None # optional, number of seconds before connection times out # search_path: public # optional, not recommended sslmode: prefer # optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. + role: # optional ra3_node: true # enables cross-database sources autocommit: true # enables autocommit after each statement region: # optional, if not provided, will be determined from host (e.g. host.123.us-east-1.redshift-serverless.amazonaws.com) @@ -105,7 +105,6 @@ my-redshift-db: host: hostname.region.redshift.amazonaws.com user: alice iam_profile: data_engineer # optional - iam_duration_seconds: 900 # optional autocreate: true # optional db_groups: ['ANALYSTS'] # optional @@ -114,10 +113,9 @@ my-redshift-db: dbname: analytics schema: analytics threads: 4 - [keepalives_idle](#keepalives_idle): 240 # default 240 seconds connect_timeout: None # optional, number of seconds before connection times out [retries](#retries): 1 # default 1 retry on error/timeout when opening connections - # search_path: public # optional, but not recommended + role: # optional sslmode: prefer # optional, set the sslmode to connect to the database. Default prefer, which will use 'verify-ca' to connect. ra3_node: true # enables cross-database sources autocommit: true # optional, enables autocommit after each statement From baa2c3535ef4406959b5cb27b9a590bcb5a9bf2d Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 12:35:21 -0700 Subject: [PATCH 12/20] Update redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index a8c83df3c1e..19fffe9984a 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -191,7 +191,7 @@ To run certain macros with autocommit, load the profile with autocommit using th ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). -### `keepalives_idle` +### `keepalives_idle` (deprecated in `dbt-redshift` 1.5, only applicable for `psycopg2` driver) If the database closes its connection while dbt is waiting for data, you may see the error `SSL SYSCALL error: EOF detected`. Lowering the [`keepalives_idle` value](https://www.postgresql.org/docs/9.3/libpq-connect.html) may prevent this, because the server will send a ping to keep the connection active more frequently. [dbt's default setting](https://github.com/dbt-labs/dbt-redshift/blob/main/dbt/adapters/redshift/connections.py#L51) is 240 (seconds), but can be configured lower (perhaps 120 or 60), at the cost of a chattier network connection. From d4bfd6902ad5375be87e110789b8dad059e456da Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Fri, 2 Jun 2023 13:48:54 -0700 Subject: [PATCH 13/20] Update redshift-setup.md --- .../docs/core/connect-data-platform/redshift-setup.md | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 19fffe9984a..4c7211105ab 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -144,7 +144,6 @@ For backwards compatibility, dbt-redshift now supports for valid inputs for `ssl Table below details accepted `sslmode` parameters and how connection will be made according to each option: - `sslmode` parameter | Expected behavior in dbt-redshift | Actions behind the scenes -- | -- | -- disable | Connection will be made without using ssl | Set `ssl` = False @@ -188,13 +187,17 @@ profile-to-my-RS-target: To run certain macros with autocommit, load the profile with autocommit using the `--profile` flag. For more context, please refer to this [PR](https://github.com/dbt-labs/dbt-redshift/pull/475/files). + +### Deprecated `profile` parameters in 1.5 + +- `iam_duration_seconds` + +- `keepalives_idle` + ### `sort` and `dist` keys Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs). -### `keepalives_idle` (deprecated in `dbt-redshift` 1.5, only applicable for `psycopg2` driver) -If the database closes its connection while dbt is waiting for data, you may see the error `SSL SYSCALL error: EOF detected`. Lowering the [`keepalives_idle` value](https://www.postgresql.org/docs/9.3/libpq-connect.html) may prevent this, because the server will send a ping to keep the connection active more frequently. -[dbt's default setting](https://github.com/dbt-labs/dbt-redshift/blob/main/dbt/adapters/redshift/connections.py#L51) is 240 (seconds), but can be configured lower (perhaps 120 or 60), at the cost of a chattier network connection. From a43adde93e546fff544fa120501c5c7fe44f8c00 Mon Sep 17 00:00:00 2001 From: Jessie Chen <121250701+jiezhen-chen@users.noreply.github.com> Date: Tue, 6 Jun 2023 09:02:15 -0700 Subject: [PATCH 14/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md Co-authored-by: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 4c7211105ab..0d7b97e6c13 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -161,7 +161,7 @@ For more details on changes of sslmode, our design choices and reasoning, please The[ autocommit mode](https://www.psycopg.org/docs/connection.html#connection.autocommit) is useful to execute commands that run outside a transaction. Connection objects used in Python must have `autocommit = True` to run operations such as `CREATE DATABASE`, and `VACUUM`. `autocommit` is off by default in `redshift_connector`, but we've changed this default to `True` to ensure certain macros run successfully in your dbt project. -If desired, you can define a seperate target with `autocommit=True` as such: +If desired, you can define a separate target with `autocommit=True` as such: From 10a58cbfe51bceb6a2d399a64f3f445d16833b60 Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:35:20 +0100 Subject: [PATCH 15/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 0d7b97e6c13..422cf7a8461 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -135,7 +135,7 @@ The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` ## Redshift notes ### `sslmode` change -Prior to dbt-redshift 1.5, `psycopg2` was used as the driver. `psycopg2` accepts `disable`, `prefer`, `allow`, `require`, `verify-ca`, `verify-full` as valid inputs of `sslmode`, and does not have a `ssl` paremeter, as indicated in postgresql [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). +Before to dbt-redshift 1.5, `psycopg2` was used as the driver. `psycopg2` accepts `disable`, `prefer`, `allow`, `require`, `verify-ca`, `verify-full` as valid inputs of `sslmode`, and does not have an `ssl` parameter, as indicated in PostgreSQL [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). In dbt-redshift 1.5, we switched to using `redshift_connector`, which accepts `verify-ca`, and `verify-full` as valid `sslmode` inputs, and has a `ssl` parameter of `True` or `False`, according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). From b77c58e88e5cc355baa6731dc4df4336b126ce48 Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:35:58 +0100 Subject: [PATCH 16/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 422cf7a8461..085700d7072 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -139,7 +139,7 @@ Before to dbt-redshift 1.5, `psycopg2` was used as the driver. `psycopg2` accept In dbt-redshift 1.5, we switched to using `redshift_connector`, which accepts `verify-ca`, and `verify-full` as valid `sslmode` inputs, and has a `ssl` parameter of `True` or `False`, according to redshift [doc](https://docs.aws.amazon.com/redshift/latest/mgmt/python-configuration-options.html#:~:text=parameter%20is%20optional.-,sslmode,-Default%20value%20%E2%80%93%20verify). -For backwards compatibility, dbt-redshift now supports for valid inputs for `sslmode` in `psycopg2`. We've added conversion logic mapping each of `psycopg2`'s accepted `sslmode` values to the corresponding `ssl` and `sslmode` parameters in `redshift_connector`. +For backward compatibility, dbt-redshift now supports valid inputs for `sslmode` in `psycopg2`. We've added conversion logic mapping each of `psycopg2`'s accepted `sslmode` values to the corresponding `ssl` and `sslmode` parameters in `redshift_connector`. Table below details accepted `sslmode` parameters and how connection will be made according to each option: From 4c9197a16e07bc8e998945908503df1f19238d5a Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:36:29 +0100 Subject: [PATCH 17/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 085700d7072..5ba6bd2644f 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -141,7 +141,7 @@ In dbt-redshift 1.5, we switched to using `redshift_connector`, which accepts `v For backward compatibility, dbt-redshift now supports valid inputs for `sslmode` in `psycopg2`. We've added conversion logic mapping each of `psycopg2`'s accepted `sslmode` values to the corresponding `ssl` and `sslmode` parameters in `redshift_connector`. -Table below details accepted `sslmode` parameters and how connection will be made according to each option: +The table below details accepted `sslmode` parameters and how the connection will be made according to each option: `sslmode` parameter | Expected behavior in dbt-redshift | Actions behind the scenes From abb1dbee4d215feb07bff9571cd247ff1d44ba45 Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:36:59 +0100 Subject: [PATCH 18/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 5ba6bd2644f..0d405b2296d 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -153,7 +153,7 @@ require | Connection will be made using verify-ca | Set `ssl` = True & `sslmode verify-ca | Connection will be made using verify-ca | Set `ssl` = True & `sslmode` = verify-ca verify-full | Connection will be made using verify-full | Set `ssl` = True & `sslmode` = verify-full -When connection is made using `verify-ca`, will look for CA certificate in `~/redshift-ca-bundle.crt`. +When a connection is made using `verify-ca`, will look for the CA certificate in `~/redshift-ca-bundle.crt`. For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). From 63f36e0d7f6a8b05a9cb20bf338da02fd2f8d7c0 Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:37:57 +0100 Subject: [PATCH 19/20] Update website/docs/docs/core/connect-data-platform/redshift-setup.md --- website/docs/docs/core/connect-data-platform/redshift-setup.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index 0d405b2296d..f205ba83ac2 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -155,7 +155,7 @@ verify-full | Connection will be made using verify-full | Set `ssl` = True & `s When a connection is made using `verify-ca`, will look for the CA certificate in `~/redshift-ca-bundle.crt`. -For more details on changes of sslmode, our design choices and reasoning, please refer to the [PR pertaining this change](https://github.com/dbt-labs/dbt-redshift/pull/439). +For more details on sslmode changes, our design choices, and reasoning — please refer to the [PR pertaining to this change](https://github.com/dbt-labs/dbt-redshift/pull/439). ### `autocommit` parameter From 2819394ff296d478a6cd2bd2f8925835aeab0990 Mon Sep 17 00:00:00 2001 From: mirnawong1 <89008547+mirnawong1@users.noreply.github.com> Date: Wed, 7 Jun 2023 10:42:46 +0100 Subject: [PATCH 20/20] Update redshift-setup.md --- .../docs/core/connect-data-platform/redshift-setup.md | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/website/docs/docs/core/connect-data-platform/redshift-setup.md b/website/docs/docs/core/connect-data-platform/redshift-setup.md index f205ba83ac2..4d5042a26be 100644 --- a/website/docs/docs/core/connect-data-platform/redshift-setup.md +++ b/website/docs/docs/core/connect-data-platform/redshift-setup.md @@ -128,12 +128,10 @@ my-redshift-db: ### Specifying an IAM Profile -:::info New in dbt v0.18.0 -The `iam_profile` config option for Redshift profiles is new in dbt v0.18.0 -::: - When the `iam_profile` configuration is set, dbt will use the specified profile from your `~/.aws/config` file instead of using the profile name `default` + ## Redshift notes + ### `sslmode` change Before to dbt-redshift 1.5, `psycopg2` was used as the driver. `psycopg2` accepts `disable`, `prefer`, `allow`, `require`, `verify-ca`, `verify-full` as valid inputs of `sslmode`, and does not have an `ssl` parameter, as indicated in PostgreSQL [doc](https://www.postgresql.org/docs/current/libpq-connect.html#LIBPQ-CONNSTRING:~:text=%2Dencrypted%20connection.-,sslmode,-This%20option%20determines). @@ -143,7 +141,6 @@ For backward compatibility, dbt-redshift now supports valid inputs for `sslmode` The table below details accepted `sslmode` parameters and how the connection will be made according to each option: - `sslmode` parameter | Expected behavior in dbt-redshift | Actions behind the scenes -- | -- | -- disable | Connection will be made without using ssl | Set `ssl` = False @@ -195,6 +192,7 @@ To run certain macros with autocommit, load the profile with autocommit using th - `keepalives_idle` ### `sort` and `dist` keys + Where possible, dbt enables the use of `sort` and `dist` keys. See the section on [Redshift specific configurations](/reference/resource-configs/redshift-configs).