From 71846c98b33d8474e608ec433b5e69fc24468168 Mon Sep 17 00:00:00 2001 From: Bruno Murino Date: Mon, 22 Jun 2020 21:39:03 +0100 Subject: [PATCH 1/7] added option to specify profile when connecting to redshift via IAM --- .../dbt/adapters/redshift/connections.py | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/plugins/redshift/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py index a60d77cde66..35f1a068434 100644 --- a/plugins/redshift/dbt/adapters/redshift/connections.py +++ b/plugins/redshift/dbt/adapters/redshift/connections.py @@ -44,6 +44,7 @@ class RedshiftCredentials(PostgresCredentials): default=None, metadata={'description': 'If using IAM auth, the name of the cluster'}, ) + iam_profile: Optional[str] = None iam_duration_seconds: int = 900 search_path: Optional[str] = None keepalives_idle: int = 240 @@ -56,7 +57,7 @@ def type(self): def _connection_keys(self): keys = super()._connection_keys() - return keys + ('method', 'cluster_id', 'iam_duration_seconds') + return keys + ('method', 'cluster_id', 'iam_profile', 'iam_duration_seconds') class RedshiftConnectionManager(PostgresConnectionManager): @@ -86,11 +87,20 @@ def fresh_transaction(self, name=None): self.begin() @classmethod - def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, + def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, iam_profile, duration_s, autocreate, db_groups): """Fetches temporary login credentials from AWS. The specified user must already exist in the database, or else an error will occur""" - boto_client = boto3.client('redshift') + + if iam_profile is None: + boto_client = boto3.client('redshift') + else: + logger.debug(f"Connecting to Redshift using 'IAM' with profile {iam_profile}") + boto_session = boto3.Session( + profile_name=iam_profile, + region_name='eu-west-1' + ) + boto_client = boto_session.client('redshift') try: return boto_client.get_cluster_credentials( @@ -123,6 +133,7 @@ def get_tmp_iam_cluster_credentials(cls, credentials): credentials.user, credentials.database, credentials.cluster_id, + credentials.iam_profile, iam_duration_s, credentials.autocreate, credentials.db_groups, From df894f91f2ad086e15883025df86ff81e268eaf3 Mon Sep 17 00:00:00 2001 From: brunomurino Date: Mon, 22 Jun 2020 22:27:57 +0100 Subject: [PATCH 2/7] updated changelog --- CHANGELOG.md | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1bd96c79041..90998bfb1fa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,7 @@ +### Next +- Added option to specify profile when connecting to Redshift via IAM + + ## dbt 0.18.0 (Release TBD) ## dbt 0.18.0b1 (June 08, 2020) From 053210330e0d2ab28590b74bdbdaaa198d935239 Mon Sep 17 00:00:00 2001 From: brunomurino Date: Mon, 22 Jun 2020 23:03:14 +0100 Subject: [PATCH 3/7] updated to comply with pep8 guidelines --- .../redshift/dbt/adapters/redshift/connections.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/plugins/redshift/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py index 35f1a068434..95d2d1a93d1 100644 --- a/plugins/redshift/dbt/adapters/redshift/connections.py +++ b/plugins/redshift/dbt/adapters/redshift/connections.py @@ -57,7 +57,12 @@ def type(self): def _connection_keys(self): keys = super()._connection_keys() - return keys + ('method', 'cluster_id', 'iam_profile', 'iam_duration_seconds') + return keys + ( + 'method', + 'cluster_id', + 'iam_profile', + 'iam_duration_seconds' + ) class RedshiftConnectionManager(PostgresConnectionManager): @@ -87,15 +92,17 @@ def fresh_transaction(self, name=None): self.begin() @classmethod - def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, iam_profile, - duration_s, autocreate, db_groups): + def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, + iam_profile, duration_s, autocreate, + db_groups): """Fetches temporary login credentials from AWS. The specified user must already exist in the database, or else an error will occur""" if iam_profile is None: boto_client = boto3.client('redshift') else: - logger.debug(f"Connecting to Redshift using 'IAM' with profile {iam_profile}") + logger.debug("Connecting to Redshift using 'IAM'" + + f"with profile {iam_profile}") boto_session = boto3.Session( profile_name=iam_profile, region_name='eu-west-1' From 78445f9879291a6fadda8185b691b76641f84b5a Mon Sep 17 00:00:00 2001 From: brunomurino Date: Mon, 22 Jun 2020 23:15:40 +0100 Subject: [PATCH 4/7] updates to make pep8 compliant --- plugins/redshift/dbt/adapters/redshift/connections.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/plugins/redshift/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py index 95d2d1a93d1..e6e304ed8f9 100644 --- a/plugins/redshift/dbt/adapters/redshift/connections.py +++ b/plugins/redshift/dbt/adapters/redshift/connections.py @@ -93,16 +93,16 @@ def fresh_transaction(self, name=None): @classmethod def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, - iam_profile, duration_s, autocreate, - db_groups): + iam_profile, duration_s, autocreate, + db_groups): """Fetches temporary login credentials from AWS. The specified user must already exist in the database, or else an error will occur""" if iam_profile is None: boto_client = boto3.client('redshift') else: - logger.debug("Connecting to Redshift using 'IAM'" - + f"with profile {iam_profile}") + logger.debug("Connecting to Redshift using 'IAM'" + + f"with profile {iam_profile}") boto_session = boto3.Session( profile_name=iam_profile, region_name='eu-west-1' From bd34dfe1f7dc5fc8e10bc9ed49b3bafd912b136e Mon Sep 17 00:00:00 2001 From: brunomurino Date: Tue, 23 Jun 2020 14:53:53 +0100 Subject: [PATCH 5/7] removed hardcoded option in fetch_cluster_credentials --- plugins/redshift/dbt/adapters/redshift/connections.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/plugins/redshift/dbt/adapters/redshift/connections.py b/plugins/redshift/dbt/adapters/redshift/connections.py index e6e304ed8f9..04df051c449 100644 --- a/plugins/redshift/dbt/adapters/redshift/connections.py +++ b/plugins/redshift/dbt/adapters/redshift/connections.py @@ -104,8 +104,7 @@ def fetch_cluster_credentials(cls, db_user, db_name, cluster_id, logger.debug("Connecting to Redshift using 'IAM'" + f"with profile {iam_profile}") boto_session = boto3.Session( - profile_name=iam_profile, - region_name='eu-west-1' + profile_name=iam_profile ) boto_client = boto_session.client('redshift') From 65c156750d6830fe23ab95e44478864b49f4bac6 Mon Sep 17 00:00:00 2001 From: brunomurino Date: Tue, 23 Jun 2020 15:09:10 +0100 Subject: [PATCH 6/7] updated changelog --- CHANGELOG.md | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 90998bfb1fa..822b00ee430 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,11 @@ -### Next -- Added option to specify profile when connecting to Redshift via IAM +## dbt 0.18.0 (Release TBD) +### Features +- Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437)) + +Contributors: +- [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581)) -## dbt 0.18.0 (Release TBD) ## dbt 0.18.0b1 (June 08, 2020) From 0c878a056e8331294ac5742bd5387f3341782f3d Mon Sep 17 00:00:00 2001 From: Bruno Murino Date: Tue, 23 Jun 2020 16:24:25 +0100 Subject: [PATCH 7/7] Update CHANGELOG.md Co-authored-by: Jacob Beck --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 822b00ee430..14f90737b50 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ ## dbt 0.18.0 (Release TBD) ### Features -- Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437)) +- Added option to specify profile when connecting to Redshift via IAM ([#2437](https://github.com/fishtown-analytics/dbt/issues/2437), [#2581](https://github.com/fishtown-analytics/dbt/pull/2581)) Contributors: - [@brunomurino](https://github.com/brunomurino) ([#2437](https://github.com/fishtown-analytics/dbt/pull/2581))