From a3fc26bc285d16d12f9a595af2d19d4e126dbcea Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 09:44:39 +0200 Subject: [PATCH 1/8] Allow AWS Secrets Manager backend to retrieve secrets using different fields --- .../amazon/aws/secrets/secrets_manager.py | 47 +++++++++++++------ .../secrets-backends/aws-secrets-manager.rst | 5 +- .../aws/secrets/test_secrets_manager.py | 2 +- 3 files changed, 36 insertions(+), 18 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 1fd6e1e47587c..23904a98acaf7 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -74,22 +74,26 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): However, these lists can be extended using the configuration parameter ``extra_conn_words``. :param connections_prefix: Specifies the prefix of the secret to read to get Connections. - If set to None (null), requests for connections will not be sent to AWS Secrets Manager + If set to None (null value in the configuration), requests for connections will not be + sent to AWS Secrets Manager. If you don't want a connections_prefix, set it as an empty string :type connections_prefix: str :param variables_prefix: Specifies the prefix of the secret to read to get Variables. - If set to None (null), requests for variables will not be sent to AWS Secrets Manager + If set to None (null value in the configuration), requests for variables will not be sent to + AWS Secrets Manager. If you don't want a variables_prefix, set it as an empty string :type variables_prefix: str :param config_prefix: Specifies the prefix of the secret to read to get Configurations. - If set to None (null), requests for configurations will not be sent to AWS Secrets Manager + If set to None (null value in the configuration), requests for configurations will not be sent to + AWS Secrets Manager. If you don't want a config_prefix, set it as an empty string :type config_prefix: str :param profile_name: The name of a profile to use. If not given, then the default profile is used. :type profile_name: str :param sep: separator used to concatenate secret_prefix and secret_id. Default: "/" :type sep: str :param full_url_mode: if True, the secrets must be stored as one conn URI in just one field per secret. - Otherwise, you can store the secret using different fields (password, user...) + Otherwise, you can store the secret using different fields (password, user...). For using it + as False, set as str "False" in backend_kwargs. :type full_url_mode: bool - :param extra_conn_words: for using just when you set full_url_mode as False and store + :param extra_conn_words: for using just when you set full_url_mode as "False" and store the secrets in different fields of secrets manager. You can add more words for each connection part beyond the default ones. The extra words to be searched should be passed as a dict of lists, each list corresponding to a connection part. The optional keys of the dict must be: user, @@ -109,21 +113,26 @@ def __init__( **kwargs, ): super().__init__() - if connections_prefix is not None: + if connections_prefix: self.connections_prefix = connections_prefix.rstrip(sep) else: self.connections_prefix = connections_prefix - if variables_prefix is not None: + if variables_prefix: self.variables_prefix = variables_prefix.rstrip(sep) else: self.variables_prefix = variables_prefix - if config_prefix is not None: + if config_prefix: self.config_prefix = config_prefix.rstrip(sep) else: self.config_prefix = config_prefix self.profile_name = profile_name self.sep = sep - self.full_url_mode = full_url_mode + if isinstance(full_url_mode, str): + self.full_url_mode = ast.literal_eval( + full_url_mode + ) # if you pass a boolean in conf, it takes the default value, so you must pass a string + else: + self.full_url_mode = full_url_mode self.extra_conn_words = extra_conn_words if extra_conn_words else {} self.kwargs = kwargs @@ -134,12 +143,17 @@ def client(self): return session.client(service_name="secretsmanager", **self.kwargs) - def _format_uri_with_extra(self, secret, conn_string): + @staticmethod + def _format_uri_with_extra(secret, conn_string): try: extra_dict = secret['extra'] except KeyError: return conn_string - conn_string = f"{conn_string}?{urlencode(extra_dict)}" + + extra = ast.literal_eval( + extra_dict + ) # json.loads doesn't work because you can have unquoted booleans values + conn_string = f"{conn_string}?{urlencode(extra)}" return conn_string @@ -176,10 +190,10 @@ def get_conn_uri(self, conn_id: str): :param conn_id: connection id :type conn_id: str """ - if self.full_url_mode: - if self.connections_prefix is None: - return None + if self.connections_prefix is None: + return None + if self.full_url_mode: return self._get_secret(self.connections_prefix, conn_id) else: try: @@ -225,7 +239,10 @@ def _get_secret(self, path_prefix, secret_id: str) -> Optional[str]: :param secret_id: Secret Key :type secret_id: str """ - secrets_path = self.build_path(path_prefix, secret_id, self.sep) + if path_prefix: + secrets_path = self.build_path(path_prefix, secret_id, self.sep) + else: + secrets_path = secret_id try: response = self.client.get_secret_value( diff --git a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst index b80e988d1b58b..0e24563d759eb 100644 --- a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst +++ b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst @@ -27,7 +27,7 @@ Here is a sample configuration: [secrets] backend = airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend - backend_kwargs = {"connections_prefix": "airflow/connections", "variables_prefix": "airflow/variables", "profile_name": "default", "full_url_mode": False} + backend_kwargs = {"connections_prefix": "airflow/connections", "variables_prefix": "airflow/variables", "profile_name": "default", "full_url_mode": "False"} To authenticate you can either supply a profile name to reference aws profile, e.g. defined in ``~/.aws/config`` or set environment variables like ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``. @@ -36,7 +36,7 @@ environment variables like ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``. Storing and Retrieving Connections """""""""""""""""""""""""""""""""" You can store the different values for a secret in two forms: storing the conn URI in one field (default mode) or using different -fields in Amazon Secrets Manager (setting ``full_url_mode`` as False in the backend config), as follow: +fields in Amazon Secrets Manager (setting ``full_url_mode`` as "False" in the backend config), as follow: .. image:: img/aws-secrets-manager.png By default you must use some of the following words for each kind of field: @@ -78,6 +78,7 @@ Verify that you can get the secret: "CreatedDate": "2020-04-08T02:10:35.132000+01:00" } +If you don't want to use any ``connections_prefix`` for retrieving connections, set it as an empty string ``""`` in the configuration. Storing and Retrieving Variables """""""""""""""""""""""""""""""" diff --git a/tests/providers/amazon/aws/secrets/test_secrets_manager.py b/tests/providers/amazon/aws/secrets/test_secrets_manager.py index 9495ccc179639..7c2b355329d1b 100644 --- a/tests/providers/amazon/aws/secrets/test_secrets_manager.py +++ b/tests/providers/amazon/aws/secrets/test_secrets_manager.py @@ -93,7 +93,7 @@ def test_get_conn_uri_broken_field_mode_extra_words_added(self): @mock_secretsmanager def test_format_uri_with_extra(self): - secret = {'extra': {'key1': 'value1', 'key2': 'value2'}} + secret = {'extra': "{'key1': 'value1', 'key2': 'value2'}"} conn_string = 'CS' secrets_manager_backend = SecretsManagerBackend() From 83ca6bfe720d409320952bb208e141aa6354dfa3 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 09:52:20 +0200 Subject: [PATCH 2/8] Enable AWS Secrets Manager backend to retrieve conns using different fields --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 23904a98acaf7..d85d69b479bfb 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -83,7 +83,7 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): :type variables_prefix: str :param config_prefix: Specifies the prefix of the secret to read to get Configurations. If set to None (null value in the configuration), requests for configurations will not be sent to - AWS Secrets Manager. If you don't want a config_prefix, set it as an empty string + AWS Secrets Manager. If you don't want a config_prefix, set it as an empty string :type config_prefix: str :param profile_name: The name of a profile to use. If not given, then the default profile is used. :type profile_name: str From e18623436dccbdc1d8762d3c61a6030dbd547f47 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 12:59:50 +0200 Subject: [PATCH 3/8] change full_url_mode doc --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index d85d69b479bfb..11d1b5feb615c 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -90,8 +90,8 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): :param sep: separator used to concatenate secret_prefix and secret_id. Default: "/" :type sep: str :param full_url_mode: if True, the secrets must be stored as one conn URI in just one field per secret. - Otherwise, you can store the secret using different fields (password, user...). For using it - as False, set as str "False" in backend_kwargs. + If False (set it as the string "False" in backend_kwargs), you can store the secret using different + fields (password, user...). :type full_url_mode: bool :param extra_conn_words: for using just when you set full_url_mode as "False" and store the secrets in different fields of secrets manager. You can add more words for each connection From 71c13ff3cef9093f2f5ef496f7701c17970a2981 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 13:44:12 +0200 Subject: [PATCH 4/8] ash suggestions --- .../providers/amazon/aws/secrets/secrets_manager.py | 11 +++-------- .../secrets-backends/aws-secrets-manager.rst | 7 ++++--- 2 files changed, 7 insertions(+), 11 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 11d1b5feb615c..9d488fd21b433 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -90,10 +90,10 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): :param sep: separator used to concatenate secret_prefix and secret_id. Default: "/" :type sep: str :param full_url_mode: if True, the secrets must be stored as one conn URI in just one field per secret. - If False (set it as the string "False" in backend_kwargs), you can store the secret using different + If False (set it as the string false in backend_kwargs), you can store the secret using different fields (password, user...). :type full_url_mode: bool - :param extra_conn_words: for using just when you set full_url_mode as "False" and store + :param extra_conn_words: for using just when you set full_url_mode as false and store the secrets in different fields of secrets manager. You can add more words for each connection part beyond the default ones. The extra words to be searched should be passed as a dict of lists, each list corresponding to a connection part. The optional keys of the dict must be: user, @@ -127,12 +127,7 @@ def __init__( self.config_prefix = config_prefix self.profile_name = profile_name self.sep = sep - if isinstance(full_url_mode, str): - self.full_url_mode = ast.literal_eval( - full_url_mode - ) # if you pass a boolean in conf, it takes the default value, so you must pass a string - else: - self.full_url_mode = full_url_mode + self.full_url_mode = full_url_mode self.extra_conn_words = extra_conn_words if extra_conn_words else {} self.kwargs = kwargs diff --git a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst index 0e24563d759eb..f67ffb24c2c74 100644 --- a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst +++ b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst @@ -19,7 +19,8 @@ AWS Secrets Manager Backend ^^^^^^^^^^^^^^^^^^^^^^^^^^^ To enable Secrets Manager, specify :py:class:`~airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend` -as the ``backend`` in ``[secrets]`` section of ``airflow.cfg``. +as the ``backend`` in ``[secrets]`` section of ``airflow.cfg``. These ``backend_kwargs`` are parsed as JSON, hence Python +values like the bool False or None will be ignored, taking for those kwargs the default values of the secrets backend. Here is a sample configuration: @@ -27,7 +28,7 @@ Here is a sample configuration: [secrets] backend = airflow.providers.amazon.aws.secrets.secrets_manager.SecretsManagerBackend - backend_kwargs = {"connections_prefix": "airflow/connections", "variables_prefix": "airflow/variables", "profile_name": "default", "full_url_mode": "False"} + backend_kwargs = {"connections_prefix": "airflow/connections", "variables_prefix": "airflow/variables", "profile_name": "default", "full_url_mode": false} To authenticate you can either supply a profile name to reference aws profile, e.g. defined in ``~/.aws/config`` or set environment variables like ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``. @@ -36,7 +37,7 @@ environment variables like ``AWS_ACCESS_KEY_ID``, ``AWS_SECRET_ACCESS_KEY``. Storing and Retrieving Connections """""""""""""""""""""""""""""""""" You can store the different values for a secret in two forms: storing the conn URI in one field (default mode) or using different -fields in Amazon Secrets Manager (setting ``full_url_mode`` as "False" in the backend config), as follow: +fields in Amazon Secrets Manager (setting ``full_url_mode`` as ``false`` in the backend config), as follow: .. image:: img/aws-secrets-manager.png By default you must use some of the following words for each kind of field: From acd171df988d8078cbcb0a80848114e5fc4e0e6d Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 13:45:14 +0200 Subject: [PATCH 5/8] ash suggestions --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 9d488fd21b433..23d6b79c80fb0 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -90,7 +90,7 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): :param sep: separator used to concatenate secret_prefix and secret_id. Default: "/" :type sep: str :param full_url_mode: if True, the secrets must be stored as one conn URI in just one field per secret. - If False (set it as the string false in backend_kwargs), you can store the secret using different + If False (set it as false in backend_kwargs), you can store the secret using different fields (password, user...). :type full_url_mode: bool :param extra_conn_words: for using just when you set full_url_mode as false and store From 01c0831eaf7a3e04b018c4c3f69b73d0bf4a2b6d Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Wed, 6 Oct 2021 15:15:14 +0200 Subject: [PATCH 6/8] clarification --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index 23d6b79c80fb0..bf7de9960b974 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -146,7 +146,7 @@ def _format_uri_with_extra(secret, conn_string): return conn_string extra = ast.literal_eval( - extra_dict + extra_dict # this is needed because extra_dict is a string and we need a dict ) # json.loads doesn't work because you can have unquoted booleans values conn_string = f"{conn_string}?{urlencode(extra)}" From d000e13200a5501337426946b2fe3d5025d0bc3c Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Thu, 7 Oct 2021 15:19:56 +0200 Subject: [PATCH 7/8] ash suggestions --- airflow/providers/amazon/aws/secrets/secrets_manager.py | 9 +++++---- .../secrets-backends/aws-secrets-manager.rst | 3 ++- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/airflow/providers/amazon/aws/secrets/secrets_manager.py b/airflow/providers/amazon/aws/secrets/secrets_manager.py index bf7de9960b974..35ddf764bd7ff 100644 --- a/airflow/providers/amazon/aws/secrets/secrets_manager.py +++ b/airflow/providers/amazon/aws/secrets/secrets_manager.py @@ -18,6 +18,7 @@ """Objects relating to sourcing secrets from AWS Secrets Manager""" import ast +import json from typing import Optional from urllib.parse import urlencode @@ -71,7 +72,9 @@ class SecretsManagerBackend(BaseSecretsBackend, LoggingMixin): "conn_type": ["conn_type", "conn_id", "connection_type", "engine"], } - However, these lists can be extended using the configuration parameter ``extra_conn_words``. + However, these lists can be extended using the configuration parameter ``extra_conn_words``. Also, + you can have a field named extra for extra parameters for the conn. Please note that this extra field + must be a valid JSON. :param connections_prefix: Specifies the prefix of the secret to read to get Connections. If set to None (null value in the configuration), requests for connections will not be @@ -145,9 +148,7 @@ def _format_uri_with_extra(secret, conn_string): except KeyError: return conn_string - extra = ast.literal_eval( - extra_dict # this is needed because extra_dict is a string and we need a dict - ) # json.loads doesn't work because you can have unquoted booleans values + extra = json.loads(extra_dict) # this is needed because extra_dict is a string and we need a dict conn_string = f"{conn_string}?{urlencode(extra)}" return conn_string diff --git a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst index f67ffb24c2c74..692b77caa453a 100644 --- a/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst +++ b/docs/apache-airflow-providers-amazon/secrets-backends/aws-secrets-manager.rst @@ -48,7 +48,8 @@ By default you must use some of the following words for each kind of field: * Port: port * You should also specify the type of connection, which can be done naming the key as conn_type, conn_id, connection_type or engine. Valid values for this field are postgres, mysql, snowflake, google_cloud, mongo... -* For the extra value of the connections, you have to type a dictionary. +* For the extra value of the connections, a field called extra must exists. Please note this extra field + should be a valid JSON. However, more words can be added to the list using the parameter ``extra_conn_words`` in the configuration. This parameter has to be a dict of lists with the following optional keys: user, password, host, schema, conn_type From f52c53d2a626dda662b1cb4128df643bb0e83138 Mon Sep 17 00:00:00 2001 From: "javier.lopez" Date: Thu, 7 Oct 2021 16:28:04 +0200 Subject: [PATCH 8/8] fix test --- tests/providers/amazon/aws/secrets/test_secrets_manager.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/providers/amazon/aws/secrets/test_secrets_manager.py b/tests/providers/amazon/aws/secrets/test_secrets_manager.py index 7c2b355329d1b..57e6257effd11 100644 --- a/tests/providers/amazon/aws/secrets/test_secrets_manager.py +++ b/tests/providers/amazon/aws/secrets/test_secrets_manager.py @@ -93,7 +93,7 @@ def test_get_conn_uri_broken_field_mode_extra_words_added(self): @mock_secretsmanager def test_format_uri_with_extra(self): - secret = {'extra': "{'key1': 'value1', 'key2': 'value2'}"} + secret = {'extra': '{"key1": "value1", "key2": "value2"}'} conn_string = 'CS' secrets_manager_backend = SecretsManagerBackend()