Added secrets manager documentation (#589)

* Added secrets manager documentation * addressed comments updated changelog --------- Co-authored-by: Hemanth Kannekanti <hemanth@fennel.ai>
fennel-ai · Oct 30, 2024 · bbf573c · bbf573c
1 parent 59e2520
commit bbf573c
Show file tree

Hide file tree

Showing 19 changed files with 157 additions and 27 deletions.
diff --git a/.wordlist.txt b/.wordlist.txt
@@ -123,6 +123,7 @@ SSL
 SSO
 Sample
 SearchRequest
+Secret
 ShardIteratorType
 Signifier
 SnapshotData
@@ -346,6 +347,7 @@ schedulable
 schemaless
 schemas
 sdk
+secret
 signup
 signups
 sinked

diff --git a/docs/api.yml b/docs/api.yml
@@ -67,6 +67,7 @@ sidebar:
       - "api-reference/source_connectors/pubsub"
       - "api-reference/source_connectors/redshift"
       - "api-reference/source_connectors/s3"
+      - "api-reference/source_connectors/secret"
       - "api-reference/source_connectors/snowflake"
       - "api-reference/source_connectors/webhook"
 

diff --git a/docs/examples/api-reference/sources/kafka.py b/docs/examples/api-reference/sources/kafka.py
@@ -39,6 +39,66 @@ class SomeDataset:
     client.commit(message="some commit msg", datasets=[SomeDataset])
 
 
+@mock
+def test_kafka_source_with_secret(client):
+    os.environ["SCHEMA_REGISTRY_URL"] = "http://localhost:8081"
+    # docsnip secret
+    from fennel.connectors import source, Kafka, Avro
+    from fennel.datasets import dataset, field
+    from fennel.integrations.aws import Secret
+
+    # docsnip-highlight start
+    aws_secret = Secret(
+        arn="arn:aws:secretsmanager:us-east-1:123456789012:secret:my-secret-name-I4hSKr",
+        role_arn="arn:aws:iam::123456789012:role/secret-access-role",
+    )
+    # docsnip-highlight end
+
+    # secret with above arn has content like below
+    # {
+    #     "kafka": {
+    #         "username": "actual-kafka-username",
+    #         "password": "actual-kafka-password"
+    #     },
+    #     "schema_registry": {
+    #         "username": "actual-schema-registry-username",
+    #         "password": "actual-schema-registry-password"
+    #     }
+    # }
+
+    kafka = Kafka(
+        name="my_kafka",
+        bootstrap_servers="localhost:9092",  # could come via os env var too
+        security_protocol="SASL_PLAINTEXT",
+        sasl_mechanism="PLAIN",
+        # docsnip-highlight start
+        sasl_plain_username=aws_secret["kafka"]["username"],
+        sasl_plain_password=aws_secret["kafka"]["password"],
+        # docsnip-highlight end
+    )
+    avro = Avro(
+        registry="confluent",
+        url=os.environ["SCHEMA_REGISTRY_URL"],
+        # docsnip-highlight start
+        username=aws_secret["schema_registry"]["username"],
+        password=aws_secret["schema_registry"]["password"],
+        # docsnip-highlight end
+    )
+
+    # docsnip-highlight start
+    @source(kafka.topic("user", format=avro), disorder="14d", cdc="upsert")
+    # docsnip-highlight end
+    @dataset
+    class SomeDataset:
+        uid: int = field(key=True)
+        email: str
+        timestamp: datetime
+
+    # /docsnip
+
+    client.commit(message="some commit msg", datasets=[SomeDataset])
+
+
 @mock
 def test_kafka_with_avro(client):
     os.environ["KAFKA_USERNAME"] = "test"

diff --git a/docs/pages/api-reference/sink_connectors/kafka.md b/docs/pages/api-reference/sink_connectors/kafka.md
@@ -30,11 +30,11 @@ Protocol used to communicate with the brokers.
 SASL mechanism to use for authentication. 
 </Expandable>
 
-<Expandable title="sasl_plain_username" type="Optional[str]">
+<Expandable title="sasl_plain_username" type="Optional[str] | Optional[Secret]">
 SASL username.
 </Expandable>
 
-<Expandable title="sasl_plain_password" type="Optional[str]">
+<Expandable title="sasl_plain_password" type="Optional[str] | Optional[Secret]">
 SASL password.
 </Expandable>
 

diff --git a/docs/pages/api-reference/sink_connectors/s3.md b/docs/pages/api-reference/sink_connectors/s3.md
@@ -16,7 +16,7 @@ AWS Access Key ID. This field is not required if role-based access is used or if
 the bucket is public.
 </Expandable>
 
-<Expandable title="aws_secrete_access_key" type="Optional[str]" defaultVal="None">
+<Expandable title="aws_secret_access_key" type="Optional[str]" defaultVal="None">
 AWS Secret Access Key. This field is not required if role-based access is used 
 or if the bucket is public.
 </Expandable>

diff --git a/docs/pages/api-reference/sink_connectors/snowflake.md b/docs/pages/api-reference/sink_connectors/snowflake.md
@@ -38,12 +38,12 @@ The name of the database where the data has to be sinked.
 The schema where the required data has to be sinked.
 </Expandable>
 
-<Expandable title="username" type="str">
+<Expandable title="username" type="str | Secret">
 The username which should be used to access Snowflake. This username should 
 have required permissions to assume the provided `role`.
 </Expandable>
 
-<Expandable title="password" type="str">
+<Expandable title="password" type="str | Secret">
 The password associated with the username.
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/avro.md b/docs/pages/api-reference/source_connectors/avro.md
@@ -22,7 +22,7 @@ time.
 The URL where the schema registry is hosted.
 </Expandable>
 
-<Expandable title="username" type="Optional[str]">
+<Expandable title="username" type="Optional[str] | Optional[Secret]">
 User name to access the schema registry (assuming the registry requires 
 authentication). If user name is provided, corresponding password must also be
 provided.
@@ -31,11 +31,11 @@ Assuming authentication is needed, either username/password must be provided or
 a token, but not both.
 </Expandable>
 
-<Expandable title="password" type="Optional[str]">
+<Expandable title="password" type="Optional[str] | Optional[Secret]">
 The password associated with the username.
 </Expandable>
 
-<Expandable title="token" type="Optional[str]">
+<Expandable title="token" type="Optional[str] | Optional[Secret]">
 Token to be used for authentication with the schema registry. Only one of 
 username/password or token must be provided.
 </Expandable>

diff --git a/docs/pages/api-reference/source_connectors/bigquery.md b/docs/pages/api-reference/source_connectors/bigquery.md
@@ -19,7 +19,7 @@ The project ID of the Google Cloud project containing the BigQuery dataset.
 The ID of the BigQuery dataset containing the table(s) to replicate.
 </Expandable>
 
-<Expandable title="service_account_key" type="Dict[str, str]">
+<Expandable title="service_account_key" type="Dict[str, str] | Secret">
 A dictionary containing the credentials for the Service Account to use to access
 BigQuery. See below for instructions on how to obtain this.
 </Expandable>

diff --git a/docs/pages/api-reference/source_connectors/kafka.md b/docs/pages/api-reference/source_connectors/kafka.md
@@ -30,11 +30,11 @@ Protocol used to communicate with the brokers.
 SASL mechanism to use for authentication. 
 </Expandable>
 
-<Expandable title="sasl_plain_username" type="Optional[str]">
+<Expandable title="sasl_plain_username" type="Optional[str] | Optional[Secret]">
 SASL username.
 </Expandable>
 
-<Expandable title="sasl_plain_password" type="Optional[str]">
+<Expandable title="sasl_plain_password" type="Optional[str] | Optional[Secret]">
 SASL password.
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/mongo.md b/docs/pages/api-reference/source_connectors/mongo.md
@@ -19,12 +19,12 @@ The hostname of the database.
 The name of the Mongo database to establish a connection with.
 </Expandable>
 
-<Expandable title="username" type="str">
+<Expandable title="username" type="str | Secret">
 The username which should be used to access the database. This username should 
 have access to the database `db_name`.
 </Expandable>
 
-<Expandable title="password" type="str">
+<Expandable title="password" type="str | Secret">
 The password associated with the username.
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/mysql.md b/docs/pages/api-reference/source_connectors/mysql.md
@@ -23,12 +23,12 @@ The port to connect to.
 The name of the MySQL database to establish a connection with.
 </Expandable>
 
-<Expandable title="username" type="str">
+<Expandable title="username" type="str | Secret">
 The username which should be used to access the database. This username should 
 have access to the database `db_name`.
 </Expandable>
 
-<Expandable title="password" type="str">
+<Expandable title="password" type="str | Secret">
 The password associated with the username.
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/postgres.md b/docs/pages/api-reference/source_connectors/postgres.md
@@ -23,12 +23,12 @@ The port to connect to.
 The name of the Postgres database to establish a connection with.
 </Expandable>
 
-<Expandable title="username" type="str">
+<Expandable title="username" type="str | Secret">
 The username which should be used to access the database. This username should 
 have access to the database `db_name`.
 </Expandable>
 
-<Expandable title="password" type="str">
+<Expandable title="password" type="str | Secret">
 The password associated with the username.
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/protobuf.md b/docs/pages/api-reference/source_connectors/protobuf.md
@@ -22,7 +22,7 @@ time.
 The URL where the schema registry is hosted.
 </Expandable>
 
-<Expandable title="username" type="Optional[str]">
+<Expandable title="username" type="Optional[str] | Optional[Secret]">
 User name to access the schema registry (assuming the registry requires 
 authentication). If user name is provided, corresponding password must also be
 provided.
@@ -31,11 +31,11 @@ Assuming authentication is needed, either username/password must be provided or
 a token, but not both.
 </Expandable>
 
-<Expandable title="password" type="Optional[str]">
+<Expandable title="password" type="Optional[str] | Optional[Secret]">
 The password associated with the username.
 </Expandable>
 
-<Expandable title="token" type="Optional[str]">
+<Expandable title="token" type="Optional[str] | Optional[Secret]">
 Token to be used for authentication with the schema registry. Only one of 
 username/password or token must be provided.
 </Expandable>

diff --git a/docs/pages/api-reference/source_connectors/pubsub.md b/docs/pages/api-reference/source_connectors/pubsub.md
@@ -15,7 +15,7 @@ A name to identify the source. The name should be unique across all Fennel conne
 The project ID of the Google Cloud project containing the Pub/Sub topic
 </Expandable>
 
-<Expandable title="service_account_key" type="Dict[str, str]">
+<Expandable title="service_account_key" type="Dict[str, str] | Secret">
 A dictionary containing the credentials for the Service Account to use to access
 Pub/Sub. See below for instructions on how to obtain this.
 </Expandable>

diff --git a/docs/pages/api-reference/source_connectors/redshift.md b/docs/pages/api-reference/source_connectors/redshift.md
@@ -33,12 +33,12 @@ Do not set this parameter when using username/password for authentication
 The name of the database where the relevant data resides.
 </Expandable>
 
-<Expandable title="username" type="Optional[str]">
+<Expandable title="username" type="Optional[str] | Optional[Secret]">
 The username which should be used to access the database. This username should have access to the 
 database `db_name`. Do not set this parameter when using IAM authentication
 </Expandable>
 
-<Expandable title="password" type="Optional[str]">
+<Expandable title="password" type="Optional[str] | Optional[Secret]">
 The password associated with the username. Do not set this parameter when using IAM authentication
 </Expandable>
 

diff --git a/docs/pages/api-reference/source_connectors/s3.md b/docs/pages/api-reference/source_connectors/s3.md
@@ -11,12 +11,12 @@ Data connector to source data from S3.
 A name to identify the source. The name should be unique across all Fennel connectors.
 </Expandable>
 
-<Expandable title="aws_access_key_id" type="Optional[str]" defaultVal="None">
+<Expandable title="aws_access_key_id" type="Optional[str] | Optional[Secret]" defaultVal="None">
 AWS Access Key ID. This field is not required if role-based access is used or if
 the bucket is public.
 </Expandable>
 
-<Expandable title="aws_secrete_access_key" type="Optional[str]" defaultVal="None">
+<Expandable title="aws_secret_access_key" type="Optional[str] | Optional[Secret]" defaultVal="None">
 AWS Secret Access Key. This field is not required if role-based access is used 
 or if the bucket is public.
 </Expandable>

diff --git a/docs/pages/api-reference/source_connectors/secret.md b/docs/pages/api-reference/source_connectors/secret.md
@@ -0,0 +1,64 @@
+---
+title: Secret
+order: 0
+status: published
+---
+
+### Secret
+Secret can be used to pass sensitive information like username/password to Fennel using Secrets Manager secret reference.
+
+In order to use Secret one of the below should be followed:
+1. Fennel Data access role should be given access to the secret. 
+2. Or a new role can be created with access to secrets needed and Fennel Data access role can be added as trusted entities for that new role. so that the new role can be assumed to access the secrets.
+
+
+#### Parameters
+
+<Expandable title="arn" type="str">
+The ARN of the secret.
+</Expandable>
+
+<Expandable title="role_arn" type="Optional[str]">
+The Optional ARN of the role to be assumed to access the secret.
+This should be provided if a new role is created for Fennel Data access role to assume.
+</Expandable>
+
+<pre snippet="api-reference/sources/kafka#secret"
+    status="success" message="Using secrets with kafka"
+></pre>
+
+```JSON message="Example Permission policy for new role"
+{
+	"Version": "2012-10-17",
+	"Statement": [
+		{
+			"Sid": "VisualEditor0",
+			"Effect": "Allow",
+			"Action": [
+				"secretsmanager:GetResourcePolicy",
+				"secretsmanager:GetSecretValue",
+				"secretsmanager:DescribeSecret",
+				"secretsmanager:ListSecretVersionIds"
+			],
+			"Resource": "arn:aws:secretsmanager:us-west-2:123456789012:secret:my-secret-name-I4hSKr"
+		}
+	]
+}
+```
+
+```JSON message="Example Trusted relationship for the new role"
+{
+    "Version": "2012-10-17",
+    "Statement": [
+        {
+            "Effect": "Allow",
+            "Principal": {
+                "AWS": [
+                    "arn:aws:iam::123456789012:role/FennelDataAccessRole"
+                ]
+            },
+            "Action": "sts:AssumeRole"
+        }
+    ]
+}
+```
diff --git a/docs/pages/api-reference/source_connectors/snowflake.md b/docs/pages/api-reference/source_connectors/snowflake.md
@@ -38,12 +38,12 @@ The name of the database where the relevant data resides.
 The schema where the required data table(s) resides.
 </Expandable>
 
-<Expandable title="username" type="str">
+<Expandable title="username" type="str | Secret">
 The username which should be used to access Snowflake. This username should 
 have required permissions to assume the provided `role`.
 </Expandable>
 
-<Expandable title="password" type="str">
+<Expandable title="password" type="str | Secret">
 The password associated with the username.
 </Expandable>
 

diff --git a/fennel/CHANGELOG.md b/fennel/CHANGELOG.md
@@ -1,5 +1,8 @@
 # Changelog
 
+## [1.5.46] - 2024-10-30
+- Add support for AWS Secrets Manager
+
 ## [1.5.45] - 2024-10-30
 - Remove print statement in dedup operator