Skip to content

Commit

Permalink
Google Cloud SQL MySQL Connector (#4949)
Browse files Browse the repository at this point in the history
Co-authored-by: Adrian Galvan <adrian@ethyca.com>
  • Loading branch information
andres-torres-marroquin and galvana authored Jun 19, 2024
1 parent 480312e commit 39d23e1
Show file tree
Hide file tree
Showing 32 changed files with 858 additions and 45 deletions.
20 changes: 12 additions & 8 deletions .github/workflows/backend_checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -337,23 +337,27 @@ jobs:

- name: Integration Tests (External)
env:
BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }}
BIGQUERY_DATASET: fidesopstest
DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }}
BIGQUERY_KEYFILE_CREDS: ${{ secrets.BIGQUERY_KEYFILE_CREDS }}
DYNAMODB_ACCESS_KEY_ID: ${{ secrets.DYNAMODB_ACCESS_KEY_ID }}
DYNAMODB_ACCESS_KEY: ${{ secrets.DYNAMODB_ACCESS_KEY }}
DYNAMODB_REGION: ${{ secrets.DYNAMODB_REGION }}
GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER }}
GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME }}
GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME }}
GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS: ${{ secrets.GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS }}
REDSHIFT_TEST_DATABASE: ${{ secrets.REDSHIFT_TEST_DATABASE }}
REDSHIFT_TEST_DB_SCHEMA: ${{ secrets.REDSHIFT_TEST_DB_SCHEMA }}
REDSHIFT_TEST_HOST: ${{ secrets.REDSHIFT_TEST_HOST }}
REDSHIFT_TEST_PASSWORD: ${{ secrets.REDSHIFT_TEST_PASSWORD }}
REDSHIFT_TEST_PORT: ${{ secrets.REDSHIFT_TEST_PORT }}
REDSHIFT_TEST_USER: ${{ secrets.REDSHIFT_TEST_USER }}
REDSHIFT_TEST_PASSWORD: ${{ secrets.REDSHIFT_TEST_PASSWORD }}
REDSHIFT_TEST_DATABASE: ${{ secrets.REDSHIFT_TEST_DATABASE }}
REDSHIFT_TEST_DB_SCHEMA: ${{ secrets.REDSHIFT_TEST_DB_SCHEMA }}
SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER: ${{ secrets.SNOWFLAKE_TEST_ACCOUNT_IDENTIFIER }}
SNOWFLAKE_TEST_USER_LOGIN_NAME: ${{ secrets.SNOWFLAKE_TEST_USER_LOGIN_NAME }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
SNOWFLAKE_TEST_WAREHOUSE_NAME: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE_NAME }}
SNOWFLAKE_TEST_DATABASE_NAME: ${{ secrets.SNOWFLAKE_TEST_DATABASE_NAME }}
SNOWFLAKE_TEST_PASSWORD: ${{ secrets.SNOWFLAKE_TEST_PASSWORD }}
SNOWFLAKE_TEST_SCHEMA_NAME: ${{ secrets.SNOWFLAKE_TEST_SCHEMA_NAME }}
SNOWFLAKE_TEST_USER_LOGIN_NAME: ${{ secrets.SNOWFLAKE_TEST_USER_LOGIN_NAME }}
SNOWFLAKE_TEST_WAREHOUSE_NAME: ${{ secrets.SNOWFLAKE_TEST_WAREHOUSE_NAME }}

run: nox -s "pytest(ops-external-datastores)"

Expand Down
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ The types of changes are:
- Support for Limited FIDES__CELERY__* Env Vars [#4980](https://github.com/ethyca/fides/pull/4980)
- Implement sending emails via property-specific messaging templates [#4950](https://github.com/ethyca/fides/pull/4950)
- New privacy request search to replace existing endpoint [#4987](https://github.com/ethyca/fides/pull/4987)
- Added new Google Cloud SQL for MySQL Connector [#4949](https://github.com/ethyca/fides/pull/4949)

### Changed
- Move new data map reporting table out of beta and remove old table from Data Lineage map. [#4963](https://github.com/ethyca/fides/pull/4963)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,12 @@
"human_readable": "MySQL",
"encoded_icon": null
},
{
"identifier": "google_cloud_sql_mysql",
"type": "database",
"human_readable": "Google Cloud SQL for MySQL",
"encoded_icon": null
},
{
"identifier": "postgres",
"type": "database",
Expand Down
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ export const CONNECTION_TYPE_LOGO_MAP = new Map<ConnectionType, string>([
[ConnectionType.DYNAMODB, "dynamodb.svg"],
[ConnectionType.GENERIC_CONSENT_EMAIL, "ethyca.svg"],
[ConnectionType.GENERIC_ERASURE_EMAIL, "ethyca.svg"],
[ConnectionType.GOOGLE_CLOUD_SQL_MYSQL, "google_cloud_sql.svg"],
[ConnectionType.MANUAL_WEBHOOK, "manual_webhook.svg"],
[ConnectionType.MARIADB, "mariadb.svg"],
[ConnectionType.MONGODB, "mongodb.svg"],
Expand Down
1 change: 1 addition & 0 deletions clients/admin-ui/src/types/api/models/ConnectionType.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,4 +27,5 @@ export enum ConnectionType {
SCYLLA = "scylla",
GENERIC_ERASURE_EMAIL = "generic_erasure_email",
GENERIC_CONSENT_EMAIL = "generic_consent_email",
GOOGLE_CLOUD_SQL_MYSQL = "google_cloud_sql_mysql",
}
210 changes: 210 additions & 0 deletions data/dataset/google_cloud_sql_mysql_example_test_dataset.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,210 @@
dataset:
- fides_key: google_cloud_sql_mysql_example_test_dataset
name: Google Cloud SQL for MySQL Example Test Dataset
description: Example of a Google Cloud SQL MySQL dataset containing a variety of related tables like customers, products, addresses, etc.
collections:
- name: address
fields:
- name: city
data_categories: [user.contact.address.city]
- name: house
data_categories: [user.contact.address.street]
- name: id
data_categories: [system.operations]
fides_meta:
primary_key: True
- name: state
data_categories: [user.contact.address.state]
- name: street
data_categories: [user.contact.address.street]
- name: zip
data_categories: [user.contact.address.postal_code]

- name: customer
fields:
- name: address_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: address.id
direction: to
- name: created
data_categories: [system.operations]
- name: email
data_categories: [user.contact.email]
fides_meta:
identity: email
data_type: string
- name: id
data_categories: [user.unique_id]
fides_meta:
primary_key: True
- name: name
data_categories: [user.name]

- name: employee
fields:
- name: address_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: address.id
direction: to
- name: email
data_categories: [user.contact.email]
fides_meta:
identity: email
data_type: string
- name: id
data_categories: [user.unique_id]
fides_meta:
primary_key: True
- name: name
data_categories: [user.name]

- name: login
fields:
- name: customer_id
data_categories: [user.unique_id]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: customer.id
direction: from
- name: id
data_categories: [system.operations]
- name: time
data_categories: [user.sensor]

- name: orders
fields:
- name: customer_id
data_categories: [user.unique_id]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: customer.id
direction: from
- name: id
data_categories: [system.operations]
fides_meta:
primary_key: True
- name: shipping_address_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: address.id
direction: to

# order_item
- name: order_item
fields:
- name: order_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: orders.id
direction: from
- name: product_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: product.id
direction: to
- name: quantity
data_categories: [system.operations]

- name: payment_card
fields:
- name: billing_address_id
data_categories: [system.operations]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: address.id
direction: to
- name: ccn
data_categories: [user.financial.bank_account]
- name: code
data_categories: [user.financial]
- name: customer_id
data_categories: [user.unique_id]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: customer.id
direction: from
- name: id
data_categories: [system.operations]
- name: name
data_categories: [user.financial]
- name: preferred
data_categories: [user]

- name: product
fields:
- name: id
data_categories: [system.operations]
- name: name
data_categories: [system.operations]
- name: price
data_categories: [system.operations]

- name: report
fields:
- name: email
data_categories: [user.contact.email]
fides_meta:
identity: email
data_type: string
- name: id
data_categories: [system.operations]
- name: month
data_categories: [system.operations]
- name: name
data_categories: [system.operations]
- name: total_visits
data_categories: [system.operations]
- name: year
data_categories: [system.operations]

- name: service_request
fields:
- name: alt_email
data_categories: [user.contact.email]
fides_meta:
identity: email
data_type: string
- name: closed
data_categories: [system.operations]
- name: email
data_categories: [system.operations]
fides_meta:
identity: email
data_type: string
- name: employee_id
data_categories: [user.unique_id]
fides_meta:
references:
- dataset: google_cloud_sql_mysql_example_test_dataset
field: employee.id
direction: from
- name: id
data_categories: [system.operations]
- name: opened
data_categories: [system.operations]

- name: visit
fields:
- name: email
data_categories: [user.contact.email]
fides_meta:
identity: email
data_type: string
- name: last_visit
data_categories: [system.operations]
6 changes: 6 additions & 0 deletions noxfiles/run_infrastructure.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,12 @@
"DYNAMODB_ACCESS_KEY_ID",
"DYNAMODB_ACCESS_KEY",
],
"google_cloud_sql_mysql": [
"GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER",
"GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME",
"GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME",
"GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS",
],
}
EXTERNAL_DATASTORES = list(EXTERNAL_DATASTORE_CONFIG.keys())
ALL_DATASTORES = DOCKERFILE_DATASTORES + EXTERNAL_DATASTORES
Expand Down
8 changes: 8 additions & 0 deletions noxfiles/setup_tests_nox.py
Original file line number Diff line number Diff line change
Expand Up @@ -157,6 +157,14 @@ def pytest_ops(session: Session, mark: str, coverage_arg: str) -> None:
"-e",
"BIGQUERY_DATASET",
"-e",
"GOOGLE_CLOUD_SQL_MYSQL_DB_IAM_USER",
"-e",
"GOOGLE_CLOUD_SQL_MYSQL_INSTANCE_CONNECTION_NAME",
"-e",
"GOOGLE_CLOUD_SQL_MYSQL_DATABASE_NAME",
"-e",
"GOOGLE_CLOUD_SQL_MYSQL_KEYFILE_CREDS",
"-e",
"DYNAMODB_REGION",
"-e",
"DYNAMODB_ACCESS_KEY_ID",
Expand Down
8 changes: 5 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -47,27 +47,28 @@ module = [
"fideslog.*",
"firebase_admin.*",
"google.api_core.*",
"nh3.*",
"joblib.*",
"jose.*",
"jwt.*",
"multidimensional_urlencode.*",
"networkx.*",
"nh3.*",
"okta.*",
"pandas.*",
"plotly.*",
"pydash.*",
"pygtrie.*",
"pymongo.*",
"pymysql.*",
"RestrictedPython.*",
"sendgrid.*",
"snowflake.*",
"sqlalchemy_utils.*",
"sqlalchemy.ext.*",
"sqlalchemy.future.*",
"sqlalchemy_utils.*",
"twilio.*",
"uvicorn.*",
"validators.*",
"pygtrie.*",
]
ignore_missing_imports = true

Expand Down Expand Up @@ -185,6 +186,7 @@ markers = [
"integration_postgres",
"integration_mongodb",
"integration_mssql",
"integration_google_cloud_sql_mysql",
"integration_redshift",
"integration_snowflake",
"integration_mariadb",
Expand Down
3 changes: 2 additions & 1 deletion requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -5,8 +5,9 @@ APScheduler==3.9.1.post1
asyncpg==0.27.0
boto3==1.26.1
celery[pytest]==5.2.7
cloud-sql-python-connector==1.9.2
colorama>=0.4.3
cryptography==38.0.3
cryptography==42.0.0
dask==2022.9.2
deepdiff==6.3.0
defusedxml==0.7.1
Expand Down
Loading

0 comments on commit 39d23e1

Please sign in to comment.