From 19d66d76f23967e522870f8d8ea50098d4972d3e Mon Sep 17 00:00:00 2001 From: Andrew Jackson Date: Mon, 18 Jul 2022 09:59:58 -0400 Subject: [PATCH] 793 upgrade password hash (#876) * Update config.py * Update crypto imports to fideslib and remove legacy file and tests * Update Identity salt * checkpoint, lowercasing all of the things * update more config values to lowercase * fix linting errors, fix config validators * update the configs to use lowercase keys * lowercase the allowed keys * Update create_test_data.py * bump fideslib version * Lowercase config variables * Fix linting issues * Fix some test failures * Fix application fixtures * Remove old celery config options * Sort script imports * Update changelog Co-authored-by: Thomas --- CHANGELOG.md | 6 +- Makefile | 4 +- data/config/fidesops.toml | 56 +++++------ docs/fidesops/docs/guides/policy_webhooks.md | 44 ++++----- fidesops-integration.toml | 50 +++++----- fidesops.toml | 62 ++++++------ requirements.txt | 2 +- scripts/create_superuser.py | 8 +- scripts/create_test_data.py | 2 +- scripts/generate_openapi.py | 1 - scripts/mssql_discover.py | 12 +-- scripts/quickstart.py | 4 +- src/fidesops/analytics.py | 4 +- src/fidesops/api/deps.py | 4 +- .../api/v1/endpoints/drp_endpoints.py | 2 +- .../api/v1/endpoints/encryption_endpoints.py | 10 +- .../api/v1/endpoints/health_endpoints.py | 6 +- .../api/v1/endpoints/oauth_endpoints.py | 6 +- .../v1/endpoints/privacy_request_endpoints.py | 4 +- .../api/v1/endpoints/user_endpoints.py | 2 +- src/fidesops/core/config.py | 98 +++++++++---------- src/fidesops/main.py | 22 ++--- src/fidesops/migrations/env.py | 4 +- .../55d61eb8ed12_add_default_policies.py | 20 ++-- src/fidesops/models/connectionconfig.py | 2 +- src/fidesops/models/policy.py | 2 +- src/fidesops/models/privacy_request.py | 6 +- src/fidesops/models/storage.py | 2 +- src/fidesops/schemas/encryption_request.py | 2 +- src/fidesops/schemas/privacy_request.py | 2 +- .../service/connectors/saas_query_config.py | 2 +- .../masking/strategy/masking_strategy_hash.py | 4 +- .../privacy_request/request_runner_service.py | 2 +- src/fidesops/task/graph_task.py | 6 +- src/fidesops/tasks/__init__.py | 6 +- src/fidesops/tasks/storage.py | 12 +-- src/fidesops/util/cache.py | 18 ++-- src/fidesops/util/cryptographic_util.py | 46 --------- .../encryption/aes_gcm_encryption_scheme.py | 20 ++-- .../util/encryption/hmac_encryption_scheme.py | 4 +- src/fidesops/util/oauth_util.py | 8 +- tests/api/test_deps.py | 12 +-- .../api/v1/endpoints/test_config_endpoints.py | 6 +- tests/api/v1/endpoints/test_drp_endpoints.py | 16 +-- .../v1/endpoints/test_encryption_endpoints.py | 6 +- .../api/v1/endpoints/test_oauth_endpoints.py | 24 ++--- .../test_privacy_request_endpoints.py | 16 +-- tests/api/v1/endpoints/test_user_endpoints.py | 20 ++-- .../test_user_permission_endpoints.py | 4 +- tests/api/v1/test_exception_handlers.py | 12 +-- tests/conftest.py | 16 +-- tests/core/test_config.py | 24 ++--- tests/fixtures/application_fixtures.py | 48 ++++----- tests/fixtures/saas/hubspot_fixtures.py | 2 +- tests/fixtures/saas/outreach_fixtures.py | 2 +- tests/fixtures/saas/salesforce_fixtures.py | 2 +- tests/fixtures/saas/zendesk_fixtures.py | 2 +- tests/graph/test_graph.py | 6 +- .../saas/test_outreach_task.py | 4 +- .../saas/test_segment_task.py | 4 +- .../saas/test_stripe_task.py | 4 +- .../saas/test_zendesk_task.py | 4 +- tests/integration_tests/test_sql_task.py | 12 +-- tests/models/test_client.py | 12 +-- .../test_authentication_strategy_basic.py | 2 +- tests/service/connectors/test_queryconfig.py | 6 +- .../request_runner_service_test.py | 2 +- tests/tasks/test_celery.py | 4 +- .../test_aes_gcm_encryption_scheme.py | 2 +- tests/util/test_cache.py | 4 +- tests/util/test_cryptographic_util.py | 47 --------- tests/util/test_jwt_util.py | 16 +-- 72 files changed, 409 insertions(+), 509 deletions(-) delete mode 100644 src/fidesops/util/cryptographic_util.py delete mode 100644 tests/util/test_cryptographic_util.py diff --git a/CHANGELOG.md b/CHANGELOG.md index d6c0cb56c4..d6ff704632 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,9 +18,11 @@ The types of changes are: ## [Unreleased](https://github.com/ethyca/fidesops/compare/1.6.3...main) -### Added +### Breaking Changes + +* Update fidesops to use bcrypt for hashing [#876](https://github.com/ethyca/fidesops/pull/876) + -* ... ## [1.6.3](https://github.com/ethyca/fidesops/compare/1.6.2...1.6.3) diff --git a/Makefile b/Makefile index c50e24951e..cf6a171608 100644 --- a/Makefile +++ b/Makefile @@ -36,7 +36,7 @@ init-db: compose-build python -c "\ from fidesops.db.database import init_db; \ from fidesops.core.config import config; \ - init_db(config.database.SQLALCHEMY_DATABASE_URI);" + init_db(config.database.sqlalchemy_database_uri);" reset-db: @echo "Resetting and re-initializing the application db..." @@ -112,7 +112,7 @@ check-migrations: compose-build python -c "\ from fidesops.db.database import check_missing_migrations; \ from fidesops.core.config import config; \ - check_missing_migrations(config.database.SQLALCHEMY_DATABASE_URI);" + check_missing_migrations(config.database.sqlalchemy_database_uri);" @make teardown isort-ci: diff --git a/data/config/fidesops.toml b/data/config/fidesops.toml index 551a2d0e1b..49a64bb02a 100644 --- a/data/config/fidesops.toml +++ b/data/config/fidesops.toml @@ -1,40 +1,40 @@ -PORT = 8080 +port = 8080 [database] -SERVER = "testserver" -USER = "testuser" -PASSWORD = "testpassword" -DB = "testdb" -TEST_DB = "test_testdb" -ENABLED = true +server = "testserver" +user = "testuser" +password = "testpassword" +db = "testdb" +test_db = "test_testdb" +enabled = true [redis] -HOST = "testredis" -PASSWORD = "testpassword" -PORT = 1234 -CHARSET = "utf8" -DEFAULT_TTL_SECONDS = 1000 -DB_INDEX = 0 -ENABLED = true +host = "testredis" +password = "testpassword" +port = 1234 +charset = "utf8" +default_ttl_seconds = 1000 +db_index = 0 +enabled = true [security] -APP_ENCRYPTION_KEY = "atestencryptionkeythatisvalidlen" -CORS_ORIGINS = [ "http://test.com", "https://test.com",] -OAUTH_ROOT_CLIENT_ID = "testrootclientid" -OAUTH_ROOT_CLIENT_SECRET = "testrootclientsecret" -DRP_JWT_SECRET = "testdrpsecret" -LOG_LEVEL = "DEBUG" +app_encryption_key = "atestencryptionkeythatisvalidlen" +cors_origins = [ "http://test.com", "https://test.com",] +oauth_root_client_id = "testrootclientid" +oauth_root_client_secret = "testrootclientsecret" +drp_jwt_secret = "testdrpsecret" +log_level = "DEBUG" [execution] -TASK_RETRY_COUNT = 0 -TASK_RETRY_DELAY = 1 -TASK_RETRY_BACKOFF = 1 -REQUIRE_MANUAL_REQUEST_APPROVAL = false -MASKING_STRICT = true +task_retry_count = 0 +task_retry_delay = 1 +task_retry_backoff = 1 +require_manual_request_approval = false +masking_strict = true [root_user] -ANALYTICS_OPT_OUT = true -ANALYTICS_ID = "internal" +analytics_opt_out = true +analytics_id = "internal" [admin_ui] -ENABLED = true +enabled = true diff --git a/docs/fidesops/docs/guides/policy_webhooks.md b/docs/fidesops/docs/guides/policy_webhooks.md index ed763413a5..81b4aacbb6 100644 --- a/docs/fidesops/docs/guides/policy_webhooks.md +++ b/docs/fidesops/docs/guides/policy_webhooks.md @@ -5,15 +5,14 @@ A Policy webhook is an HTTPS Callback that you've defined on a Policy to call an external REST API endpoint *before* or *after* a Privacy Request executes. -You can define as many webhooks as you'd like. Webhooks can be `one_way`, where we will just ping your API and move on, -or `two_way`, where we will wait for a response. Any `derived_identities` returned from a `two_way` webhook will be saved +You can define as many webhooks as you'd like. Webhooks can be `one_way`, where we will just ping your API and move on, +or `two_way`, where we will wait for a response. Any `derived_identities` returned from a `two_way` webhook will be saved and can be used to locate other user information. For example, a webhook might take a known `email` `identity` and use that to find a `phone_number` `derived)identity`. Another use case for a Policy Webhook might be to log a user out of your mobile app after you've cleared their data from your system. In this case, you'd create a `Policy` and a `ConnectionConfig` to describe the URL to hit -to clear the cache. You'd then create a `one-way` `PolicyPostWebhook` to run after your PrivacyRequest executes. - +to clear the cache. You'd then create a `one-way` `PolicyPostWebhook` to run after your PrivacyRequest executes. ## Configuration @@ -24,7 +23,7 @@ You will then define a `PolicyPreWebhook` or a `PolicyPostWebhook`for a specific The information that describes how to connect to your API endpoint lives on a `ConnectionConfig`. We also use `ConnectionConfigs` to connect to databases like `PostgreSQL` and `MongoDB`. This same construct can help us store -how to connect to an external API endpoint. +how to connect to an external API endpoint. For more information on ConnectionConfigs, see how to [Create a ConnectionConfig.](/fidesops/api/#operations-Connections-put_connections_api_v1_connection_put) @@ -54,11 +53,12 @@ See API docs on how to [Set a ConnectionConfig's Secrets](/fidesops/api#operatio ``` ### Define pre-execution or post-execution webhooks + After you've defined a `ConnectionConfig`, you can create lists of webhooks to run *before* (`PolicyPreWebhooks`) or *after* (`PolicyPostWebhooks`) a PrivacyRequest is executed. If you are defining PolicyPreWebhooks, all desired PolicyPreWebhooks should be included in the request -body in the desired order. Any PolicyPreWebhooks on the Policy *not* included in the request, will be removed from the +body in the desired order. Any PolicyPreWebhooks on the Policy *not* included in the request, will be removed from the Policy. The same applies for PolicyPostWebhooks. To update your list of PolicyPreWebhooks: @@ -88,19 +88,17 @@ Similarly, to update your list of Post-Execution webhooks on a Policy: PUT /policy//webhook/post_execution ``` - See API docs for more information on how to [Update PolicyPreWebhooks](/fidesops/api#operations-Policy_Webhooks-create_or_update_pre_execution_webhooks_api_v1_policy__policy_key__webhook_pre_execution_put) and how to [Update PolicyPostWebhooks](/fidesops/api#operations-Policy_Webhooks-create_or_update_post_execution_webhooks_api_v1_policy__policy_key__webhook_post_execution_put). - ### Update a single webhook To update a single PolicyPreWebhook or PolicyPostWebhook, send a PATCH request to update selected attributes. Note that updates to order can likewise update the order of related webhooks. -The following example will update the PolicyPreWebhook with key `webhook_hook` to be `two_way` instead of -`one_way` and will update its order from 0 to 1. Because we've defined two PolicyPreWebhooks, this causes the -webhook at position 1 to move to position 0. +The following example will update the PolicyPreWebhook with key `webhook_hook` to be `two_way` instead of +`one_way` and will update its order from 0 to 1. Because we've defined two PolicyPreWebhooks, this causes the +webhook at position 1 to move to position 0. ```json title="PATCH /policy//webhook/pre-execution/wake_up_snowflake_db" { @@ -109,8 +107,9 @@ webhook at position 1 to move to position 0. } ``` -Because this PATCH request updated the order of other webhooks, a reordered summary is included under the +Because this PATCH request updated the order of other webhooks, a reordered summary is included under the `new_order` attribute: + ```json title="Response" { "resource": { @@ -142,10 +141,8 @@ PATCH /policy//webhook/post_execution/ See API docs for more information on how to [PATCH a PolicyPreWebhook](/fidesops/api#operations-Policy_Webhooks-update_pre_execution_webhook_api_v1_policy__policy_key__webhook_pre_execution__pre_webhook_key__patch) and how to [PATCH a PolicyPostWebhook](/fidesops/api#operations-Policy_Webhooks-update_post_execution_webhook_api_v1_policy__policy_key__webhook_post_execution__post_webhook_key__patch). - ## Webhook request format - Before and after running access or erasure requests, fidesops will send requests to any configured webhooks in sequential order with the following request body: @@ -161,11 +158,11 @@ with the following request body: } ``` -Most of these attributes were configured by you: the `direction`, the `callback_type` ("pre" for `PolicyPreWebhook`s that will run +Most of these attributes were configured by you: the `direction`, the `callback_type` ("pre" for `PolicyPreWebhook`s that will run before PrivacyRequest execution or "post" for `PolicyPostWebhook`s that will run after PrivacyRequestExecution). Known identities are also embedded in the request. -For `two-way` `PolicyPreWebhooks`, we include specific headers in case you need to pause PrivacyRequest +For `two-way` `PolicyPreWebhooks`, we include specific headers in case you need to pause PrivacyRequest execution while you take care of additional processing on your end. ```json @@ -174,18 +171,19 @@ execution while you take care of additional processing on your end. "reply-to-token": "" } ``` + To resume, you should send a request back to the `reply-to` URL with the `reply-to-token`. The `reply-to-token` will -expire when your redis cache expires: `config.redis.DEFAULT_TTL_SECONDS` (Ffdesops uses the redis cache to temporarily +expire when your redis cache expires: `config.redis.default_ttl_seconds` (Fidesops uses the redis cache to temporarily store identity data). At this point, your PrivacyRequest will be given an `error` status, and you would have to resubmit the PrivacyRequest. ## Webhook response format -Your webhook should respond immediately. If more processing time is needed, either make sure it is configured as a +Your webhook should respond immediately. If more processing time is needed, either make sure it is configured as a `one-way` webhook, or reply with `halt=True` if you want to pause execution and wait for your processing to finish. -Note that only `PolicyPreWebhooks` can pause execution. +Note that only `PolicyPreWebhooks` can pause execution. -We don't expect a response from `one-way` webhooks, but `two-way` webhooks should respond with the following: +We don't expect a response from `one-way` webhooks, but `two-way` webhooks should respond with the following: ```json { @@ -201,7 +199,7 @@ Derived identity is optional: a returned email or phone number will replace curr ## Resuming request execution -If your webhook needed more processing time, once completed, send a request to the `reply-to` URL +If your webhook needed more processing time, once completed, send a request to the `reply-to` URL given to you in the original request header with the `reply-to-token` auth token. ```json title="POST privacy_request//resume" @@ -216,6 +214,6 @@ given to you in the original request header with the `reply-to-token` auth token If there are no derived identities, an empty `{}` request body will suffice. -The `reply-to-token` is a JWE containing the current webhook id, scopes to access the callback endpoint, -and the datetime the token is issued. We unpack this and resume the privacy request execution after the +The `reply-to-token` is a JWE containing the current webhook id, scopes to access the callback endpoint, +and the datetime the token is issued. We unpack this and resume the privacy request execution after the specified webhook. The `reply-to-token` expires after a set amount of time, specified by the `config.execution.PRIVACY_REQUEST_DELAY_TIMEOUT` config variable. Once the redis cache expires, fidesops no longer has the original identity data and the privacy request should be resubmitted. diff --git a/fidesops-integration.toml b/fidesops-integration.toml index d71c06da8f..851770596b 100644 --- a/fidesops-integration.toml +++ b/fidesops-integration.toml @@ -1,23 +1,23 @@ [postgres_example] -SERVER="postgres_example" -USER="postgres" -PASSWORD="postgres" -DB="postgres_example" -PORT= 5432 +server="postgres_example" +user="postgres" +password="postgres" +db="postgres_example" +port= 5432 [mongodb_example] -SERVER= "mongodb_example" -USER="mongo_user" -PASSWORD="mongo_pass" -DB="mongo_test" -PORT=27017 +server= "mongodb_example" +user="mongo_user" +password="mongo_pass" +db="mongo_test" +port=27017 [mysql_example] -SERVER="mysql_example" -USER="mysql_user" -PASSWORD="mysql_pw" -DB="mysql_example" -PORT=3306 +server="mysql_example" +user="mysql_user" +password="mysql_pw" +db="mysql_example" +port=3306 [redshift] external_uri="" @@ -32,15 +32,15 @@ dataset="" [bigquery.keyfile_creds] [mssql_example] -SERVER="mssql_example" -USER="sa" -PASSWORD="Mssql_pw1" -DB="mssql_example" -PORT=1433 +server="mssql_example" +user="sa" +password="Mssql_pw1" +db="mssql_example" +port=1433 [mariadb_example] -SERVER="mariadb_example" -USER="mariadb_user" -PASSWORD="mariadb_pw" -DB="mariadb_example" -PORT=3306 +server="mariadb_example" +user="mariadb_user" +password="mariadb_pw" +db="mariadb_example" +port=3306 diff --git a/fidesops.toml b/fidesops.toml index 1ce87750cf..5c013bc5ca 100644 --- a/fidesops.toml +++ b/fidesops.toml @@ -1,43 +1,43 @@ -PORT = 8080 +port = 8080 [database] -SERVER = "db" -USER = "postgres" -PASSWORD = "216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a" -DB = "app" -TEST_DB = "test" -ENABLED = true +server = "db" +user = "postgres" +password = "216f4b49bea5da4f84f05288258471852c3e325cd336821097e1e65ff92b528a" +db = "app" +test_db = "test" +enabled = true [redis] -HOST = "redis" -PASSWORD = "testpassword" -PORT = 6379 -CHARSET = "utf8" -DEFAULT_TTL_SECONDS = 604800 -DB_INDEX = 0 -ENABLED = true -SSL = false -SSL_CERT_REQS = "required" +host = "redis" +password = "testpassword" +port = 6379 +charset = "utf8" +default_ttl_seconds = 604800 +db_index = 0 +enabled = true +ssl = false +ssl_cert_reqs = "required" [security] -APP_ENCRYPTION_KEY = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" -CORS_ORIGINS = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",] -ENCODING = "UTF-8" -OAUTH_ROOT_CLIENT_ID = "fidesopsadmin" -OAUTH_ROOT_CLIENT_SECRET = "fidesopsadminsecret" -DRP_JWT_SECRET = "secret" -LOG_LEVEL = "INFO" +app_encryption_key = "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" +cors_origins = [ "http://localhost", "http://localhost:8080", "http://localhost:3000", "http://localhost:3001",] +encoding = "UTF-8" +oauth_root_client_id = "fidesopsadmin" +oauth_root_client_secret = "fidesopsadminsecret" +drp_jwt_secret = "secret" +log_level = "INFO" [execution] -MASKING_STRICT = true -REQUIRE_MANUAL_REQUEST_APPROVAL = false -TASK_RETRY_COUNT = 0 -TASK_RETRY_DELAY = 1 -TASK_RETRY_BACKOFF = 1 -WORKER_ENABLED = false +masking_strict = true +require_manual_request_approval = false +task_retry_count = 0 +task_retry_delay = 1 +task_retry_backoff = 1 +worker_enabled = false [root_user] -ANALYTICS_OPT_OUT = false +analytics_opt_out = false [admin_ui] -ENABLED = true +enabled = true diff --git a/requirements.txt b/requirements.txt index 53552004b6..ba2bd50c0a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ fastapi-caching[redis] fastapi-pagination[sqlalchemy]~= 0.9.3 fastapi[all]==0.78.0 fideslang==1.0.0 -fideslib==2.2.2 +fideslib==3.0.0 fideslog==1.2.1 multidimensional_urlencode==0.0.4 pandas==1.3.3 diff --git a/scripts/create_superuser.py b/scripts/create_superuser.py index ff7eb60919..1ca95a9490 100644 --- a/scripts/create_superuser.py +++ b/scripts/create_superuser.py @@ -3,6 +3,7 @@ import getpass from typing import List +from fideslib.cryptography.cryptographic_util import str_to_b64_str from fideslib.db.session import get_db_session from fideslib.exceptions import KeyOrNameAlreadyExists from fideslib.models.client import ADMIN_UI_ROOT, ClientDetail @@ -14,7 +15,6 @@ from fidesops.api.v1.scope_registry import CLIENT_CREATE, SCOPE_REGISTRY from fidesops.core.config import config from fidesops.db.database import init_db -from fidesops.util.cryptographic_util import str_to_b64_str def get_username(prompt: str) -> str: @@ -79,8 +79,8 @@ def create_user_and_client(db: Session) -> FidesUser: ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, scopes=scopes, fides_key=ADMIN_UI_ROOT, user_id=superuser.id, @@ -92,7 +92,7 @@ def create_user_and_client(db: Session) -> FidesUser: if __name__ == "__main__": - init_db(config.database.SQLALCHEMY_DATABASE_URI) + init_db(config.database.sqlalchemy_database_uri) session_local = get_db_session(config) with session_local() as session: create_user_and_client(session) diff --git a/scripts/create_test_data.py b/scripts/create_test_data.py index e243e8bd72..5c55284590 100644 --- a/scripts/create_test_data.py +++ b/scripts/create_test_data.py @@ -201,7 +201,7 @@ def create_test_data(db: orm.Session) -> FidesUser: if __name__ == "__main__": - init_db(config.database.SQLALCHEMY_DATABASE_URI) + init_db(config.database.sqlalchemy_database_uri) session_local = get_db_session(config) with session_local() as session: create_test_data(session) diff --git a/scripts/generate_openapi.py b/scripts/generate_openapi.py index 1e7944433f..025b9d1a46 100644 --- a/scripts/generate_openapi.py +++ b/scripts/generate_openapi.py @@ -11,7 +11,6 @@ from fidesops.main import app - if __name__ == "__main__": outfile_path = "openapi.json" diff --git a/scripts/mssql_discover.py b/scripts/mssql_discover.py index 1ac047d614..e1f773e5d2 100644 --- a/scripts/mssql_discover.py +++ b/scripts/mssql_discover.py @@ -1,14 +1,8 @@ -import sqlalchemy - # This file is not committed to the repo, please create secrets.py with the required # variables in the same dir as this file before running this script -from secrets import ( - USER, - PASS, - IP, - PORT, - DB, -) +from secrets import DB, IP, PASS, PORT, USER + +import sqlalchemy MASTER_MSSQL_URL = f"mssql+pyodbc://{USER}:{PASS}@{IP}:{PORT}/{DB}?driver=ODBC+Driver+17+for+SQL+Server" diff --git a/scripts/quickstart.py b/scripts/quickstart.py index cf5f023fca..b643d517a6 100644 --- a/scripts/quickstart.py +++ b/scripts/quickstart.py @@ -540,8 +540,8 @@ def print_results(request_id: str) -> None: input() root_token = get_access_token( - client_id=config.security.OAUTH_ROOT_CLIENT_ID, - client_secret=config.security.OAUTH_ROOT_CLIENT_SECRET, + client_id=config.security.oauth_root_client_id, + client_secret=config.security.oauth_root_client_secret, ) root_oauth_header = {"Authorization": f"Bearer {root_token}"} client = create_oauth_client() diff --git a/src/fidesops/analytics.py b/src/fidesops/analytics.py index 3370b44cfe..0533fd3164 100644 --- a/src/fidesops/analytics.py +++ b/src/fidesops/analytics.py @@ -26,7 +26,7 @@ def accessed_through_local_host(hostname: Optional[str]) -> bool: analytics_client = AnalyticsClient( - client_id=config.root_user.ANALYTICS_ID, + client_id=config.root_user.analytics_id, developer_mode=config.dev_mode, extra_data=None, os=system(), @@ -36,7 +36,7 @@ def accessed_through_local_host(hostname: Optional[str]) -> bool: def send_analytics_event(event: AnalyticsEvent) -> None: - if config.root_user.ANALYTICS_OPT_OUT: + if config.root_user.analytics_opt_out: return try: analytics_client.send(event) diff --git a/src/fidesops/api/deps.py b/src/fidesops/api/deps.py index 136b9d0bb5..c9439ba0c5 100644 --- a/src/fidesops/api/deps.py +++ b/src/fidesops/api/deps.py @@ -9,7 +9,7 @@ def get_db() -> Generator: """Return our database session""" - if not config.database.ENABLED: + if not config.database.enabled: raise FunctionalityNotConfigured( "Application database required, but it is currently disabled! Please update your application configuration to enable integration with an application database." ) @@ -23,7 +23,7 @@ def get_db() -> Generator: def get_cache() -> Generator: """Return a connection to our redis cache""" - if not config.redis.ENABLED: + if not config.redis.enabled: raise FunctionalityNotConfigured( "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." ) diff --git a/src/fidesops/api/v1/endpoints/drp_endpoints.py b/src/fidesops/api/v1/endpoints/drp_endpoints.py index 9bba9623e9..6bc0a9a074 100644 --- a/src/fidesops/api/v1/endpoints/drp_endpoints.py +++ b/src/fidesops/api/v1/endpoints/drp_endpoints.py @@ -65,7 +65,7 @@ def create_drp_privacy_request( a corresponding Fidesops PrivacyRequest """ - jwt_key: str = config.security.DRP_JWT_SECRET + jwt_key: str = config.security.drp_jwt_secret if jwt_key is None: raise HTTPException( status_code=HTTP_500_INTERNAL_SERVER_ERROR, diff --git a/src/fidesops/api/v1/endpoints/encryption_endpoints.py b/src/fidesops/api/v1/endpoints/encryption_endpoints.py index e639210dc0..ac72020f11 100644 --- a/src/fidesops/api/v1/endpoints/encryption_endpoints.py +++ b/src/fidesops/api/v1/endpoints/encryption_endpoints.py @@ -2,6 +2,8 @@ import secrets from fastapi import Security +from fideslib.cryptography import cryptographic_util +from fideslib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC from fidesops.api.v1.urn_registry import ( @@ -17,9 +19,7 @@ AesEncryptionRequest, AesEncryptionResponse, ) -from fidesops.util import cryptographic_util from fidesops.util.api_router import APIRouter -from fidesops.util.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from fidesops.util.encryption.aes_gcm_encryption_scheme import ( decrypt as aes_gcm_decrypt, ) @@ -42,7 +42,7 @@ def get_encryption_key() -> str: logger.info("Generating encryption key") return cryptographic_util.generate_secure_random_string( - config.security.AES_ENCRYPTION_KEY_LENGTH + config.security.aes_encryption_key_length ) @@ -53,7 +53,7 @@ def get_encryption_key() -> str: ) def aes_encrypt(encryption_request: AesEncryptionRequest) -> AesEncryptionResponse: logger.info("Starting AES Encryption") - nonce: bytes = secrets.token_bytes(config.security.AES_GCM_NONCE_LENGTH) + nonce: bytes = secrets.token_bytes(config.security.aes_gcm_nonce_length) encrypted_value: str = aes_gcm_encrypt( encryption_request.value, @@ -76,7 +76,7 @@ def aes_decrypt(decryption_request: AesDecryptionRequest) -> AesDecryptionRespon decrypted_value: str = aes_gcm_decrypt( decryption_request.value, - decryption_request.key.encode(config.security.ENCODING), + decryption_request.key.encode(config.security.encoding), nonce, ) return AesDecryptionResponse(decrypted_value=decrypted_value) diff --git a/src/fidesops/api/v1/endpoints/health_endpoints.py b/src/fidesops/api/v1/endpoints/health_endpoints.py index 381320fcb0..e29e6fd4d7 100644 --- a/src/fidesops/api/v1/endpoints/health_endpoints.py +++ b/src/fidesops/api/v1/endpoints/health_endpoints.py @@ -24,14 +24,14 @@ def health_check() -> Dict[str, Union[bool, str]]: return { "webserver": "healthy", - "database": get_db_health(config.database.SQLALCHEMY_DATABASE_URI), + "database": get_db_health(config.database.sqlalchemy_database_uri), "cache": get_cache_health(), } def get_db_health(database_url: Optional[str]) -> str: """Checks if the db is reachable and up to date in alembic migrations""" - if not database_url or not config.database.ENABLED: + if not database_url or not config.database.enabled: return "no db configured" try: engine = create_engine(database_url) @@ -52,7 +52,7 @@ def get_db_health(database_url: Optional[str]) -> str: def get_cache_health() -> str: """Checks if the cache is reachable""" - if not config.redis.ENABLED: + if not config.redis.enabled: return "no cache configured" try: get_cache() diff --git a/src/fidesops/api/v1/endpoints/oauth_endpoints.py b/src/fidesops/api/v1/endpoints/oauth_endpoints.py index eb9147757e..06c8128699 100644 --- a/src/fidesops/api/v1/endpoints/oauth_endpoints.py +++ b/src/fidesops/api/v1/endpoints/oauth_endpoints.py @@ -90,7 +90,7 @@ async def acquire_access_token( logger.info("Creating access token") access_code = client_detail.create_access_code_jwe( - config.security.APP_ENCRYPTION_KEY + config.security.app_encryption_key ) return AccessToken(access_token=access_code) @@ -115,8 +115,8 @@ def create_client( client, secret = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, scopes=scopes, ) return ClientCreatedResponse(client_id=client.id, client_secret=secret) diff --git a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py index bdc85ebf1f..339092ce3d 100644 --- a/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py +++ b/src/fidesops/api/v1/endpoints/privacy_request_endpoints.py @@ -128,7 +128,7 @@ def create_privacy_request( You cannot update privacy requests after they've been created. """ - if not config.redis.ENABLED: + if not config.redis.enabled: raise FunctionalityNotConfigured( "Application redis cache required, but it is currently disabled! Please update your application configuration to enable integration with a redis cache." ) @@ -193,7 +193,7 @@ def create_privacy_request( None, ) - if not config.execution.REQUIRE_MANUAL_REQUEST_APPROVAL: + if not config.execution.require_manual_request_approval: queue_privacy_request(privacy_request.id) except common_exceptions.RedisConnectionError as exc: diff --git a/src/fidesops/api/v1/endpoints/user_endpoints.py b/src/fidesops/api/v1/endpoints/user_endpoints.py index e7cdac0cd7..11bbae3a08 100644 --- a/src/fidesops/api/v1/endpoints/user_endpoints.py +++ b/src/fidesops/api/v1/endpoints/user_endpoints.py @@ -86,7 +86,7 @@ def update_user_password( _validate_current_user(user_id, current_user) if not current_user.credentials_valid( - b64_str_to_str(data.old_password), config.security.ENCODING + b64_str_to_str(data.old_password), config.security.encoding ): raise HTTPException( status_code=HTTP_401_UNAUTHORIZED, detail="Incorrect password." diff --git a/src/fidesops/core/config.py b/src/fidesops/core/config.py index eeb748b43d..15da707e5e 100644 --- a/src/fidesops/core/config.py +++ b/src/fidesops/core/config.py @@ -33,14 +33,14 @@ class Config: class ExecutionSettings(FidesSettings): """Configuration settings for execution.""" - PRIVACY_REQUEST_DELAY_TIMEOUT: int = 3600 - TASK_RETRY_COUNT: int - TASK_RETRY_DELAY: int # In seconds - TASK_RETRY_BACKOFF: int - REQUIRE_MANUAL_REQUEST_APPROVAL: bool = False - MASKING_STRICT: bool = True - WORKER_ENABLED: bool = True - CELERY_CONFIG_PATH: Optional[str] = "celery.toml" + privacy_request_delay_timeout: int = 3600 + task_retry_count: int + task_retry_delay: int # In seconds + task_retry_backoff: int + require_manual_request_approval: bool = False + masking_strict: bool = True + worker_enabled: bool = True + celery_config_path: Optional[str] = "celery.toml" class Config: env_prefix = "FIDESOPS__EXECUTION__" @@ -49,20 +49,20 @@ class Config: class RedisSettings(FidesSettings): """Configuration settings for Redis.""" - HOST: str - PORT: int = 6379 - USER: Optional[str] = "" - PASSWORD: str - CHARSET: str = "utf8" - DECODE_RESPONSES: bool = True - DEFAULT_TTL_SECONDS: int = 604800 - DB_INDEX: Optional[int] - ENABLED: bool = True - SSL: bool = False - SSL_CERT_REQS: Optional[str] = "required" - CONNECTION_URL: Optional[str] = None - - @validator("CONNECTION_URL", pre=True) + host: str + port: int = 6379 + user: Optional[str] = "" + password: str + charset: str = "utf8" + decode_responses: bool = True + default_ttl_seconds: int = 604800 + db_index: Optional[int] + enabled: bool = True + ssl: bool = False + ssl_cert_reqs: Optional[str] = "required" + connection_url: Optional[str] = None + + @validator("connection_url", pre=True) @classmethod def assemble_connection_url( cls, @@ -74,7 +74,7 @@ def assemble_connection_url( # If the whole URL is provided via the config, preference that return v - return f"redis://{values.get('USER', '')}:{values['PASSWORD']}@{values['HOST']}:{values['PORT']}/{values.get('DB_INDEX', '')}" + return f"redis://{values.get('user', '')}:{values['password']}@{values['host']}:{values['port']}/{values.get('db_index', '')}" class Config: env_prefix = "FIDESOPS__REDIS__" @@ -83,9 +83,9 @@ class Config: class FidesopsSecuritySettings(SecuritySettings): """Configuration settings for Security variables.""" - LOG_LEVEL: str = "INFO" + log_level: str = "INFO" - @validator("LOG_LEVEL", pre=True) + @validator("log_level", pre=True) def validate_log_level(cls, value: str) -> str: """Ensure the provided LOG_LEVEL is a valid value.""" valid_values = [ @@ -113,10 +113,10 @@ class Config: class RootUserSettings(FidesSettings): """Configuration settings for Analytics variables.""" - ANALYTICS_OPT_OUT: Optional[bool] - ANALYTICS_ID: Optional[str] + analytics_opt_out: Optional[bool] + analytics_id: Optional[str] - @validator("ANALYTICS_ID", pre=True) + @validator("analytics_id", pre=True) def populate_analytics_id(cls, v: Optional[str]) -> str: """ Populates the appropriate value for analytics id based on config @@ -128,7 +128,7 @@ def generate_and_store_client_id() -> str: update_obj: Dict[str, Dict] = {} client_id: str = generate_client_id(FIDESOPS) logger.debug("analytics client id generated") - update_obj.update(root_user={"ANALYTICS_ID": client_id}) + update_obj.update(root_user={"analytics_id": client_id}) update_config_file(update_obj) return client_id @@ -139,7 +139,7 @@ class Config: class AdminUiSettings(FidesSettings): """Configuration settings for Analytics variables.""" - ENABLED: bool + enabled: bool class Config: env_prefix = "FIDESOPS__ADMIN_UI__" @@ -155,7 +155,7 @@ class FidesopsConfig(FidesSettings): root_user: RootUserSettings admin_ui: AdminUiSettings - PORT: int + port: int is_test_mode: bool = os.getenv("TESTING", "").lower() == "true" hot_reloading: bool = os.getenv("FIDESOPS__HOT_RELOAD", "").lower() == "true" dev_mode: bool = os.getenv("FIDESOPS__DEV_MODE", "").lower() == "true" @@ -190,30 +190,30 @@ def log_all_config_values(self) -> None: CONFIG_KEY_ALLOWLIST = { "database": [ - "SERVER", - "USER", - "PORT", - "DB", - "TEST_DB", + "server", + "user", + "port", + "db", + "test_db", ], "redis": [ - "HOST", - "PORT", - "CHARSET", - "DECODE_RESPONSES", - "DEFAULT_TTL_SECONDS", - "DB_INDEX", + "host", + "port", + "charset", + "decode_responses", + "default_ttl_seconds", + "db_index", ], "security": [ - "CORS_ORIGINS", - "ENCODING", - "OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES", + "cors_origins", + "encoding", + "oauth_access_token_expire_minutes", ], "execution": [ - "TASK_RETRY_COUNT", - "TASK_RETRY_DELAY", - "TASK_RETRY_BACKOFF", - "REQUIRE_MANUAL_REQUEST_APPROVAL", + "task_retry_count", + "task_retry_delay", + "task_retry_backoff", + "require_manual_request_approval", ], } diff --git a/src/fidesops/main.py b/src/fidesops/main.py index f282150e14..5685d6cbe8 100644 --- a/src/fidesops/main.py +++ b/src/fidesops/main.py @@ -36,17 +36,17 @@ from fidesops.util.logger import get_fides_log_record_factory from fidesops.util.oauth_util import get_db, verify_oauth_client -logging.basicConfig(level=config.security.LOG_LEVEL) +logging.basicConfig(level=config.security.log_level) logging.setLogRecordFactory(get_fides_log_record_factory()) logger = logging.getLogger(__name__) app = FastAPI(title="fidesops", openapi_url=f"{V1_URL_PREFIX}/openapi.json") # Set all CORS enabled origins -if config.security.CORS_ORIGINS: +if config.security.cors_origins: app.add_middleware( CORSMiddleware, - allow_origins=[str(origin) for origin in config.security.CORS_ORIGINS], + allow_origins=[str(origin) for origin in config.security.cors_origins], allow_credentials=True, allow_methods=["*"], allow_headers=["*"], @@ -101,7 +101,7 @@ def prepare_and_log_request( """ # this check prevents AnalyticsEvent from being called with invalid endpoint during unit tests - if config.root_user.ANALYTICS_OPT_OUT: + if config.root_user.analytics_opt_out: return send_analytics_event( AnalyticsEvent( @@ -129,7 +129,7 @@ def prepare_and_log_request( WEBAPP_DIRECTORY = Path("src/fidesops/build/static") WEBAPP_INDEX = WEBAPP_DIRECTORY / "index.html" -if config.admin_ui.ENABLED: +if config.admin_ui.enabled: @app.on_event("startup") def check_if_admin_ui_index_exists() -> None: @@ -175,15 +175,15 @@ def start_webserver() -> None: ) config.log_all_config_values() - if config.database.ENABLED: + if config.database.enabled: logger.info("Running any pending DB migrations...") try: - init_db(config.database.SQLALCHEMY_DATABASE_URI) + init_db(config.database.sqlalchemy_database_uri) except Exception as error: # pylint: disable=broad-except logger.error(f"Connection to database failed: {error}") return - if config.redis.ENABLED: + if config.redis.enabled: logger.info("Running Redis connection test...") try: get_cache() @@ -193,7 +193,7 @@ def start_webserver() -> None: scheduler.start() - if config.database.ENABLED: + if config.database.enabled: logger.info("Starting scheduled request intake...") initiate_scheduled_request_intake() @@ -205,7 +205,7 @@ def start_webserver() -> None: ) ) - if not config.execution.WORKER_ENABLED: + if not config.execution.worker_enabled: logger.info("Starting worker...") subprocess.Popen(["fidesops", "worker"]) @@ -213,7 +213,7 @@ def start_webserver() -> None: uvicorn.run( "fidesops.main:app", host="0.0.0.0", - port=config.PORT, + port=config.port, log_config=None, reload=config.hot_reloading, ) diff --git a/src/fidesops/migrations/env.py b/src/fidesops/migrations/env.py index da0bc9ceca..31f561bef1 100644 --- a/src/fidesops/migrations/env.py +++ b/src/fidesops/migrations/env.py @@ -41,9 +41,9 @@ def get_url(): the URL will point at the test database. """ - database_uri = fides_config.database.SQLALCHEMY_DATABASE_URI + database_uri = fides_config.database.sqlalchemy_database_uri if fides_config.is_test_mode: - database_uri = fides_config.database.SQLALCHEMY_TEST_DATABASE_URI + database_uri = fides_config.database.sqlalchemy_test_database_uri return database_uri diff --git a/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py b/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py index 30b756843c..8284970136 100644 --- a/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py +++ b/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py @@ -10,6 +10,11 @@ from uuid import uuid4 from alembic import op +from fideslib.cryptography.cryptographic_util import ( + generate_salt, + generate_secure_random_string, + hash_with_salt, +) from fideslib.db.base_class import FidesBase from sqlalchemy import text from sqlalchemy.dialects import postgresql @@ -28,11 +33,6 @@ from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( STRING_REWRITE_STRATEGY_NAME, ) -from fidesops.util.cryptographic_util import ( - generate_salt, - generate_secure_random_string, - hash_with_salt, -) from fidesops.util.data_category import DataCategory logging.basicConfig() @@ -302,7 +302,7 @@ def autogenerate_erasure_policy(connection: Connection, client_id: str) -> None: rule_id: str = generate_uuid(Rule) encryption: StringEncryptedType = StringEncryptedType( - JSONTypeOverride, config.security.APP_ENCRYPTION_KEY, AesGcmEngine, "pkcs5" + JSONTypeOverride, config.security.app_encryption_key, AesGcmEngine, "pkcs5" ) connection.execute( rule_insert_query, @@ -362,10 +362,10 @@ def autogenerate_client(connection: Connection) -> str: logger.info(f"Creating autogenerated client: '{FIDESOPS_AUTOGENERATED_CLIENT_KEY}'") client_id: str = generate_secure_random_string( - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES + config.security.oauth_client_id_length_bytes ) secret: str = generate_secure_random_string( - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES + config.security.oauth_client_secret_length_bytes ) salt: str = generate_salt() connection.execute( @@ -373,8 +373,8 @@ def autogenerate_client(connection: Connection) -> str: { "client_id": client_id, "hashed_secret": hash_with_salt( - secret.encode(config.security.ENCODING), - salt.encode(config.security.ENCODING), + secret.encode(config.security.encoding), + salt.encode(config.security.encoding), ), "salt": salt, "scopes": SCOPE_REGISTRY, diff --git a/src/fidesops/models/connectionconfig.py b/src/fidesops/models/connectionconfig.py index 29be23dab5..f98ec50fd0 100644 --- a/src/fidesops/models/connectionconfig.py +++ b/src/fidesops/models/connectionconfig.py @@ -71,7 +71,7 @@ class ConnectionConfig(Base): MutableDict.as_mutable( StringEncryptedType( JSONTypeOverride, - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, AesGcmEngine, "pkcs5", ) diff --git a/src/fidesops/models/policy.py b/src/fidesops/models/policy.py index 5732b185a8..226090f869 100644 --- a/src/fidesops/models/policy.py +++ b/src/fidesops/models/policy.py @@ -226,7 +226,7 @@ class Rule(Base): MutableDict.as_mutable( StringEncryptedType( JSONTypeOverride, - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, AesGcmEngine, "pkcs5", ) diff --git a/src/fidesops/models/privacy_request.py b/src/fidesops/models/privacy_request.py index b60d1d64f5..7e80a9f67d 100644 --- a/src/fidesops/models/privacy_request.py +++ b/src/fidesops/models/privacy_request.py @@ -120,7 +120,7 @@ def generate_request_callback_jwe(webhook: PolicyPreWebhook) -> str: scopes=[PRIVACY_REQUEST_CALLBACK_RESUME], iat=datetime.now().isoformat(), ) - return generate_jwe(json.dumps(jwe.dict()), config.security.APP_ENCRYPTION_KEY) + return generate_jwe(json.dumps(jwe.dict()), config.security.app_encryption_key) class PrivacyRequest(Base): # pylint: disable=R0904 @@ -575,7 +575,7 @@ class ProvidedIdentity(Base): # pylint: disable=R0904 MutableDict.as_mutable( StringEncryptedType( JSONTypeOverride, - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, AesGcmEngine, "pkcs5", ) @@ -590,7 +590,7 @@ def hash_value( encoding: str = "UTF-8", ) -> tuple[str, str]: """Utility function to hash a user's password with a generated salt""" - SALT = "a-salt" + SALT = "$2b$12$UErimNtlsE6qgYf2BrI1Du" hashed_value = hash_with_salt( value.encode(encoding), SALT.encode(encoding), diff --git a/src/fidesops/models/storage.py b/src/fidesops/models/storage.py index 6d4f2a9cad..6de285b57a 100644 --- a/src/fidesops/models/storage.py +++ b/src/fidesops/models/storage.py @@ -64,7 +64,7 @@ class StorageConfig(Base): MutableDict.as_mutable( StringEncryptedType( JSONTypeOverride, - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, AesGcmEngine, "pkcs5", ) diff --git a/src/fidesops/schemas/encryption_request.py b/src/fidesops/schemas/encryption_request.py index 96510a4688..a31a9cdd1d 100644 --- a/src/fidesops/schemas/encryption_request.py +++ b/src/fidesops/schemas/encryption_request.py @@ -13,7 +13,7 @@ class AesEncryptionRequest(BaseModel): @validator("key") def validate_key(cls, v: str) -> bytes: """Convert string into bytes and verify this is the correct length""" - key = v.encode(config.security.ENCODING) + key = v.encode(config.security.encoding) verify_encryption_key(key) return key diff --git a/src/fidesops/schemas/privacy_request.py b/src/fidesops/schemas/privacy_request.py index cc6b622e0f..34dcd0848d 100644 --- a/src/fidesops/schemas/privacy_request.py +++ b/src/fidesops/schemas/privacy_request.py @@ -66,7 +66,7 @@ def validate_encryption_key( ) -> Optional[str]: """Validate encryption key where applicable""" if value: - verify_encryption_key(value.encode(config.security.ENCODING)) + verify_encryption_key(value.encode(config.security.encoding)) return value diff --git a/src/fidesops/service/connectors/saas_query_config.py b/src/fidesops/service/connectors/saas_query_config.py index 864ccac7f0..b906499b18 100644 --- a/src/fidesops/service/connectors/saas_query_config.py +++ b/src/fidesops/service/connectors/saas_query_config.py @@ -70,7 +70,7 @@ def get_masking_request(self) -> Optional[SaaSRequest]: gdpr_delete: Optional[SaaSRequest] = None delete: Optional[SaaSRequest] = None - if not config.execution.MASKING_STRICT: + if not config.execution.masking_strict: gdpr_delete = self.data_protection_request delete = self.get_request_by_action("delete") diff --git a/src/fidesops/service/masking/strategy/masking_strategy_hash.py b/src/fidesops/service/masking/strategy/masking_strategy_hash.py index 27a26ae976..15bec85754 100644 --- a/src/fidesops/service/masking/strategy/masking_strategy_hash.py +++ b/src/fidesops/service/masking/strategy/masking_strategy_hash.py @@ -111,13 +111,13 @@ def data_type_supported(data_type: Optional[str]) -> bool: @staticmethod def _hash_sha256(value: str, salt: str) -> str: return hashlib.sha256( - (value + salt).encode(config.security.ENCODING) + (value + salt).encode(config.security.encoding) ).hexdigest() @staticmethod def _hash_sha512(value: str, salt: str) -> str: return hashlib.sha512( - (value + salt).encode(config.security.ENCODING) + (value + salt).encode(config.security.encoding) ).hexdigest() @staticmethod diff --git a/src/fidesops/service/privacy_request/request_runner_service.py b/src/fidesops/service/privacy_request/request_runner_service.py index dee4748509..23e99dbd4e 100644 --- a/src/fidesops/service/privacy_request/request_runner_service.py +++ b/src/fidesops/service/privacy_request/request_runner_service.py @@ -278,7 +278,7 @@ def initiate_paused_privacy_request_followup(privacy_request: PrivacyRequest) -> id=privacy_request.id, replace_existing=True, trigger="date", - run_date=(datetime.now() + timedelta(seconds=config.redis.DEFAULT_TTL_SECONDS)), + run_date=(datetime.now() + timedelta(seconds=config.redis.default_ttl_seconds)), ) diff --git a/src/fidesops/task/graph_task.py b/src/fidesops/task/graph_task.py index 114cb8ee3b..5e86555132 100644 --- a/src/fidesops/task/graph_task.py +++ b/src/fidesops/task/graph_task.py @@ -58,12 +58,12 @@ def retry( def decorator(func: Callable) -> Callable: @wraps(func) def result(*args: Any, **kwargs: Any) -> List[Optional[Row]]: - func_delay = config.execution.TASK_RETRY_DELAY + func_delay = config.execution.task_retry_delay method_name = func.__name__ self = args[0] raised_ex: Optional[Union[BaseException, Exception]] = None - for attempt in range(config.execution.TASK_RETRY_COUNT + 1): + for attempt in range(config.execution.task_retry_count + 1): try: self.skip_if_disabled() # Create ExecutionLog with status in_processing or retrying @@ -88,7 +88,7 @@ def result(*args: Any, **kwargs: Any) -> List[Optional[Row]]: self.log_skipped(action_type, exc) return default_return except BaseException as ex: # pylint: disable=W0703 - func_delay *= config.execution.TASK_RETRY_BACKOFF + func_delay *= config.execution.task_retry_backoff logger.warning( f"Retrying {method_name} {self.traversal_node.address} in {func_delay} seconds..." ) diff --git a/src/fidesops/tasks/__init__.py b/src/fidesops/tasks/__init__.py index d3b64c3061..b59a8b87fc 100644 --- a/src/fidesops/tasks/__init__.py +++ b/src/fidesops/tasks/__init__.py @@ -10,7 +10,7 @@ logger = get_task_logger(__name__) -def _create_celery(config_path: str = config.execution.CELERY_CONFIG_PATH) -> Celery: +def _create_celery(config_path: str = config.execution.celery_config_path) -> Celery: """ Returns a configured version of the Celery application """ @@ -19,8 +19,8 @@ def _create_celery(config_path: str = config.execution.CELERY_CONFIG_PATH) -> Ce celery_config: Dict[str, Any] = { # Defaults for the celery config - "broker_url": config.redis.CONNECTION_URL, - "result_backend": config.redis.CONNECTION_URL, + "broker_url": config.redis.connection_url, + "result_backend": config.redis.connection_url, } try: diff --git a/src/fidesops/tasks/storage.py b/src/fidesops/tasks/storage.py index df5c67c467..7018feea35 100644 --- a/src/fidesops/tasks/storage.py +++ b/src/fidesops/tasks/storage.py @@ -12,12 +12,12 @@ import pandas as pd import requests from botocore.exceptions import ClientError, ParamValidationError +from fideslib.cryptography.cryptographic_util import bytes_to_b64_str from fidesops.core.config import config from fidesops.models.storage import ResponseFormat from fidesops.schemas.storage.storage import StorageSecrets from fidesops.util.cache import get_cache, get_encryption_cache_key -from fidesops.util.cryptographic_util import bytes_to_b64_str from fidesops.util.encryption.aes_gcm_encryption_scheme import ( encrypt_to_bytes_verify_secrets_length, ) @@ -40,16 +40,16 @@ def encrypt_access_request_results(data: Union[str, bytes], request_id: str) -> encryption_attr="key", ) if isinstance(data, bytes): - data = data.decode(config.security.ENCODING) + data = data.decode(config.security.encoding) encryption_key: str | None = cache.get(encryption_cache_key) if not encryption_key: return data bytes_encryption_key: bytes = encryption_key.encode( - encoding=config.security.ENCODING + encoding=config.security.encoding ) - nonce: bytes = secrets.token_bytes(config.security.AES_GCM_NONCE_LENGTH) + nonce: bytes = secrets.token_bytes(config.security.aes_gcm_nonce_length) # b64encode the entire nonce and the encrypted message together return bytes_to_b64_str( nonce @@ -73,7 +73,7 @@ def write_to_in_memory_buffer( json_str = json.dumps(data, indent=2, default=_handle_json_encoding) return BytesIO( encrypt_access_request_results(json_str, request_id).encode( - config.security.ENCODING + config.security.encoding ) ) @@ -83,7 +83,7 @@ def write_to_in_memory_buffer( for key in data: df = pd.json_normalize(data[key]) buffer = BytesIO() - df.to_csv(buffer, index=False, encoding=config.security.ENCODING) + df.to_csv(buffer, index=False, encoding=config.security.encoding) buffer.seek(0) f.writestr( f"{key}.csv", diff --git a/src/fidesops/util/cache.py b/src/fidesops/util/cache.py index 4cfc5fa91a..2e099cb295 100644 --- a/src/fidesops/util/cache.py +++ b/src/fidesops/util/cache.py @@ -28,7 +28,7 @@ class FidesopsRedis(Redis): def set_with_autoexpire(self, key: str, value: RedisValue) -> Optional[bool]: """Call the connection class' default set method with ex= our default TTL""" - return self.set(key, value, ex=config.redis.DEFAULT_TTL_SECONDS) + return self.set(key, value, ex=config.redis.default_ttl_seconds) def get_keys_by_prefix(self, prefix: str, chunk_size: int = 1000) -> List[str]: """Retrieve all keys that match a given prefix.""" @@ -95,14 +95,14 @@ def get_cache() -> FidesopsRedis: global _connection # pylint: disable=W0603 if _connection is None: _connection = FidesopsRedis( - charset=config.redis.CHARSET, - decode_responses=config.redis.DECODE_RESPONSES, - host=config.redis.HOST, - port=config.redis.PORT, - db=config.redis.DB_INDEX, - password=config.redis.PASSWORD, - ssl=config.redis.SSL, - ssl_cert_reqs=config.redis.SSL_CERT_REQS, + charset=config.redis.charset, + decode_responses=config.redis.decode_responses, + host=config.redis.host, + port=config.redis.port, + db=config.redis.db_index, + password=config.redis.password, + ssl=config.redis.ssl, + ssl_cert_reqs=config.redis.ssl_cert_reqs, ) connected = _connection.ping() diff --git a/src/fidesops/util/cryptographic_util.py b/src/fidesops/util/cryptographic_util.py deleted file mode 100644 index b614c7d248..0000000000 --- a/src/fidesops/util/cryptographic_util.py +++ /dev/null @@ -1,46 +0,0 @@ -import hashlib -import secrets -from base64 import b64decode, b64encode - -import bcrypt - -from fidesops.core.config import config - - -def hash_with_salt(text: bytes, salt: bytes) -> str: - """Hashes the text using SHA-512 with the provided salt and returns the hex string - representation""" - return hashlib.sha512(text + salt).hexdigest() - - -def generate_secure_random_string(length: int) -> str: - """Generates a securely random string using Python secrets library - that is twice the length of the specified input""" - return secrets.token_hex(length) - - -def generate_salt() -> str: - """Generates a salt using bcrypt and returns a string using the configured default encoding""" - return bcrypt.gensalt().decode(config.security.ENCODING) - - -def bytes_to_b64_str(bytestring: bytes) -> str: - """Converts random bytes into a utf-8 encoded string""" - return b64encode(bytestring).decode(config.security.ENCODING) - - -def b64_str_to_bytes(encoded_str: str) -> bytes: - """Converts encoded string into bytes""" - return b64decode(encoded_str.encode(config.security.ENCODING)) - - -def b64_str_to_str(encoded_str: str) -> str: - """Converts encoded string into str""" - return b64decode(encoded_str).decode(config.security.ENCODING) - - -def str_to_b64_str(string: str) -> str: - """Converts str into a utf-8 encoded string""" - return b64encode(string.encode(config.security.ENCODING)).decode( - config.security.ENCODING - ) diff --git a/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py b/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py index 69d3eeb859..e1604f26b5 100644 --- a/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py +++ b/src/fidesops/util/encryption/aes_gcm_encryption_scheme.py @@ -2,9 +2,9 @@ from typing import Optional from cryptography.hazmat.primitives.ciphers.aead import AESGCM +from fideslib.cryptography.cryptographic_util import bytes_to_b64_str from fidesops.core.config import config -from fidesops.util.cryptographic_util import bytes_to_b64_str def encrypt_to_bytes_verify_secrets_length( @@ -23,7 +23,7 @@ def _encrypt_to_bytes(plain_value: Optional[str], key: bytes, nonce: bytes) -> b if plain_value is None: raise ValueError("plain_value cannot be null") gcm = AESGCM(key) - value_bytes = plain_value.encode(config.security.ENCODING) + value_bytes = plain_value.encode(config.security.encoding) encrypted_bytes = gcm.encrypt(nonce, value_bytes, nonce) return encrypted_bytes @@ -51,13 +51,13 @@ def decrypt_combined_nonce_and_message(encrypted_value: str, key: bytes) -> str: encrypted_combined: bytes = base64.b64decode(encrypted_value) # Separate the nonce out as the first 12 characters of the combined message - nonce: bytes = encrypted_combined[0 : config.security.AES_GCM_NONCE_LENGTH] + nonce: bytes = encrypted_combined[0 : config.security.aes_gcm_nonce_length] encrypted_message: bytes = encrypted_combined[ - config.security.AES_GCM_NONCE_LENGTH : + config.security.aes_gcm_nonce_length : ] decrypted_bytes: bytes = gcm.decrypt(nonce, encrypted_message, nonce) - decrypted_str = decrypted_bytes.decode(config.security.ENCODING) + decrypted_str = decrypted_bytes.decode(config.security.encoding) return decrypted_str @@ -69,19 +69,19 @@ def decrypt(encrypted_value: str, key: bytes, nonce: bytes) -> str: gcm = AESGCM(key) encrypted_bytes = base64.b64decode(encrypted_value) decrypted_bytes = gcm.decrypt(nonce, encrypted_bytes, nonce) - decrypted_str = decrypted_bytes.decode(config.security.ENCODING) + decrypted_str = decrypted_bytes.decode(config.security.encoding) return decrypted_str def verify_nonce(nonce: bytes) -> None: - if len(nonce) != config.security.AES_GCM_NONCE_LENGTH: + if len(nonce) != config.security.aes_gcm_nonce_length: raise ValueError( - f"Nonce must be {config.security.AES_GCM_NONCE_LENGTH} bytes long" + f"Nonce must be {config.security.aes_gcm_nonce_length} bytes long" ) def verify_encryption_key(key: bytes) -> None: - if len(key) != config.security.AES_ENCRYPTION_KEY_LENGTH: + if len(key) != config.security.aes_encryption_key_length: raise ValueError( - f"Encryption key must be {config.security.AES_ENCRYPTION_KEY_LENGTH} bytes long" + f"Encryption key must be {config.security.aes_encryption_key_length} bytes long" ) diff --git a/src/fidesops/util/encryption/hmac_encryption_scheme.py b/src/fidesops/util/encryption/hmac_encryption_scheme.py index e60ed82bcc..0fb8ec8e55 100644 --- a/src/fidesops/util/encryption/hmac_encryption_scheme.py +++ b/src/fidesops/util/encryption/hmac_encryption_scheme.py @@ -56,7 +56,7 @@ def _hmac_sha512(value: str, hmac_key: str, salt: str) -> hmac.HMAC: def _hmac(value: str, hmac_key: str, salt: str, hashing_alg: Callable) -> hmac.HMAC: return hmac.new( - key=hmac_key.encode(config.security.ENCODING), - msg=(value + salt).encode(config.security.ENCODING), + key=hmac_key.encode(config.security.encoding), + msg=(value + salt).encode(config.security.encoding), digestmod=hashing_alg, ) diff --git a/src/fidesops/util/oauth_util.py b/src/fidesops/util/oauth_util.py index 9c61a55d61..0f95b17758 100644 --- a/src/fidesops/util/oauth_util.py +++ b/src/fidesops/util/oauth_util.py @@ -60,7 +60,7 @@ def is_callback_token_expired(issued_at: datetime | None) -> bool: return ( datetime.now() - issued_at - ).total_seconds() / 60.0 > config.execution.PRIVACY_REQUEST_DELAY_TIMEOUT + ).total_seconds() / 60.0 > config.execution.privacy_request_delay_timeout def verify_callback_oauth( @@ -80,7 +80,7 @@ def verify_callback_oauth( raise AuthenticationError(detail="Authentication Failure") token_data = json.loads( - extract_payload(authorization, config.security.APP_ENCRYPTION_KEY) + extract_payload(authorization, config.security.app_encryption_key) ) try: token = WebhookJWE(**token_data) @@ -118,7 +118,7 @@ async def verify_oauth_client( raise AuthenticationError(detail="Authentication Failure") token_data = json.loads( - extract_payload(authorization, config.security.APP_ENCRYPTION_KEY) + extract_payload(authorization, config.security.app_encryption_key) ) issued_at = token_data.get(JWE_ISSUED_AT, None) @@ -127,7 +127,7 @@ async def verify_oauth_client( if is_token_expired( datetime.fromisoformat(issued_at), - config.security.OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES, + config.security.oauth_access_token_expire_minutes, ): raise AuthorizationError(detail="Not Authorized for this action") diff --git a/tests/api/test_deps.py b/tests/api/test_deps.py index 61cd16700a..0851380f7b 100644 --- a/tests/api/test_deps.py +++ b/tests/api/test_deps.py @@ -7,13 +7,13 @@ @pytest.fixture def mock_config(): - db_enabled = config.config.database.ENABLED - redis_enabled = config.config.redis.ENABLED - config.config.database.ENABLED = False - config.config.redis.ENABLED = False + db_enabled = config.config.database.enabled + redis_enabled = config.config.redis.enabled + config.config.database.enabled = False + config.config.redis.enabled = False yield - config.config.database.ENABLED = db_enabled - config.config.redis.ENABLED = redis_enabled + config.config.database.enabled = db_enabled + config.config.redis.enabled = redis_enabled @pytest.mark.usefixtures("mock_config") diff --git a/tests/api/v1/endpoints/test_config_endpoints.py b/tests/api/v1/endpoints/test_config_endpoints.py index f4b0ec24a2..1ab97a5870 100644 --- a/tests/api/v1/endpoints/test_config_endpoints.py +++ b/tests/api/v1/endpoints/test_config_endpoints.py @@ -32,9 +32,9 @@ def test_get_config( security_keys.difference( set( [ - "CORS_ORIGINS", - "ENCODING", - "OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES", + "cors_origins", + "encoding", + "oauth_access_token_expire_minutes", ] ) ) diff --git a/tests/api/v1/endpoints/test_drp_endpoints.py b/tests/api/v1/endpoints/test_drp_endpoints.py index 06e7e44b09..626bec1139 100644 --- a/tests/api/v1/endpoints/test_drp_endpoints.py +++ b/tests/api/v1/endpoints/test_drp_endpoints.py @@ -50,7 +50,7 @@ def test_create_drp_privacy_request( "phone_number": TEST_PHONE_NUMBER, } encoded_identity: str = jwt.encode( - identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + identity, config.security.drp_jwt_secret, algorithm="HS256" ) data = { "meta": {"version": "0.5"}, @@ -117,7 +117,7 @@ def test_create_drp_privacy_request_unsupported_identity_props( identity = {"email": "test@example.com", "address": "something"} encoded_identity: str = jwt.encode( - identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + identity, config.security.drp_jwt_secret, algorithm="HS256" ) data = { "meta": {"version": "0.5"}, @@ -178,8 +178,8 @@ def test_create_drp_privacy_request_no_jwt( policy_drp_action, ): - original_secret = config.security.DRP_JWT_SECRET - config.security.DRP_JWT_SECRET = None + original_secret = config.security.drp_jwt_secret + config.security.drp_jwt_secret = None identity = {"email": "test@example.com"} encoded_identity: str = jwt.encode(identity, "secret", algorithm="HS256") data = { @@ -190,7 +190,7 @@ def test_create_drp_privacy_request_no_jwt( } resp = api_client.post(url, json=data) assert resp.status_code == 500 - config.security.DRP_JWT_SECRET = original_secret + config.security.drp_jwt_secret = original_secret def test_create_drp_privacy_request_no_exercise( self, @@ -202,7 +202,7 @@ def test_create_drp_privacy_request_no_exercise( identity = {"email": "test@example.com"} encoded_identity: str = jwt.encode( - identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + identity, config.security.drp_jwt_secret, algorithm="HS256" ) data = { "meta": {"version": "0.5"}, @@ -223,7 +223,7 @@ def test_create_drp_privacy_request_invalid_exercise( identity = {"email": "test@example.com"} encoded_identity: str = jwt.encode( - identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + identity, config.security.drp_jwt_secret, algorithm="HS256" ) data = { "meta": {"version": "0.5"}, @@ -244,7 +244,7 @@ def test_create_drp_privacy_request_no_associated_policy( identity = {"email": "test@example.com"} encoded_identity: str = jwt.encode( - identity, config.security.DRP_JWT_SECRET, algorithm="HS256" + identity, config.security.drp_jwt_secret, algorithm="HS256" ) data = { "meta": {"version": "0.5"}, diff --git a/tests/api/v1/endpoints/test_encryption_endpoints.py b/tests/api/v1/endpoints/test_encryption_endpoints.py index 7942120466..1b0519820b 100644 --- a/tests/api/v1/endpoints/test_encryption_endpoints.py +++ b/tests/api/v1/endpoints/test_encryption_endpoints.py @@ -3,6 +3,7 @@ from unittest.mock import Mock import pytest +from fideslib.cryptography.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from starlette.testclient import TestClient from fidesops.api.v1.scope_registry import ENCRYPTION_EXEC, STORAGE_CREATE_OR_UPDATE @@ -13,7 +14,6 @@ V1_URL_PREFIX, ) from fidesops.core.config import config -from fidesops.util.cryptographic_util import b64_str_to_bytes, bytes_to_b64_str from fidesops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, encrypt_verify_secret_length, @@ -113,7 +113,7 @@ def test_aes_encrypt( assert response.status_code == 200 decrypted = decrypt( encrypted_value, - key.encode(config.security.ENCODING), + key.encode(config.security.encoding), nonce, ) assert decrypted == plain_val @@ -154,7 +154,7 @@ def test_aes_decrypt( nonce = b'\x18\xf5"+\xdbj\xe6O\xc7|\x19\xd2' orig_data = "test_data" encrypted_data = encrypt_verify_secret_length( - orig_data, key.encode(config.security.ENCODING), nonce + orig_data, key.encode(config.security.encoding), nonce ) request = { diff --git a/tests/api/v1/endpoints/test_oauth_endpoints.py b/tests/api/v1/endpoints/test_oauth_endpoints.py index c70e5c42f8..2a3180c19a 100644 --- a/tests/api/v1/endpoints/test_oauth_endpoints.py +++ b/tests/api/v1/endpoints/test_oauth_endpoints.py @@ -60,7 +60,7 @@ def test_create_client_lacks_client(self, api_client: TestClient, url) -> None: # Build auth header without client auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } response = api_client.post(url, headers=auth_header) @@ -76,7 +76,7 @@ def test_create_client_with_expired_token( } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } response = api_client.post(url, headers=auth_header) assert 403 == response.status_code @@ -380,8 +380,8 @@ def test_invalid_client(self, db, url, api_client): def test_invalid_client_secret(self, db, url, api_client): new_client, _ = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, ) response = api_client.post( url, data={"client_id": new_client.id, "secret": "badsecret"} @@ -392,8 +392,8 @@ def test_invalid_client_secret(self, db, url, api_client): def test_get_access_token_root_client(self, url, api_client): data = { - "client_id": config.security.OAUTH_ROOT_CLIENT_ID, - "client_secret": config.security.OAUTH_ROOT_CLIENT_SECRET, + "client_id": config.security.oauth_root_client_id, + "client_secret": config.security.oauth_root_client_secret, } response = api_client.post(url, data=data) @@ -401,12 +401,12 @@ def test_get_access_token_root_client(self, url, api_client): assert 200 == response.status_code assert ( data["client_id"] - == json.loads(extract_payload(jwt, config.security.APP_ENCRYPTION_KEY))[ + == json.loads(extract_payload(jwt, config.security.app_encryption_key))[ JWE_PAYLOAD_CLIENT_ID ] ) assert ( - json.loads(extract_payload(jwt, config.security.APP_ENCRYPTION_KEY))[ + json.loads(extract_payload(jwt, config.security.app_encryption_key))[ JWE_PAYLOAD_SCOPES ] == SCOPE_REGISTRY @@ -415,8 +415,8 @@ def test_get_access_token_root_client(self, url, api_client): def test_get_access_token(self, db, url, api_client): new_client, secret = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, ) data = { @@ -429,12 +429,12 @@ def test_get_access_token(self, db, url, api_client): assert 200 == response.status_code assert ( data["client_id"] - == json.loads(extract_payload(jwt, config.security.APP_ENCRYPTION_KEY))[ + == json.loads(extract_payload(jwt, config.security.app_encryption_key))[ JWE_PAYLOAD_CLIENT_ID ] ) assert ( - json.loads(extract_payload(jwt, config.security.APP_ENCRYPTION_KEY))[ + json.loads(extract_payload(jwt, config.security.app_encryption_key))[ JWE_PAYLOAD_SCOPES ] == [] diff --git a/tests/api/v1/endpoints/test_privacy_request_endpoints.py b/tests/api/v1/endpoints/test_privacy_request_endpoints.py index 7446ef17e6..a8e4fb2e50 100644 --- a/tests/api/v1/endpoints/test_privacy_request_endpoints.py +++ b/tests/api/v1/endpoints/test_privacy_request_endpoints.py @@ -148,7 +148,7 @@ def test_create_privacy_request_require_manual_approval( api_client: TestClient, policy, ): - config.execution.REQUIRE_MANUAL_REQUEST_APPROVAL = True + config.execution.require_manual_request_approval = True data = [ { @@ -166,7 +166,7 @@ def test_create_privacy_request_require_manual_approval( pr.delete(db=db) assert not run_access_request_mock.called - config.execution.REQUIRE_MANUAL_REQUEST_APPROVAL = False + config.execution.require_manual_request_approval = False @mock.patch( "fidesops.service.privacy_request.request_runner_service.run_privacy_request.delay" @@ -1609,7 +1609,7 @@ def test_approve_privacy_request( } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } body = {"request_ids": [privacy_request.id]} @@ -1710,7 +1710,7 @@ def test_deny_privacy_request_without_denial_reason( } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } body = {"request_ids": [privacy_request.id]} @@ -1761,7 +1761,7 @@ def test_deny_privacy_request_with_denial_reason( } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } denial_reason = "Your request was denied because reasons" body = {"request_ids": [privacy_request.id], "reason": denial_reason} @@ -1817,7 +1817,7 @@ def test_resume_privacy_request_invalid_jwe_format( auth_header = { "Authorization": "Bearer " + generate_jwe( - json.dumps({"unexpected": "format"}), config.security.APP_ENCRYPTION_KEY + json.dumps({"unexpected": "format"}), config.security.app_encryption_key ) } response = api_client.post(url, headers=auth_header, json={}) @@ -1845,7 +1845,7 @@ def test_resume_privacy_request_invalid_scopes( "iat": datetime.now().isoformat(), } ), - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, ) } response = api_client.post(url, headers=auth_header, json={}) @@ -1869,7 +1869,7 @@ def test_resume_privacy_request_invalid_webhook( "iat": datetime.now().isoformat(), } ), - config.security.APP_ENCRYPTION_KEY, + config.security.app_encryption_key, ) } response = api_client.post(url, headers=auth_header, json={}) diff --git a/tests/api/v1/endpoints/test_user_endpoints.py b/tests/api/v1/endpoints/test_user_endpoints.py index 4a0600fe12..a1aa9095fb 100644 --- a/tests/api/v1/endpoints/test_user_endpoints.py +++ b/tests/api/v1/endpoints/test_user_endpoints.py @@ -229,8 +229,8 @@ def test_delete_self(self, api_client, db, generate_auth_header): client, _ = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, scopes=[USER_DELETE], user_id=user.id, ) @@ -242,7 +242,7 @@ def test_delete_self(self, api_client, db, generate_auth_header): JWE_PAYLOAD_CLIENT_ID: client.id, JWE_ISSUED_AT: datetime.now().isoformat(), } - jwe = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) auth_header = {"Authorization": "Bearer " + jwe} response = api_client.delete( @@ -278,8 +278,8 @@ def test_delete_user_as_root(self, api_client, db, generate_auth_header, user): user_client, _ = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, scopes=[USER_DELETE], user_id=other_user.id, ) @@ -297,7 +297,7 @@ def test_delete_user_as_root(self, api_client, db, generate_auth_header, user): JWE_PAYLOAD_CLIENT_ID: user.client.id, JWE_ISSUED_AT: datetime.now().isoformat(), } - jwe = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) auth_header = {"Authorization": "Bearer " + jwe} response = api_client.delete( @@ -692,7 +692,7 @@ def test_login_creates_client(self, db, url, user, api_client): assert "token_data" in list(response.json().keys()) token = response.json()["token_data"]["access_token"] token_data = json.loads( - extract_payload(token, config.security.APP_ENCRYPTION_KEY) + extract_payload(token, config.security.app_encryption_key) ) assert token_data["client-id"] == user.client.id assert token_data["scopes"] == [ @@ -733,7 +733,7 @@ def test_login_uses_existing_client(self, db, url, user, api_client): assert "token_data" in list(response.json().keys()) token = response.json()["token_data"]["access_token"] token_data = json.loads( - extract_payload(token, config.security.APP_ENCRYPTION_KEY) + extract_payload(token, config.security.app_encryption_key) ) assert token_data["client-id"] == existing_client_id assert token_data["scopes"] == [ @@ -761,7 +761,7 @@ def test_user_not_deleted_on_logout(self, db, url, api_client, user): } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } response = api_client.post(url, headers=auth_header, json={}) assert response.status_code == HTTP_204_NO_CONTENT @@ -793,7 +793,7 @@ def test_user_not_deleted_on_logout(self, db, url, api_client, user): } auth_header = { "Authorization": "Bearer " - + generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + + generate_jwe(json.dumps(payload), config.security.app_encryption_key) } response = api_client.post(url, headers=auth_header, json={}) assert HTTP_403_FORBIDDEN == response.status_code diff --git a/tests/api/v1/endpoints/test_user_permission_endpoints.py b/tests/api/v1/endpoints/test_user_permission_endpoints.py index 834a5334a5..fb5ee7e205 100644 --- a/tests/api/v1/endpoints/test_user_permission_endpoints.py +++ b/tests/api/v1/endpoints/test_user_permission_endpoints.py @@ -163,8 +163,8 @@ def test_edit_user_permissions(self, db, api_client, generate_auth_header) -> No ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, scopes=[PRIVACY_REQUEST_READ], user_id=user.id, ) diff --git a/tests/api/v1/test_exception_handlers.py b/tests/api/v1/test_exception_handlers.py index 372221fcfe..d9e52a6e27 100644 --- a/tests/api/v1/test_exception_handlers.py +++ b/tests/api/v1/test_exception_handlers.py @@ -10,18 +10,18 @@ @pytest.fixture def mock_config_db_disabled(): - db_enabled = config.config.database.ENABLED - config.config.database.ENABLED = False + db_enabled = config.config.database.enabled + config.config.database.enabled = False yield - config.config.database.ENABLED = db_enabled + config.config.database.enabled = db_enabled @pytest.fixture def mock_config_redis_disabled(): - redis_enabled = config.config.redis.ENABLED - config.config.redis.ENABLED = False + redis_enabled = config.config.redis.enabled + config.config.redis.enabled = False yield - config.config.redis.ENABLED = redis_enabled + config.config.redis.enabled = redis_enabled class TestExceptionHandlers: diff --git a/tests/conftest.py b/tests/conftest.py index 419d29366e..a76457d07e 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -55,8 +55,8 @@ def migrate_test_db() -> None: """Apply migrations at beginning and end of testing session""" logger.debug("Applying migrations...") assert config.is_test_mode - if config.database.ENABLED: - init_db(config.database.SQLALCHEMY_TEST_DATABASE_URI) + if config.database.enabled: + init_db(config.database.sqlalchemy_test_database_uri) logger.debug("Migrations successfully applied") @@ -66,7 +66,7 @@ def db() -> Generator: # Create the test DB enginge assert config.is_test_mode engine = get_db_engine( - database_uri=config.database.SQLALCHEMY_TEST_DATABASE_URI, + database_uri=config.database.sqlalchemy_test_database_uri, ) logger.debug(f"Configuring database at: {engine.url}") @@ -153,7 +153,7 @@ def generate_auth_header_for_user(user, scopes) -> Dict[str, str]: JWE_PAYLOAD_CLIENT_ID: user.client.id, JWE_ISSUED_AT: datetime.now().isoformat(), } - jwe = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) return {"Authorization": "Bearer " + jwe} @@ -171,7 +171,7 @@ def _build_jwt(scopes: List[str]) -> Dict[str, str]: JWE_PAYLOAD_CLIENT_ID: client_id, JWE_ISSUED_AT: datetime.now().isoformat(), } - jwe = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + jwe = generate_jwe(json.dumps(payload), config.security.app_encryption_key) return {"Authorization": "Bearer " + jwe} return _build_jwt @@ -221,7 +221,7 @@ def run_privacy_request_task(celery_session_app): @pytest.fixture(autouse=True, scope="session") def analytics_opt_out(): """Disable sending analytics when running tests.""" - original_value = config.root_user.ANALYTICS_OPT_OUT - config.root_user.ANALYTICS_OPT_OUT = True + original_value = config.root_user.analytics_opt_out + config.root_user.analytics_opt_out = True yield - config.root_user.ANALYTICS_OPT_OUT = original_value + config.root_user.analytics_opt_out = original_value diff --git a/tests/core/test_config.py b/tests/core/test_config.py index c29752a623..34308fae2d 100644 --- a/tests/core/test_config.py +++ b/tests/core/test_config.py @@ -13,9 +13,9 @@ def test_config_from_default() -> None: "Test building a config from default local TOML" config = get_config(FidesopsConfig) - assert config.database.SERVER == "db" - assert config.redis.HOST == "redis" - assert config.security.APP_ENCRYPTION_KEY == "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" + assert config.database.server == "db" + assert config.redis.host == "redis" + assert config.security.app_encryption_key == "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" @patch.dict( @@ -28,10 +28,10 @@ def test_config_from_default() -> None: def test_config_from_path() -> None: """Test reading config using the FIDESOPS__CONFIG_PATH option.""" config = get_config(FidesopsConfig) - assert config.database.SERVER == "testserver" - assert config.redis.HOST == "testredis" - assert config.security.APP_ENCRYPTION_KEY == "atestencryptionkeythatisvalidlen" - assert config.admin_ui.ENABLED == True + assert config.database.server == "testserver" + assert config.redis.host == "testredis" + assert config.security.app_encryption_key == "atestencryptionkeythatisvalidlen" + assert config.admin_ui.enabled == True @patch.dict( @@ -45,9 +45,9 @@ def test_config_from_path() -> None: def test_config_from_env_vars() -> None: """Test overriding config using ENV vars.""" config = get_config(FidesopsConfig) - assert config.database.SERVER == "envserver" - assert config.redis.HOST == "envhost" - assert config.security.APP_ENCRYPTION_KEY == "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" + assert config.database.server == "envserver" + assert config.redis.host == "envhost" + assert config.security.app_encryption_key == "OLMkv91j8DHiDAULnK5Lxx3kSCov30b3" def test_config_app_encryption_key_validation() -> None: @@ -62,7 +62,7 @@ def test_config_app_encryption_key_validation() -> None: clear=True, ): config = get_config(FidesopsConfig) - assert config.security.APP_ENCRYPTION_KEY == app_encryption_key + assert config.security.app_encryption_key == app_encryption_key @pytest.mark.parametrize( @@ -104,7 +104,7 @@ def test_config_log_level(log_level, expected_log_level): clear=True, ): config = get_config(FidesopsConfig) - assert config.security.LOG_LEVEL == expected_log_level + assert config.security.log_level == expected_log_level def test_config_log_level_invalid(): diff --git a/tests/fixtures/application_fixtures.py b/tests/fixtures/application_fixtures.py index 7d3141a244..7bbdfdcd11 100644 --- a/tests/fixtures/application_fixtures.py +++ b/tests/fixtures/application_fixtures.py @@ -60,38 +60,38 @@ integration_secrets = { "postgres_example": { - "host": pydash.get(integration_config, "postgres_example.SERVER"), - "port": pydash.get(integration_config, "postgres_example.PORT"), - "dbname": pydash.get(integration_config, "postgres_example.DB"), - "username": pydash.get(integration_config, "postgres_example.USER"), - "password": pydash.get(integration_config, "postgres_example.PASSWORD"), + "host": pydash.get(integration_config, "postgres_example.server"), + "port": pydash.get(integration_config, "postgres_example.port"), + "dbname": pydash.get(integration_config, "postgres_example.db"), + "username": pydash.get(integration_config, "postgres_example.user"), + "password": pydash.get(integration_config, "postgres_example.password"), }, "mongo_example": { - "host": pydash.get(integration_config, "mongodb_example.SERVER"), - "defaultauthdb": pydash.get(integration_config, "mongodb_example.DB"), - "username": pydash.get(integration_config, "mongodb_example.USER"), - "password": pydash.get(integration_config, "mongodb_example.PASSWORD"), + "host": pydash.get(integration_config, "mongodb_example.server"), + "defaultauthdb": pydash.get(integration_config, "mongodb_example.db"), + "username": pydash.get(integration_config, "mongodb_example.user"), + "password": pydash.get(integration_config, "mongodb_example.password"), }, "mysql_example": { - "host": pydash.get(integration_config, "mysql_example.SERVER"), - "port": pydash.get(integration_config, "mysql_example.PORT"), - "dbname": pydash.get(integration_config, "mysql_example.DB"), - "username": pydash.get(integration_config, "mysql_example.USER"), - "password": pydash.get(integration_config, "mysql_example.PASSWORD"), + "host": pydash.get(integration_config, "mysql_example.server"), + "port": pydash.get(integration_config, "mysql_example.port"), + "dbname": pydash.get(integration_config, "mysql_example.db"), + "username": pydash.get(integration_config, "mysql_example.user"), + "password": pydash.get(integration_config, "mysql_example.password"), }, "mssql_example": { - "host": pydash.get(integration_config, "mssql_example.SERVER"), - "port": pydash.get(integration_config, "mssql_example.PORT"), - "dbname": pydash.get(integration_config, "mssql_example.DB"), - "username": pydash.get(integration_config, "mssql_example.USER"), - "password": pydash.get(integration_config, "mssql_example.PASSWORD"), + "host": pydash.get(integration_config, "mssql_example.server"), + "port": pydash.get(integration_config, "mssql_example.port"), + "dbname": pydash.get(integration_config, "mssql_example.db"), + "username": pydash.get(integration_config, "mssql_example.user"), + "password": pydash.get(integration_config, "mssql_example.password"), }, "mariadb_example": { - "host": pydash.get(integration_config, "mariadb_example.SERVER"), - "port": pydash.get(integration_config, "mariadb_example.PORT"), - "dbname": pydash.get(integration_config, "mariadb_example.DB"), - "username": pydash.get(integration_config, "mariadb_example.USER"), - "password": pydash.get(integration_config, "mariadb_example.PASSWORD"), + "host": pydash.get(integration_config, "mariadb_example.server"), + "port": pydash.get(integration_config, "mariadb_example.port"), + "dbname": pydash.get(integration_config, "mariadb_example.db"), + "username": pydash.get(integration_config, "mariadb_example.user"), + "password": pydash.get(integration_config, "mariadb_example.password"), }, } diff --git a/tests/fixtures/saas/hubspot_fixtures.py b/tests/fixtures/saas/hubspot_fixtures.py index 9d77a7b77a..e95a254532 100644 --- a/tests/fixtures/saas/hubspot_fixtures.py +++ b/tests/fixtures/saas/hubspot_fixtures.py @@ -5,6 +5,7 @@ import pydash import pytest from fideslib.core.config import load_toml +from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( @@ -15,7 +16,6 @@ from fidesops.models.datasetconfig import DatasetConfig from fidesops.schemas.saas.shared_schemas import HTTPMethod, SaaSRequestParams from fidesops.service.connectors import SaaSConnector -from fidesops.util import cryptographic_util from fidesops.util.saas_util import format_body, load_config from tests.fixtures.application_fixtures import load_dataset from tests.test_helpers.saas_test_utils import poll_for_existence diff --git a/tests/fixtures/saas/outreach_fixtures.py b/tests/fixtures/saas/outreach_fixtures.py index 6d8ba4debd..3f59de5a1e 100644 --- a/tests/fixtures/saas/outreach_fixtures.py +++ b/tests/fixtures/saas/outreach_fixtures.py @@ -5,6 +5,7 @@ import pytest import requests from fideslib.core.config import load_toml +from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -14,7 +15,6 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util import cryptographic_util from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset diff --git a/tests/fixtures/saas/salesforce_fixtures.py b/tests/fixtures/saas/salesforce_fixtures.py index 2e2a99762d..98bce17c82 100644 --- a/tests/fixtures/saas/salesforce_fixtures.py +++ b/tests/fixtures/saas/salesforce_fixtures.py @@ -5,6 +5,7 @@ import pytest import requests from fideslib.core.config import load_toml +from fideslib.cryptography import cryptographic_util from fideslib.db import session from sqlalchemy.orm import Session @@ -14,7 +15,6 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util import cryptographic_util from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset diff --git a/tests/fixtures/saas/zendesk_fixtures.py b/tests/fixtures/saas/zendesk_fixtures.py index b62070925f..fc119afe92 100644 --- a/tests/fixtures/saas/zendesk_fixtures.py +++ b/tests/fixtures/saas/zendesk_fixtures.py @@ -5,6 +5,7 @@ import pytest import requests from fideslib.core.config import load_toml +from fideslib.cryptography import cryptographic_util from sqlalchemy.orm import Session from fidesops.models.connectionconfig import ( @@ -13,7 +14,6 @@ ConnectionType, ) from fidesops.models.datasetconfig import DatasetConfig -from fidesops.util import cryptographic_util from fidesops.util.saas_util import load_config from tests.fixtures.application_fixtures import load_dataset diff --git a/tests/graph/test_graph.py b/tests/graph/test_graph.py index 97da4a3bab..a8020830a4 100644 --- a/tests/graph/test_graph.py +++ b/tests/graph/test_graph.py @@ -70,9 +70,9 @@ def test_retry_decorator(privacy_request, policy): CollectionAddress("postgres_example", "payment_card") ] - config.execution.TASK_RETRY_COUNT = 5 - config.execution.TASK_RETRY_DELAY = 0.1 - config.execution.TASK_RETRY_BACKOFF = 0.01 + config.execution.task_retry_count = 5 + config.execution.task_retry_delay = 0.1 + config.execution.task_retry_backoff = 0.01 class TestRetryDecorator: def __init__(self): diff --git a/tests/integration_tests/saas/test_outreach_task.py b/tests/integration_tests/saas/test_outreach_task.py index ec0ab6a75b..89b63989f1 100644 --- a/tests/integration_tests/saas/test_outreach_task.py +++ b/tests/integration_tests/saas/test_outreach_task.py @@ -97,7 +97,7 @@ def test_outreach_erasure_request_task( outreach_create_erasure_data, ) -> None: """Full erasure request based on the Outreach SaaS config""" - config.execution.MASKING_STRICT = False # Allow Delete + config.execution.masking_strict = False # Allow Delete privacy_request = PrivacyRequest( id=f"test_outreach_erasure_request_task_{random.randint(0, 1000)}" @@ -142,4 +142,4 @@ def test_outreach_erasure_request_task( # cannot verify success immediately as this can take days, weeks to process assert x == {f"{dataset_name}:prospects": 1, f"{dataset_name}:recipients": 1} - config.execution.MASKING_STRICT = True + config.execution.masking_strict = True diff --git a/tests/integration_tests/saas/test_segment_task.py b/tests/integration_tests/saas/test_segment_task.py index be9f10f782..c84cfa6117 100644 --- a/tests/integration_tests/saas/test_segment_task.py +++ b/tests/integration_tests/saas/test_segment_task.py @@ -148,7 +148,7 @@ def test_segment_saas_erasure_request_task( segment_erasure_data, ) -> None: """Full erasure request based on the Segment SaaS config""" - config.execution.MASKING_STRICT = False # Allow GDPR Delete + config.execution.masking_strict = False # Allow GDPR Delete # Create user for GDPR delete erasure_email = segment_erasure_identity_email @@ -225,4 +225,4 @@ def test_segment_saas_erasure_request_task( "segment_connector_example:track_events": 0, } - config.execution.MASKING_STRICT = True # Reset + config.execution.masking_strict = True # Reset diff --git a/tests/integration_tests/saas/test_stripe_task.py b/tests/integration_tests/saas/test_stripe_task.py index cbb4e6a374..687103418f 100644 --- a/tests/integration_tests/saas/test_stripe_task.py +++ b/tests/integration_tests/saas/test_stripe_task.py @@ -1072,7 +1072,7 @@ def test_stripe_erasure_request_task( ) # Run erasure with masking_strict = False so both update and delete actions can be used - config.execution.MASKING_STRICT = False + config.execution.masking_strict = False x = graph_task.run_erasure( privacy_request, @@ -1170,4 +1170,4 @@ def test_stripe_erasure_request_task( assert subscriptions == [] # reset - config.execution.MASKING_STRICT = True + config.execution.masking_strict = True diff --git a/tests/integration_tests/saas/test_zendesk_task.py b/tests/integration_tests/saas/test_zendesk_task.py index 092fa2ea78..9d9228fa35 100644 --- a/tests/integration_tests/saas/test_zendesk_task.py +++ b/tests/integration_tests/saas/test_zendesk_task.py @@ -187,7 +187,7 @@ def test_zendesk_erasure_request_task( zendesk_create_erasure_data, ) -> None: """Full erasure request based on the zendesk SaaS config""" - config.execution.MASKING_STRICT = False # Allow Delete + config.execution.masking_strict = False # Allow Delete privacy_request = PrivacyRequest( id=f"test_zendesk_erasure_request_task_{random.randint(0, 1000)}" @@ -332,4 +332,4 @@ def test_zendesk_erasure_request_task( # Since ticket is deleted, it won't be available so response is 404 assert response.status_code == 404 - config.execution.MASKING_STRICT = True + config.execution.masking_strict = True diff --git a/tests/integration_tests/test_sql_task.py b/tests/integration_tests/test_sql_task.py index 368d901e38..7c35849396 100644 --- a/tests/integration_tests/test_sql_task.py +++ b/tests/integration_tests/test_sql_task.py @@ -956,9 +956,9 @@ def test_retry_access_request( policy, integration_postgres_config, ): - config.execution.TASK_RETRY_COUNT = 1 - config.execution.TASK_RETRY_DELAY = 0.1 - config.execution.TASK_RETRY_BACKOFF = 0.01 + config.execution.task_retry_count = 1 + config.execution.task_retry_delay = 0.1 + config.execution.task_retry_backoff = 0.01 dataset = FidesopsDataset(**example_datasets[0]) graph = convert_dataset_to_graph(dataset, integration_postgres_config.key) @@ -1007,9 +1007,9 @@ def test_retry_erasure( policy, integration_postgres_config, ): - config.execution.TASK_RETRY_COUNT = 2 - config.execution.TASK_RETRY_DELAY = 0.1 - config.execution.TASK_RETRY_BACKOFF = 0.01 + config.execution.task_retry_count = 2 + config.execution.task_retry_delay = 0.1 + config.execution.task_retry_backoff = 0.01 dataset = FidesopsDataset(**example_datasets[0]) graph = convert_dataset_to_graph(dataset, integration_postgres_config.key) diff --git a/tests/models/test_client.py b/tests/models/test_client.py index 836c6f1534..a551864bfb 100644 --- a/tests/models/test_client.py +++ b/tests/models/test_client.py @@ -10,8 +10,8 @@ class TestClientModel: def test_create_client_and_secret(self, db: Session) -> None: new_client, secret = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, ) assert new_client.hashed_secret is not None @@ -26,13 +26,13 @@ def test_create_client_and_secret(self, db: Session) -> None: def test_get_client(self, db: Session, oauth_client) -> None: client = ClientDetail.get( db, - object_id=config.security.OAUTH_ROOT_CLIENT_ID, + object_id=config.security.oauth_root_client_id, config=config, scopes=SCOPE_REGISTRY, ) hashed_access_key = hash_with_salt( - config.security.OAUTH_ROOT_CLIENT_SECRET.encode(config.security.ENCODING), + config.security.oauth_root_client_secret.encode(config.security.ENCODING), client.salt.encode(config.security.ENCODING), ) @@ -47,8 +47,8 @@ def test_get_client(self, db: Session, oauth_client) -> None: def test_credentials_valid(self, db: Session) -> None: new_client, secret = ClientDetail.create_client_and_secret( db, - config.security.OAUTH_CLIENT_ID_LENGTH_BYTES, - config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES, + config.security.oauth_client_id_length_bytes, + config.security.oauth_client_secret_length_bytes, ) assert new_client.credentials_valid("this-is-not-the-right-secret") is False diff --git a/tests/service/authentication/test_authentication_strategy_basic.py b/tests/service/authentication/test_authentication_strategy_basic.py index 281b2288a7..edc896179f 100644 --- a/tests/service/authentication/test_authentication_strategy_basic.py +++ b/tests/service/authentication/test_authentication_strategy_basic.py @@ -1,10 +1,10 @@ import pytest +from fideslib.cryptography.cryptographic_util import bytes_to_b64_str from requests import PreparedRequest, Request from fidesops.common_exceptions import ValidationError as FidesopsValidationError from fidesops.models.connectionconfig import ConnectionConfig from fidesops.service.authentication.authentication_strategy_factory import get_strategy -from fidesops.util.cryptographic_util import bytes_to_b64_str def test_basic_auth_with_username_and_password(): diff --git a/tests/service/connectors/test_queryconfig.py b/tests/service/connectors/test_queryconfig.py index f28f3a0d65..089f9dda30 100644 --- a/tests/service/connectors/test_queryconfig.py +++ b/tests/service/connectors/test_queryconfig.py @@ -924,14 +924,14 @@ def test_get_masking_request( method="DELETE", path="/api/0///" ) # Delete endpoint not used because MASKING_STRICT is True - assert config.execution.MASKING_STRICT is True + assert config.execution.masking_strict is True query_config = SaaSQueryConfig(conversations, endpoints, {}) saas_request = query_config.get_masking_request() assert saas_request is None # Override MASKING_STRICT to False - config.execution.MASKING_STRICT = False + config.execution.masking_strict = False # Now delete endpoint is selected as conversations masking request saas_request: SaaSRequest = query_config.get_masking_request() @@ -950,5 +950,5 @@ def test_get_masking_request( assert saas_request.method == "PUT" # Reset - config.execution.MASKING_STRICT = True + config.execution.masking_strict = True del endpoints["conversations"].requests["delete"] diff --git a/tests/service/privacy_request/request_runner_service_test.py b/tests/service/privacy_request/request_runner_service_test.py index 2bd9a4ee4c..1a28428f97 100644 --- a/tests/service/privacy_request/request_runner_service_test.py +++ b/tests/service/privacy_request/request_runner_service_test.py @@ -1390,7 +1390,7 @@ def test_run_webhooks_ap_scheduler_cleanup( privacy_request, policy_pre_execution_webhooks, ): - config.redis.DEFAULT_TTL_SECONDS = ( + config.redis.default_ttl_seconds = ( 1 # Set redis cache to expire very quickly for testing purposes ) mock_trigger_policy_webhook.side_effect = PrivacyRequestPaused( diff --git a/tests/tasks/test_celery.py b/tests/tasks/test_celery.py index 35a54a723f..c6f0b28e77 100644 --- a/tests/tasks/test_celery.py +++ b/tests/tasks/test_celery.py @@ -16,8 +16,8 @@ def multiply(x, y): def test_celery_default_config() -> None: celery_app = _create_celery() - assert celery_app.conf["broker_url"] == config.redis.CONNECTION_URL - assert celery_app.conf["result_backend"] == config.redis.CONNECTION_URL + assert celery_app.conf["broker_url"] == config.redis.connection_url + assert celery_app.conf["result_backend"] == config.redis.connection_url assert celery_app.conf["event_queue_prefix"] == "fidesops_worker" assert celery_app.conf["default_queue_name"] == "fidesops" diff --git a/tests/util/encryption/test_aes_gcm_encryption_scheme.py b/tests/util/encryption/test_aes_gcm_encryption_scheme.py index 35f48209eb..7109f9553e 100644 --- a/tests/util/encryption/test_aes_gcm_encryption_scheme.py +++ b/tests/util/encryption/test_aes_gcm_encryption_scheme.py @@ -1,6 +1,6 @@ import pytest +from fideslib.cryptography import cryptographic_util -from fidesops.util import cryptographic_util from fidesops.util.encryption.aes_gcm_encryption_scheme import ( decrypt, encrypt_verify_secret_length, diff --git a/tests/util/test_cache.py b/tests/util/test_cache.py index 4e53af3e14..d3441924c5 100644 --- a/tests/util/test_cache.py +++ b/tests/util/test_cache.py @@ -18,8 +18,8 @@ def test_cache_set_with_autoexpire(cache: FidesopsRedis) -> None: ttl_range = list( range( # Let's give this a 3 second buffer time to still pass - config.redis.DEFAULT_TTL_SECONDS - 2, - config.redis.DEFAULT_TTL_SECONDS + 1, + config.redis.default_ttl_seconds - 2, + config.redis.default_ttl_seconds + 1, ) ) cache.set_with_autoexpire(key, value) diff --git a/tests/util/test_cryptographic_util.py b/tests/util/test_cryptographic_util.py deleted file mode 100644 index f2833c6c5a..0000000000 --- a/tests/util/test_cryptographic_util.py +++ /dev/null @@ -1,47 +0,0 @@ -from fidesops.core.config import config -from fidesops.util.cryptographic_util import ( - b64_str_to_bytes, - b64_str_to_str, - bytes_to_b64_str, - hash_with_salt, - str_to_b64_str, -) - - -def test_hash_with_salt() -> None: - plain_text = "This is Plaintext. Not hashed. or salted. or chopped. or grilled." - salt = "adobo" - - expected_hash = "3318b888645e6599289be9bee8ac0af2e63eb095213b7269f84845303abde55c7c0f9879cd69d7f453716e439ba38dd8d9b7f0bec67fe9258fb55d90e94c972d" - hashed = hash_with_salt( - plain_text.encode(config.security.ENCODING), - salt.encode(config.security.ENCODING), - ) - - assert hashed == expected_hash - - -def test_bytes_to_b64_str() -> None: - byte_string = b"https://www.google.com" - b64_string = "aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbQ==" - result = bytes_to_b64_str(byte_string) - assert b64_string == result - - -def test_b64_str_to_bytes() -> None: - b64_string = "aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbQ==" - result = b64_str_to_bytes(b64_string) - assert b"https://www.google.com" == result - - -def test_b64_str_to_str() -> None: - b64_string = "aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbQ==" - result = b64_str_to_str(b64_string) - assert "https://www.google.com" == result - - -def test_str_to_b64_str() -> None: - orig_string = "https://www.google.com" - b64_string = "aHR0cHM6Ly93d3cuZ29vZ2xlLmNvbQ==" - result = str_to_b64_str(orig_string) - assert b64_string == result diff --git a/tests/util/test_jwt_util.py b/tests/util/test_jwt_util.py index 33b9ad093b..b5d4c6764c 100644 --- a/tests/util/test_jwt_util.py +++ b/tests/util/test_jwt_util.py @@ -14,9 +14,9 @@ def test_jwe_create_and_extract() -> None: payload = {"hello": "hi there"} - jwt_string = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + jwt_string = generate_jwe(json.dumps(payload), config.security.app_encryption_key) payload_from_svc = json.loads( - extract_payload(jwt_string, config.security.APP_ENCRYPTION_KEY) + extract_payload(jwt_string, config.security.app_encryption_key) ) assert payload_from_svc["hello"] == payload["hello"] @@ -29,31 +29,31 @@ def test_token_expired(oauth_client): } # Create a token with a very old issued at date. - access_token = generate_jwe(json.dumps(payload), config.security.APP_ENCRYPTION_KEY) + access_token = generate_jwe(json.dumps(payload), config.security.app_encryption_key) extracted = json.loads( - extract_payload(access_token, config.security.APP_ENCRYPTION_KEY) + extract_payload(access_token, config.security.app_encryption_key) ) assert extracted[JWE_PAYLOAD_CLIENT_ID] == oauth_client.id issued_at = datetime.fromisoformat(extracted[JWE_ISSUED_AT]) assert issued_at == datetime(2020, 1, 1) assert extracted[JWE_PAYLOAD_SCOPES] == oauth_client.scopes assert ( - is_token_expired(issued_at, config.security.OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES) + is_token_expired(issued_at, config.security.oauth_access_token_expire_minutes) is True ) # Create a token now access_token = oauth_client.create_access_code_jwe( - config.security.APP_ENCRYPTION_KEY + config.security.app_encryption_key ) extracted = json.loads( - extract_payload(access_token, config.security.APP_ENCRYPTION_KEY) + extract_payload(access_token, config.security.app_encryption_key) ) assert ( is_token_expired( datetime.fromisoformat(extracted[JWE_ISSUED_AT]), - config.security.OAUTH_ACCESS_TOKEN_EXPIRE_MINUTES, + config.security.oauth_access_token_expire_minutes, ) is False )