diff --git a/CHANGELOG.md b/CHANGELOG.md index 6a4953297..1940db908 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -31,6 +31,7 @@ The types of changes are: * Adds searching of ConnectionConfigs [#641](https://github.com/ethyca/fidesops/pull/641) * Added `AdminUiSettings` to the `log_all_config_values` helper method [#647](https://github.com/ethyca/fidesops/pull/647) * Prettier formatting CI check for frontend code [#655](https://github.com/ethyca/fidesops/pull/655) +* Adds default policies [#654](https://github.com/ethyca/fidesops/pull/654) ### Changed diff --git a/docs/fidesops/docs/guides/configuration_reference.md b/docs/fidesops/docs/guides/configuration_reference.md index c13a66da8..6989c2490 100644 --- a/docs/fidesops/docs/guides/configuration_reference.md +++ b/docs/fidesops/docs/guides/configuration_reference.md @@ -117,6 +117,8 @@ Please note: The configuration is case-sensitive, so the variables must be speci | `FIDESOPS__DEV_MODE` | False | If "True", the fidesops server will log error tracebacks, and log details of third party requests. This variable should always be set to "False" in production systems.| | `FIDESOPS__CONFIG_PATH` | None | If this variable is set to a path, that path will be used to load .toml files first. That is, any .toml files on this path will override any installed .toml files. | | `FIDESOPS__DATABASE__SQLALCHEMY_DATABASE_URI` | None | An optional override for the URI used for the database connection. | +| `TESTING` | False | This variable does not need to be set - Pytest will set it to True when running unit tests, so we run against the test database. | + ## - Reporting a running application's configuration diff --git a/docs/fidesops/docs/guides/policies.md b/docs/fidesops/docs/guides/policies.md index 26553eab7..bc878e5ac 100644 --- a/docs/fidesops/docs/guides/policies.md +++ b/docs/fidesops/docs/guides/policies.md @@ -1,5 +1,13 @@ # How-To: Configure Policies +In this section we'll cover: + +- What is a Fidesops Policy? +- How do I create a Policy? +- How do I add Rules to my Policy? +- What default Policies ship with Fidesops? + +## Policy Definition A Policy is a set of instructions (or "Rules") that are executed when a user submits a request to retrieve or delete their data (the user makes a "Privacy Request"). Each Rule contains an "execution strategy": @@ -32,7 +40,7 @@ Each operation takes an array of objects, so you can create more than one at a t - any objects existing that are not specified in the request will not be deleted -## Create a Policy +## Creating a Policy Let's say you want to make a Policy that contains rules about a user's email address. You would start by first creating a Policy object: @@ -140,3 +148,10 @@ It's illegal to erase the same data twice within a Policy, so you should take ca And lastly, access rules will always run before erasure rules. +## Default Policies + +Fidesops ships with two default Policies: `download` (for access requests) and `delete` (for erasure requests). +The `download` Policy is configured to retrieve `user.provided.identifiable` data and upload to a local storage location. +The `delete` Policy is set up to mask `user.provided.identifiable` data with the string: `MASKED`. + +These autogenerated Policies are intended for use in a test environment. In production deployments, you should configure separate Policies with proper storage destinations that target and process the appropriate fields. diff --git a/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py b/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py new file mode 100644 index 000000000..74e1eb1d5 --- /dev/null +++ b/src/fidesops/migrations/versions/55d61eb8ed12_add_default_policies.py @@ -0,0 +1,384 @@ +"""add default policies + +Revision ID: 55d61eb8ed12 +Revises: b3b68c87c4a0 +Create Date: 2022-06-13 19:26:24.197262 + +""" +import logging +from typing import Optional, Tuple +from uuid import uuid4 + +from alembic import op +from sqlalchemy import text +from sqlalchemy.dialects import postgresql +from sqlalchemy.engine import LegacyRow +from sqlalchemy.engine.base import Connection +from sqlalchemy.sql.elements import TextClause +from sqlalchemy_utils import StringEncryptedType +from sqlalchemy_utils.types.encrypted.encrypted_type import AesGcmEngine + +from fidesops.api.v1.scope_registry import SCOPE_REGISTRY +from fidesops.core.config import config +from fidesops.db.base import Policy, Rule, RuleTarget, StorageConfig +from fidesops.db.base_class import FidesopsBase, JSONTypeOverride +from fidesops.models.policy import ActionType, DrpAction +from fidesops.schemas.storage.storage import StorageType +from fidesops.service.masking.strategy.masking_strategy_string_rewrite import ( + STRING_REWRITE_STRATEGY_NAME, +) +from fidesops.util.cryptographic_util import ( + generate_salt, + generate_secure_random_string, + hash_with_salt, +) +from fidesops.util.data_category import DataCategory + +logging.basicConfig() +logger = logging.getLogger(__name__) +logger.setLevel(logging.INFO) +logging.getLogger("sqlalchemy.engine").setLevel(logging.INFO) + +revision = "55d61eb8ed12" +down_revision = "b3b68c87c4a0" +branch_labels = None +depends_on = None + +FIDESOPS_AUTOGENERATED_CLIENT_KEY = "fidesops_autogenerated_client" +FIDESOPS_AUTOGENERATED_STORAGE_KEY = "fidesops_autogenerated_storage_destination" +AUTOGENERATED_ACCESS_KEY = "download" +AUTOGENERATED_ERASURE_KEY = "delete" + +client_select_query: TextClause = text( + """SELECT client.id FROM client WHERE fides_key = :fides_key""" +) + +client_insert_query: TextClause = text( + """ + INSERT INTO client (id, hashed_secret, salt, scopes, fides_key) + VALUES (:client_id, :hashed_secret, :salt, :scopes, :fides_key) + """ +) + +client_delete_query: TextClause = text( + """DELETE FROM client WHERE client.id = :client_id""" +) + +storage_select_query: TextClause = text( + """SELECT storageconfig.id FROM storageconfig WHERE storageconfig.key = :storage_config_key""" +) +storage_insert_query: TextClause = text( + """ + INSERT INTO storageconfig(id, name, type, details, key, secrets, format) + VALUES(:storage_config_id, :storage_name, :storage_type, '{"naming": "request_id"}', + :storage_key, NULL, 'json') + """ +) +storage_delete_query: TextClause = text( + """DELETE FROM storageconfig WHERE storageconfig.id = :storage_id""" +) +policy_select_query: TextClause = text( + """ + SELECT policy.id FROM policy WHERE policy.key = :policy_key AND client_id = :client_id + """ +) +policy_insert_query: TextClause = text( + """ + INSERT INTO policy(id, name, key, drp_action, client_id) + VALUES(:policy_id, :policy_name, :key, :drp_action, :client_id) + """ +) +rule_select_query = text("""SELECT rule.id FROM rule where policy_id = :policy_id""") +rule_insert_query: TextClause = text( + """ + INSERT INTO rule (id, name, key, policy_id, action_type, masking_strategy, storage_destination_id, client_id) + VALUES (:rule_id, :rule_name, :rule_key, :policy_id, :action_type, :masking_strategy, :storage_id, :client_id) + """ +) +rule_target_insert_query: TextClause = text( + """ + INSERT INTO ruletarget (id, name, key, data_category, rule_id, client_id) + VALUES (:target_id, :target_name, :target_key, :data_category, :rule_id, :client_id) + """ +) + +delete_policy_query = text("""DELETE FROM policy WHERE policy.id = :policy_id""") +delete_rule_query = text("""DELETE FROM rule WHERE policy_id = :policy_id""") +delete_target_query = text("""DELETE FROM ruletarget WHERE rule_id IN :rule_ids""") + + +def generate_uuid(cls: FidesopsBase) -> str: + """ + Generates a uuid with a prefix based on the tablename to be used as the + record's ID value + """ + try: + prefix = f"{cls.__tablename__[:3]}_" + except AttributeError: + prefix = "" + uuid = str(uuid4()) + return f"{prefix}{uuid}" + + +def upgrade() -> None: + """Data migration only. + + Create an autogenerated client and storage destination, then use those to create + autogenerated 'download' and 'delete' policies if they don't already exist.""" + if config.is_test_mode: + logger.info(f"Skipping data migration in test mode (pytest)'") + return + + connection: Connection = op.get_bind() + storage_config_id: str = autogenerate_local_storage(connection) + client_id: str = autogenerate_client(connection) + + policy_query_by_key: TextClause = text( + """SELECT policy.id FROM policy WHERE policy.key = :policy_key""" + ) + access_results: Optional[LegacyRow] = connection.execute( + policy_query_by_key, {"policy_key": AUTOGENERATED_ACCESS_KEY} + ).first() + if not access_results: + # Only create a "download" policy if one does not already exist + autogenerate_access_policy(connection, client_id, storage_config_id) + + erasure_results: Optional[LegacyRow] = connection.execute( + policy_query_by_key, {"policy_key": AUTOGENERATED_ERASURE_KEY} + ).first() + if not erasure_results: + # Only create a "delete" policy if one does not already exist + autogenerate_erasure_policy(connection, client_id) + + +def downgrade() -> None: + """Data migration only. + + Remove 'download' and delete' policies if they were created by the autogenerated client, and then + attempt to remove the autogenerated client and local storage destination. + """ + if config.is_test_mode: + logger.info(f"Skipping data migration in test mode (pytest)'") + return + + connection: Connection = op.get_bind() + client_result: Optional[LegacyRow] = connection.execute( + client_select_query, {"fides_key": FIDESOPS_AUTOGENERATED_CLIENT_KEY} + ).first() + + if not client_result: + logger.info(f"No autogenerated client: '{FIDESOPS_AUTOGENERATED_CLIENT_KEY}'") + return + + access_policy_result: Optional[LegacyRow] = connection.execute( + policy_select_query, + {"policy_key": AUTOGENERATED_ACCESS_KEY, "client_id": client_result[0]}, + ).first() + + if access_policy_result: + logger.info( + f"Deleting autogenerated '{AUTOGENERATED_ACCESS_KEY}' access policy" + ) + access_rules: Tuple = tuple( + [ + rul.id + for rul in connection.execute( + rule_select_query, {"policy_id": access_policy_result[0]} + ) + ] + ) + + # Only delete "download" policy if it was created by the autogenerated client + connection.execute(delete_target_query, {"rule_ids": access_rules}) + connection.execute(delete_rule_query, {"policy_id": access_policy_result[0]}) + connection.execute(delete_policy_query, {"policy_id": access_policy_result[0]}) + + erasure_policy_result: Optional[LegacyRow] = connection.execute( + policy_select_query, + {"policy_key": AUTOGENERATED_ERASURE_KEY, "client_id": client_result[0]}, + ).first() + + if erasure_policy_result: + # Only delete "delete" policy if it was created by the autogenerated client + logger.info( + f"Deleting autogenerated '{AUTOGENERATED_ERASURE_KEY}' erasure policy" + ) + erasure_rules: Tuple = tuple( + [ + rul.id + for rul in connection.execute( + rule_select_query, {"policy_id": erasure_policy_result[0]} + ) + ] + ) + connection.execute(delete_target_query, {"rule_ids": erasure_rules}) + connection.execute(delete_rule_query, {"policy_id": erasure_policy_result[0]}) + connection.execute(delete_policy_query, {"policy_id": erasure_policy_result[0]}) + + try: + logger.info( + f"Deleting autogenerated client: '{FIDESOPS_AUTOGENERATED_CLIENT_KEY}'" + ) + connection.execute(client_delete_query, {"client_id": client_result[0]}) + + storage_result: Optional[LegacyRow] = connection.execute( + storage_select_query, + {"storage_config_key": FIDESOPS_AUTOGENERATED_STORAGE_KEY}, + ).first() + if storage_result: + logger.info( + f"Deleting autogenerated local storage: '{FIDESOPS_AUTOGENERATED_STORAGE_KEY}'" + ) + connection.execute(storage_delete_query, {"storage_id": storage_result[0]}) + except Exception: + # It's possible the client or storage config have been attached to other things + pass + + +def autogenerate_access_policy( + connection: Connection, client_id: str, storage_id: str +) -> None: + """Create an autogenerated 'download' access policy, with an access rule attached, + targeting user.provided.identifiable data""" + logger.info(f"Creating autogenerated '{AUTOGENERATED_ACCESS_KEY}' policy") + + policy_id: str = generate_uuid(Policy) + connection.execute( + policy_insert_query, + { + "policy_id": policy_id, + "policy_name": "Fidesops Autogenerated Access Policy", + "key": AUTOGENERATED_ACCESS_KEY, + "drp_action": DrpAction.access.value, + "client_id": client_id, + }, + ) + + rule_id: str = generate_uuid(Rule) + connection.execute( + rule_insert_query, + { + "rule_id": rule_id, + "rule_key": "fidesops_autogenerated_access_rule", + "rule_name": "Fidesops Autogenerated Access Rule", + "policy_id": policy_id, + "action_type": ActionType.access.value, + "storage_id": storage_id, + "client_id": client_id, + "masking_strategy": None, + }, + ) + + rule_target_id = generate_uuid(RuleTarget) + connection.execute( + rule_target_insert_query, + { + "target_id": rule_target_id, + "target_name": "Fidesops Autogenerated Access Target", + "target_key": "fidesops_autogenerated_access_target", + "data_category": DataCategory("user.provided.identifiable").value, + "rule_id": rule_id, + "client_id": client_id, + }, + ) + + +def autogenerate_erasure_policy(connection: Connection, client_id: str) -> None: + """Create an autogenerated 'deletion' erasure policy, with an erasure rule attached, + targeting user.provided.identifiable data""" + logger.info(f"Creating autogenerated '{AUTOGENERATED_ERASURE_KEY}' policy") + + policy_id: str = generate_uuid(Policy) + connection.execute( + policy_insert_query, + { + "policy_id": policy_id, + "policy_name": "Fidesops Autogenerated Erasure Policy", + "key": AUTOGENERATED_ERASURE_KEY, + "drp_action": DrpAction.deletion.value, + "client_id": client_id, + }, + ) + + rule_id: str = generate_uuid(Rule) + encryption: StringEncryptedType = StringEncryptedType( + JSONTypeOverride, config.security.APP_ENCRYPTION_KEY, AesGcmEngine, "pkcs5" + ) + connection.execute( + rule_insert_query, + { + "rule_id": rule_id, + "rule_name": "Fidesops Autogenerated Erasure Rule", + "rule_key": "fidesops_autogenerated_erasure_rule", + "policy_id": policy_id, + "action_type": ActionType.erasure.value, + "client_id": client_id, + "storage_id": None, + "masking_strategy": encryption.process_bind_param( + { + "strategy": STRING_REWRITE_STRATEGY_NAME, + "configuration": {"rewrite_value": "MASKED"}, + }, + postgresql, + ), + }, + ) + + rule_target_id: str = generate_uuid(Rule) + connection.execute( + rule_target_insert_query, + { + "target_id": rule_target_id, + "target_name": "Fidesops Autogenerated Erasure Target", + "target_key": "fidesops_autogenerated_erasure_target", + "data_category": DataCategory("user.provided.identifiable").value, + "rule_id": rule_id, + "client_id": client_id, + }, + ) + + +def autogenerate_local_storage(connection: Connection) -> str: + """Generate local storage for the access rule""" + logger.info( + f"Creating autogenerated local storage: '{FIDESOPS_AUTOGENERATED_STORAGE_KEY}'" + ) + + storage_config_id: str = generate_uuid(StorageConfig) + connection.execute( + storage_insert_query, + { + "storage_config_id": storage_config_id, + "storage_key": FIDESOPS_AUTOGENERATED_STORAGE_KEY, + "storage_type": StorageType.local.value, + "storage_name": "Fidesops Autogenerated Local Storage", + }, + ) + return storage_config_id + + +def autogenerate_client(connection: Connection) -> str: + """Generate a client for creating policies, rules, and ruletargets""" + logger.info(f"Creating autogenerated client: '{FIDESOPS_AUTOGENERATED_CLIENT_KEY}'") + + client_id: str = generate_secure_random_string( + config.security.OAUTH_CLIENT_ID_LENGTH_BYTES + ) + secret: str = generate_secure_random_string( + config.security.OAUTH_CLIENT_SECRET_LENGTH_BYTES + ) + salt: str = generate_salt() + connection.execute( + client_insert_query, + { + "client_id": client_id, + "hashed_secret": hash_with_salt( + secret.encode(config.security.ENCODING), + salt.encode(config.security.ENCODING), + ), + "salt": salt, + "scopes": SCOPE_REGISTRY, + "fides_key": FIDESOPS_AUTOGENERATED_CLIENT_KEY, + }, + ) + return client_id