diff --git a/.fides/dataset.yml b/.fides/dataset.yml index 14b825c475..54d964d29f 100644 --- a/.fides/dataset.yml +++ b/.fides/dataset.yml @@ -144,7 +144,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: data_categories + - name: ctl_data_categories description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -220,7 +220,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: data_qualifiers + - name: ctl_data_qualifiers description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -296,7 +296,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: data_subjects + - name: ctl_data_subjects description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -381,7 +381,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: data_uses + - name: ctl_data_uses description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -493,7 +493,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: datasets + - name: ctl_datasets description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -635,7 +635,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: evaluations + - name: ctl_evaluations description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -814,7 +814,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: organizations + - name: ctl_organizations description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -921,7 +921,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: policies + - name: ctl_policies description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -990,7 +990,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: registries + - name: ctl_registries description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified @@ -1052,7 +1052,7 @@ dataset: data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified retention: null fields: null - - name: systems + - name: ctl_systems description: null data_categories: null data_qualifier: aggregated.anonymized.unlinked_pseudonymized.pseudonymized.identified diff --git a/CHANGELOG.md b/CHANGELOG.md index 471eea0dc9..182e1b77bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -25,6 +25,7 @@ The types of changes are: ### Changed * Upgraded base Docker version to Python 3.9 and updated all other references from 3.8 -> 3.9 [#974](https://github.com/ethyca/fides/pull/974) +* Prepend all database tables with `ctl_` [#979](https://github.com/ethyca/fides/pull/979) * Moved the `admin-ui` code down one level into a `ctl` subdir [#970](https://github.com/ethyca/fides/pull/970) ## [1.8.1](https://github.com/ethyca/fides/compare/1.8.0...1.8.1) - 2022-08-08 diff --git a/src/fidesctl/api/ctl/migrations/versions/4fc34906c389_.py b/src/fidesctl/api/ctl/migrations/versions/4fc34906c389_.py new file mode 100644 index 0000000000..ed1bdf0c8a --- /dev/null +++ b/src/fidesctl/api/ctl/migrations/versions/4fc34906c389_.py @@ -0,0 +1,23 @@ +"""Merge heads + +Revision ID: 4fc34906c389 +Revises: aaa81d97a6f6, f131a1263a10 +Create Date: 2022-08-17 19:35:33.413593 + +""" +import sqlalchemy as sa +from alembic import op + +# revision identifiers, used by Alembic. +revision = "4fc34906c389" +down_revision = ("aaa81d97a6f6", "f131a1263a10") +branch_labels = None +depends_on = None + + +def upgrade(): + pass + + +def downgrade(): + pass diff --git a/src/fidesctl/api/ctl/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py b/src/fidesctl/api/ctl/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py index c7c6a0a8ab..9927ddbf42 100644 --- a/src/fidesctl/api/ctl/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py +++ b/src/fidesctl/api/ctl/migrations/versions/7c851d8a102a_add_created_at_and_updated_at.py @@ -16,40 +16,45 @@ branch_labels = None depends_on = None +SQL_MODEL_LIST = [ + "data_categories", + "data_qualifiers", + "data_uses", + "data_subjects", + "datasets", + "evaluations", + "policies", + "registries", + "systems", + "organizations", +] + def upgrade(): - for k, model in sql_model_map.items(): - if k not in ( - "client_detail", - "fides_user", - "fides_user_permissions", - ): - op.add_column( - model.__tablename__, - Column( - "created_at", - DateTime(timezone=True), - server_default=text("now()"), - nullable=True, - ), - ) - op.add_column( - model.__tablename__, - Column( - "updated_at", - DateTime(timezone=True), - server_default=text("now()"), - nullable=True, - ), - ) + + for table_name in SQL_MODEL_LIST: + op.add_column( + table_name, + Column( + "created_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) + op.add_column( + table_name, + Column( + "updated_at", + DateTime(timezone=True), + server_default=text("now()"), + nullable=True, + ), + ) def downgrade(): - for k, model in sql_model_map.items(): - if k not in ( - "client_detail", - "fides_user", - "fides_user_permissions", - ): - op.drop_column(model.__tablename__, "created_at") - op.drop_column(model.__tablename__, "updated_at") + + for table_name in SQL_MODEL_LIST: + op.drop_column(table_name, "created_at") + op.drop_column(table_name, "updated_at") diff --git a/src/fidesctl/api/ctl/migrations/versions/aaa81d97a6f6_prepend_tables_with_ctl.py b/src/fidesctl/api/ctl/migrations/versions/aaa81d97a6f6_prepend_tables_with_ctl.py new file mode 100644 index 0000000000..0a121dc8ac --- /dev/null +++ b/src/fidesctl/api/ctl/migrations/versions/aaa81d97a6f6_prepend_tables_with_ctl.py @@ -0,0 +1,168 @@ +"""prepend tables with ctl + +Revision ID: aaa81d97a6f6 +Revises: f53e04e5b7f5 +Create Date: 2022-08-11 09:59:38.709501 + +""" +from alembic import op + +# revision identifiers, used by Alembic. +revision = "aaa81d97a6f6" +down_revision = "f53e04e5b7f5" +branch_labels = None +depends_on = None + + +def upgrade(): + """Rename tables to avoid collisions with fidesops.""" + + # Data Categories + op.rename_table("data_categories", "ctl_data_categories") + op.execute("ALTER INDEX data_categories_pkey RENAME TO ctl_data_categories_pkey") + op.execute( + "ALTER INDEX ix_data_categories_fides_key RENAME TO ix_ctl_data_categories_fides_key" + ) + op.execute("ALTER INDEX ix_data_categories_id RENAME TO ix_ctl_data_categories_id") + + # Data Subjects + op.rename_table("data_subjects", "ctl_data_subjects") + op.execute("ALTER INDEX data_subjects_pkey RENAME TO ctl_data_subjects_pkey") + op.execute( + "ALTER INDEX ix_data_subjects_fides_key RENAME TO ix_ctl_data_subjects_fides_key" + ) + op.execute("ALTER INDEX ix_data_subjects_id RENAME TO ix_ctl_data_subjects_id") + + # Data Uses + op.rename_table("data_uses", "ctl_data_uses") + op.execute("ALTER INDEX data_uses_pkey RENAME TO ctl_data_uses_pkey") + op.execute( + "ALTER INDEX ix_data_uses_fides_key RENAME TO ix_ctl_data_uses_fides_key" + ) + op.execute("ALTER INDEX ix_data_uses_id RENAME TO ix_ctl_data_uses_id") + + # Data Qualifiers + op.rename_table("data_qualifiers", "ctl_data_qualifiers") + op.execute("ALTER INDEX data_qualifiers_pkey RENAME TO ctl_data_qualifiers_pkey") + op.execute( + "ALTER INDEX ix_data_qualifiers_fides_key RENAME TO ix_ctl_data_qualifiers_fides_key" + ) + op.execute("ALTER INDEX ix_data_qualifiers_id RENAME TO ix_ctl_data_qualifiers_id") + + # Datasets + op.rename_table("datasets", "ctl_datasets") + op.execute("ALTER INDEX data_sets_pkey RENAME TO ctl_datasets_pkey") + op.execute("ALTER INDEX ix_datasets_fides_key RENAME TO ix_ctl_datasets_fides_key") + op.execute("ALTER INDEX ix_datasets_id RENAME TO ix_ctl_datasets_id") + + # Evaluations + op.rename_table("evaluations", "ctl_evaluations") + op.execute("ALTER INDEX evaluations_pkey RENAME TO ctl_evaluations_pkey") + op.execute( + "ALTER INDEX ix_evaluations_fides_key RENAME TO ix_ctl_evaluations_fides_key" + ) + op.execute("ALTER INDEX ix_evaluations_id RENAME TO ix_ctl_evaluations_id") + + # Organizations + op.rename_table("organizations", "ctl_organizations") + op.execute("ALTER INDEX organizations_pkey RENAME TO ctl_organizations_pkey") + op.execute( + "ALTER INDEX ix_organizations_fides_key RENAME TO ix_ctl_organizations_fides_key" + ) + op.execute("ALTER INDEX ix_organizations_id RENAME TO ix_ctl_organizations_id") + + # Policies + op.rename_table("policies", "ctl_policies") + op.execute("ALTER INDEX policies_pkey RENAME TO ctl_policies_pkey") + op.execute("ALTER INDEX ix_policies_fides_key RENAME TO ix_ctl_policies_fides_key") + op.execute("ALTER INDEX ix_policies_id RENAME TO ix_ctl_policies_id") + + # Registries + op.rename_table("registries", "ctl_registries") + op.execute("ALTER INDEX registries_pkey RENAME TO ctl_registries_pkey") + op.execute( + "ALTER INDEX ix_registries_fides_key RENAME TO ix_ctl_registries_fides_key" + ) + op.execute("ALTER INDEX ix_registries_id RENAME TO ix_ctl_registries_id") + + # Systems + op.rename_table("systems", "ctl_systems") + op.execute("ALTER INDEX systems_pkey RENAME TO ctl_systems_pkey") + op.execute("ALTER INDEX ix_systems_fides_key RENAME TO ix_ctl_systems_fides_key") + op.execute("ALTER INDEX ix_systems_id RENAME TO ix_ctl_systems_id") + + +def downgrade(): + + op.rename_table("ctl_data_categories", "data_categories") + op.execute("ALTER INDEX ctl_data_categories_pkey RENAME TO data_categories_pkey") + op.execute( + "ALTER INDEX ix_ctl_data_categories_fides_key RENAME TO ix_data_categories_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_data_categories_id RENAME TO ix_data_categories_id") + + # Data Subjects + op.rename_table("ctl_data_subjects", "data_subjects") + op.execute("ALTER INDEX ctl_data_subjects_pkey RENAME TO data_subjects_pkey") + op.execute( + "ALTER INDEX ix_ctl_data_subjects_fides_key RENAME TO ix_data_subjects_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_data_subjects_id RENAME TO ix_data_subjects_id") + + # Data Uses + op.rename_table("ctl_data_uses", "data_uses") + op.execute("ALTER INDEX ctl_data_uses_pkey RENAME TO data_uses_pkey") + op.execute( + "ALTER INDEX ix_ctl_data_uses_fides_key RENAME TO ix_data_uses_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_data_uses_id RENAME TO ix_data_uses_id") + + # Data Qualifiers + op.rename_table("ctl_data_qualifiers", "data_qualifiers") + op.execute("ALTER INDEX ctl_data_qualifiers_pkey RENAME TO data_qualifiers_pkey") + op.execute( + "ALTER INDEX ix_ctl_data_qualifiers_fides_key RENAME TO ix_data_qualifiers_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_data_qualifiers_id RENAME TO ix_data_qualifiers_id") + + # Datasets + op.rename_table("ctl_datasets", "datasets") + op.execute("ALTER INDEX ctl_datasets_pkey RENAME TO datasets_pkey") + op.execute("ALTER INDEX ix_ctl_datasets_fides_key RENAME TO ix_datasets_fides_key") + op.execute("ALTER INDEX ix_ctl_datasets_id RENAME TO ix_datasets_id") + + # Evaluations + op.rename_table("ctl_evaluations", "evaluations") + op.execute("ALTER INDEX ctl_evaluations_pkey RENAME TO evaluations_pkey") + op.execute( + "ALTER INDEX ix_ctl_evaluations_fides_key RENAME TO ix_evaluations_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_evaluations_id RENAME TO ix_evaluations_id") + + # Organizations + op.rename_table("ctl_organizations", "organizations") + op.execute("ALTER INDEX ctl_organizations_pkey RENAME TO organizations_pkey") + op.execute( + "ALTER INDEX ix_ctl_organizations_fides_key RENAME TO ix_organizations_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_organizations_id RENAME TO ix_organizations_id") + + # Policies + op.rename_table("ctl_policies", "policies") + op.execute("ALTER INDEX ctl_policies_pkey RENAME TO policies_pkey") + op.execute("ALTER INDEX ix_ctl_policies_fides_key RENAME TO ix_policies_fides_key") + op.execute("ALTER INDEX ix_ctl_policies_id RENAME TO ix_policies_id") + + # Registries + op.rename_table("ctl_registries", "registries") + op.execute("ALTER INDEX ctl_registries_pkey RENAME TO registries_pkey") + op.execute( + "ALTER INDEX ix_ctl_registries_fides_key RENAME TO ix_registries_fides_key" + ) + op.execute("ALTER INDEX ix_ctl_registries_id RENAME TO ix_registries_id") + + # Systems + op.rename_table("ctl_systems", "systems") + op.execute("ALTER INDEX ctl_systems_pkey RENAME TO systems_pkey") + op.execute("ALTER INDEX ix_ctl_systems_fides_key RENAME TO ix_systems_fides_key") + op.execute("ALTER INDEX ix_ctl_systems_id RENAME TO ix_systems_id") diff --git a/src/fidesctl/api/ctl/sql_models.py b/src/fidesctl/api/ctl/sql_models.py index b8aa1b41bf..74eb9213ec 100644 --- a/src/fidesctl/api/ctl/sql_models.py +++ b/src/fidesctl/api/ctl/sql_models.py @@ -91,7 +91,7 @@ class DataCategory(Base, FidesBase): The SQL model for the DataCategory resource. """ - __tablename__ = "data_categories" + __tablename__ = "ctl_data_categories" parent_key = Column(Text) is_default = Column(BOOLEAN, default=False) @@ -102,7 +102,7 @@ class DataQualifier(Base, FidesBase): The SQL model for the DataQualifier resource. """ - __tablename__ = "data_qualifiers" + __tablename__ = "ctl_data_qualifiers" parent_key = Column(Text) is_default = Column(BOOLEAN, default=False) @@ -113,7 +113,7 @@ class DataSubject(Base, FidesBase): The SQL model for the DataSubject resource. """ - __tablename__ = "data_subjects" + __tablename__ = "ctl_data_subjects" rights = Column(JSON, nullable=True) automated_decisions_or_profiling = Column(BOOLEAN, nullable=True) is_default = Column(BOOLEAN, default=False) @@ -124,7 +124,7 @@ class DataUse(Base, FidesBase): The SQL model for the DataUse resource. """ - __tablename__ = "data_uses" + __tablename__ = "ctl_data_uses" parent_key = Column(Text) legal_basis = Column(Text) @@ -141,7 +141,7 @@ class Dataset(Base, FidesBase): The SQL model for the Dataset resource. """ - __tablename__ = "datasets" + __tablename__ = "ctl_datasets" meta = Column(JSON) data_categories = Column(ARRAY(String)) @@ -159,7 +159,7 @@ class Evaluation(Base): The SQL model for the Evaluation resource. """ - __tablename__ = "evaluations" + __tablename__ = "ctl_evaluations" fides_key = Column(String, primary_key=True, index=True, unique=True) status = Column(String) @@ -174,7 +174,7 @@ class Organization(Base, FidesBase): """ # It inherits this from FidesModel but Organization's don't have this field - __tablename__ = "organizations" + __tablename__ = "ctl_organizations" organization_parent_key = Column(String, nullable=True) controller = Column(PGEncryptedString, nullable=True) @@ -190,7 +190,7 @@ class Policy(Base, FidesBase): The SQL model for the Policy resource. """ - __tablename__ = "policies" + __tablename__ = "ctl_policies" rules = Column(JSON) @@ -201,7 +201,7 @@ class Registry(Base, FidesBase): The SQL model for the Registry resource. """ - __tablename__ = "registries" + __tablename__ = "ctl_registries" # System @@ -210,7 +210,7 @@ class System(Base, FidesBase): The SQL model for the system resource. """ - __tablename__ = "systems" + __tablename__ = "ctl_systems" registry_id = Column(String) meta = Column(JSON)