From 035a63741684b5ea03d6756592bd4627b68b047f Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 6 May 2024 14:33:24 +0200 Subject: [PATCH 01/29] Rename datasets to s3_datasets --- .../modules/dataset_sharing/__init__.py | 2 +- .../modules/dataset_sharing/api/resolvers.py | 6 ++-- .../db/share_object_repositories.py | 4 +-- .../services/data_sharing_service.py | 2 +- .../services/dataset_sharing_alarm_service.py | 2 +- .../services/dataset_sharing_service.py | 10 +++--- .../services/share_item_service.py | 4 +-- .../share_managers/lf_share_manager.py | 2 +- .../s3_access_point_share_manager.py | 2 +- .../share_managers/s3_bucket_share_manager.py | 2 +- .../services/share_notification_service.py | 4 +-- .../services/share_object_service.py | 6 ++-- .../lakeformation_process_share.py | 2 +- .../s3_access_point_process_share.py | 2 +- .../s3_bucket_process_share.py | 2 +- .../tasks/dataset_subscription_task.py | 10 +++--- .../modules/datasets/api/dataset/__init__.py | 3 -- .../{datasets => s3_datasets}/__init__.py | 34 +++++++++---------- .../{datasets => s3_datasets}/api/__init__.py | 2 +- .../s3_datasets/api/dataset/__init__.py | 3 ++ .../api/dataset/input_types.py | 2 +- .../api/dataset/mutations.py | 4 +-- .../api/dataset/queries.py | 6 ++-- .../api/dataset/resolvers.py | 14 ++++---- .../api/dataset/types.py | 4 +-- .../api/profiling}/__init__.py | 2 +- .../api/profiling/input_types.py | 0 .../api/profiling/mutations.py | 2 +- .../api/profiling/queries.py | 2 +- .../api/profiling/resolvers.py | 6 ++-- .../api/profiling/types.py | 2 +- .../api/storage_location/__init__.py | 2 +- .../api/storage_location/input_types.py | 0 .../api/storage_location/mutations.py | 6 ++-- .../api/storage_location/queries.py | 2 +- .../api/storage_location/resolvers.py | 14 ++++---- .../api/storage_location/types.py | 2 +- .../api/table/__init__.py | 2 +- .../api/table/input_types.py | 2 +- .../api/table/mutations.py | 4 +-- .../api/table/queries.py | 6 ++-- .../api/table/resolvers.py | 8 ++--- .../api/table/types.py | 4 +-- .../api/table_column}/__init__.py | 2 +- .../api/table_column/input_types.py | 0 .../api/table_column/mutations.py | 2 +- .../api/table_column/queries.py | 2 +- .../api/table_column/resolvers.py | 4 +-- .../api/table_column/types.py | 2 +- .../{datasets => s3_datasets}/aws/__init__.py | 0 .../aws/athena_table_client.py | 2 +- .../aws/glue_dataset_client.py | 2 +- .../aws/glue_profiler_client.py | 4 +-- .../aws/glue_table_client.py | 2 +- .../aws/kms_dataset_client.py | 0 .../aws/lf_dataset_client.py | 2 +- .../aws/lf_table_client.py | 2 +- .../aws/s3_dataset_bucket_policy_client.py | 2 +- .../aws/s3_dataset_client.py | 2 +- .../aws/s3_location_client.py | 2 +- .../aws/s3_profiler_client.py | 0 .../aws/sns_dataset_client.py | 2 +- .../{datasets => s3_datasets}/cdk/__init__.py | 2 +- .../gluedatabasecustomresource/__init__.py | 0 .../gluedatabasecustomresource/index.py | 0 .../assets/glueprofilingjob/glue_script.py | 0 .../lakeformationdefaultsettings/__init__.py | 0 .../lakeformationdefaultsettings/index.py | 0 .../cdk/dataset_custom_resources_extension.py | 0 .../cdk/dataset_glue_profiler_extension.py | 0 .../cdk/dataset_stack.py | 6 ++-- .../cdk/env_role_dataset_databrew_policy.py | 2 +- .../cdk/env_role_dataset_glue_policy.py | 2 +- .../cdk/env_role_dataset_s3_policy.py | 4 +-- .../cdk/pivot_role_datasets_policy.py | 4 +-- .../{datasets => s3_datasets}/db/__init__.py | 0 .../db/dataset_bucket_repositories.py | 2 +- .../db/dataset_column_repositories.py | 2 +- .../db/dataset_location_repositories.py | 2 +- .../db/dataset_models.py | 2 +- .../db/dataset_profiling_repositories.py | 2 +- .../db/dataset_repositories.py | 4 +-- .../db/dataset_table_repositories.py | 2 +- .../handlers/__init__.py | 2 +- .../handlers/glue_dataset_handler.py | 6 ++-- .../handlers/glue_profiling_handler.py | 8 ++--- .../handlers/glue_table_sync_handler.py | 6 ++-- .../indexers/__init__.py | 0 .../indexers/dataset_catalog_indexer.py | 10 +++--- .../indexers/dataset_indexer.py | 4 +-- .../indexers/location_indexer.py | 6 ++-- .../indexers/table_indexer.py | 6 ++-- .../services/__init__.py | 0 .../services/dataset_alarm_service.py | 2 +- .../services/dataset_column_service.py | 16 ++++----- .../services/dataset_location_service.py | 16 ++++----- .../services/dataset_permissions.py | 0 .../services/dataset_profiling_service.py | 18 +++++----- .../services/dataset_service.py | 24 ++++++------- .../services/dataset_table_service.py | 20 +++++------ .../services/datasets_enums.py | 4 +-- .../tasks/__init__.py | 0 .../tasks/dataset_stack_finder.py | 4 +-- .../tasks/tables_syncer.py | 14 ++++---- .../5e5c84138af7_backfill_confidentiality.py | 2 +- .../72b8a90b6ee8__share_request_purpose.py | 2 +- .../8c79fb896983_add_table_for_buckets.py | 2 +- .../versions/97050ec09354_release_3_7_8.py | 2 +- ...9d_add_backfill_read_folder_permissions.py | 4 +-- ...215e_backfill_dataset_table_permissions.py | 4 +-- ...b_rename_imported_dataset_aws_resources.py | 2 +- deploy/stacks/backend_stack.py | 2 +- deploy/stacks/container.py | 14 ++++---- tests/conftest.py | 2 +- tests/modules/datasets/conftest.py | 20 +++++------ tests/modules/datasets/tasks/conftest.py | 2 +- .../tasks/test_dataset_catalog_indexer.py | 6 ++-- .../tasks/test_dataset_subscriptions.py | 2 +- .../tasks/test_dataset_tables_sync.py | 10 +++--- .../datasets/tasks/test_lf_share_manager.py | 2 +- .../test_s3_access_point_share_manager.py | 2 +- .../tasks/test_s3_bucket_share_manager.py | 2 +- .../test_stacks_updater_with_datasets.py | 2 +- tests/modules/datasets/test_dataset.py | 14 ++++---- tests/modules/datasets/test_dataset_feed.py | 2 +- .../modules/datasets/test_dataset_glossary.py | 2 +- .../modules/datasets/test_dataset_indexers.py | 6 ++-- .../modules/datasets/test_dataset_location.py | 8 ++--- .../datasets/test_dataset_permissions.py | 8 ++--- .../datasets/test_dataset_profiling.py | 2 +- .../datasets/test_dataset_resource_found.py | 4 +-- tests/modules/datasets/test_dataset_stack.py | 10 +++--- tests/modules/datasets/test_dataset_table.py | 4 +-- .../test_environment_stack_with_dataset.py | 2 +- tests/modules/datasets/test_glue_profiler.py | 2 +- .../test_import_dataset_check_unit.py | 4 +-- tests/modules/datasets/test_share.py | 2 +- 137 files changed, 295 insertions(+), 295 deletions(-) delete mode 100644 backend/dataall/modules/datasets/api/dataset/__init__.py rename backend/dataall/modules/{datasets => s3_datasets}/__init__.py (74%) rename backend/dataall/modules/{datasets => s3_datasets}/api/__init__.py (58%) create mode 100644 backend/dataall/modules/s3_datasets/api/dataset/__init__.py rename backend/dataall/modules/{datasets => s3_datasets}/api/dataset/input_types.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/api/dataset/mutations.py (93%) rename backend/dataall/modules/{datasets => s3_datasets}/api/dataset/queries.py (91%) rename backend/dataall/modules/{datasets => s3_datasets}/api/dataset/resolvers.py (92%) rename backend/dataall/modules/{datasets => s3_datasets}/api/dataset/types.py (97%) rename backend/dataall/modules/{datasets/api/table_column => s3_datasets/api/profiling}/__init__.py (72%) rename backend/dataall/modules/{datasets => s3_datasets}/api/profiling/input_types.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/api/profiling/mutations.py (76%) rename backend/dataall/modules/{datasets => s3_datasets}/api/profiling/queries.py (81%) rename backend/dataall/modules/{datasets => s3_datasets}/api/profiling/resolvers.py (84%) rename backend/dataall/modules/{datasets => s3_datasets}/api/profiling/types.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/__init__.py (69%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/input_types.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/mutations.py (82%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/queries.py (75%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/resolvers.py (77%) rename backend/dataall/modules/{datasets => s3_datasets}/api/storage_location/types.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/__init__.py (73%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/input_types.py (92%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/mutations.py (84%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/queries.py (80%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/resolvers.py (84%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table/types.py (95%) rename backend/dataall/modules/{datasets/api/profiling => s3_datasets/api/table_column}/__init__.py (71%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table_column/input_types.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table_column/mutations.py (84%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table_column/queries.py (81%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table_column/resolvers.py (87%) rename backend/dataall/modules/{datasets => s3_datasets}/api/table_column/types.py (95%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/athena_table_client.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/glue_dataset_client.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/glue_profiler_client.py (90%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/glue_table_client.py (95%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/kms_dataset_client.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/lf_dataset_client.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/lf_table_client.py (97%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/s3_dataset_bucket_policy_client.py (97%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/s3_dataset_client.py (97%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/s3_location_client.py (93%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/s3_profiler_client.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/aws/sns_dataset_client.py (94%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/__init__.py (87%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/assets/gluedatabasecustomresource/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/assets/gluedatabasecustomresource/index.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/assets/glueprofilingjob/glue_script.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/assets/lakeformationdefaultsettings/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/assets/lakeformationdefaultsettings/index.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/dataset_custom_resources_extension.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/dataset_glue_profiler_extension.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/dataset_stack.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/env_role_dataset_databrew_policy.py (95%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/env_role_dataset_glue_policy.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/env_role_dataset_s3_policy.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/cdk/pivot_role_datasets_policy.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/db/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_bucket_repositories.py (92%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_column_repositories.py (94%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_location_repositories.py (97%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_models.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_profiling_repositories.py (97%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_repositories.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/db/dataset_table_repositories.py (98%) rename backend/dataall/modules/{datasets => s3_datasets}/handlers/__init__.py (62%) rename backend/dataall/modules/{datasets => s3_datasets}/handlers/glue_dataset_handler.py (75%) rename backend/dataall/modules/{datasets => s3_datasets}/handlers/glue_profiling_handler.py (70%) rename backend/dataall/modules/{datasets => s3_datasets}/handlers/glue_table_sync_handler.py (88%) rename backend/dataall/modules/{datasets => s3_datasets}/indexers/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/indexers/dataset_catalog_indexer.py (69%) rename backend/dataall/modules/{datasets => s3_datasets}/indexers/dataset_indexer.py (93%) rename backend/dataall/modules/{datasets => s3_datasets}/indexers/location_indexer.py (90%) rename backend/dataall/modules/{datasets => s3_datasets}/indexers/table_indexer.py (91%) rename backend/dataall/modules/{datasets => s3_datasets}/services/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_alarm_service.py (94%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_column_service.py (81%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_location_service.py (89%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_permissions.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_profiling_service.py (85%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_service.py (96%) rename backend/dataall/modules/{datasets => s3_datasets}/services/dataset_table_service.py (90%) rename backend/dataall/modules/{datasets => s3_datasets}/services/datasets_enums.py (89%) rename backend/dataall/modules/{datasets => s3_datasets}/tasks/__init__.py (100%) rename backend/dataall/modules/{datasets => s3_datasets}/tasks/dataset_stack_finder.py (80%) rename backend/dataall/modules/{datasets => s3_datasets}/tasks/tables_syncer.py (85%) diff --git a/backend/dataall/modules/dataset_sharing/__init__.py b/backend/dataall/modules/dataset_sharing/__init__.py index aa78fa117..4364702cc 100644 --- a/backend/dataall/modules/dataset_sharing/__init__.py +++ b/backend/dataall/modules/dataset_sharing/__init__.py @@ -24,7 +24,7 @@ def depends_on() -> List[Type['ModuleInterface']]: def __init__(self): from dataall.modules.dataset_sharing import api from dataall.modules.dataset_sharing.services.managed_share_policy_service import SharePolicyService - from dataall.modules.datasets.services.dataset_service import DatasetService + from dataall.modules.s3_datasets.services.dataset_service import DatasetService from dataall.modules.dataset_sharing.services.dataset_sharing_service import DatasetSharingService EnvironmentResourceManager.register(ShareEnvironmentResource()) diff --git a/backend/dataall/modules/dataset_sharing/api/resolvers.py b/backend/dataall/modules/dataset_sharing/api/resolvers.py index 26fe3045d..3779516fa 100644 --- a/backend/dataall/modules/dataset_sharing/api/resolvers.py +++ b/backend/dataall/modules/dataset_sharing/api/resolvers.py @@ -12,8 +12,8 @@ from dataall.modules.dataset_sharing.services.share_object_service import ShareObjectService from dataall.modules.dataset_sharing.services.dataset_sharing_service import DatasetSharingService from dataall.modules.dataset_sharing.aws.glue_client import GlueClient -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset log = logging.getLogger(__name__) @@ -335,6 +335,6 @@ def list_shared_tables_by_env_dataset(context: Context, source, datasetUri: str, return DatasetSharingService.list_shared_tables_by_env_dataset(datasetUri, envUri) -@is_feature_enabled('modules.datasets.features.aws_actions') +@is_feature_enabled('modules.s3_datasets.features.aws_actions') def get_dataset_shared_assume_role_url(context: Context, source, datasetUri: str = None): return DatasetSharingService.get_dataset_shared_assume_role_url(uri=datasetUri) diff --git a/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py b/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py index 716dcb152..54f97087f 100644 --- a/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py +++ b/backend/dataall/modules/dataset_sharing/db/share_object_repositories.py @@ -20,8 +20,8 @@ PrincipalType, ) from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetBucket logger = logging.getLogger(__name__) diff --git a/backend/dataall/modules/dataset_sharing/services/data_sharing_service.py b/backend/dataall/modules/dataset_sharing/services/data_sharing_service.py index 528072bf1..1a08c8cf8 100644 --- a/backend/dataall/modules/dataset_sharing/services/data_sharing_service.py +++ b/backend/dataall/modules/dataset_sharing/services/data_sharing_service.py @@ -26,7 +26,7 @@ ShareItemStatus, PrincipalType, ) -from dataall.modules.datasets.db.dataset_models import DatasetLock +from dataall.modules.s3_datasets.db.dataset_models import DatasetLock log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_alarm_service.py b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_alarm_service.py index 4532131ca..6ad18beeb 100644 --- a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_alarm_service.py +++ b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_alarm_service.py @@ -3,7 +3,7 @@ from dataall.core.environment.db.environment_models import Environment from dataall.modules.dataset_sharing.db.share_object_models import ShareObject -from dataall.modules.datasets.db.dataset_models import DatasetTable, DatasetStorageLocation, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, DatasetStorageLocation, DatasetBucket from dataall.base.utils.alarm_service import AlarmService log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py index b950cf2e2..3bb60e052 100644 --- a/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py +++ b/backend/dataall/modules/dataset_sharing/services/dataset_sharing_service.py @@ -11,8 +11,8 @@ ) from dataall.modules.dataset_sharing.services.share_permissions import SHARE_OBJECT_APPROVER from dataall.modules.dataset_sharing.services.share_item_service import ShareItemService -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.dataset_permissions import ( MANAGE_DATASETS, UPDATE_DATASET, DELETE_DATASET, @@ -21,9 +21,9 @@ CREDENTIALS_DATASET, ) -from dataall.modules.datasets.db.dataset_models import Dataset -from dataall.modules.datasets.services.datasets_enums import DatasetRole -from dataall.modules.datasets.services.dataset_service import DatasetServiceInterface +from dataall.modules.s3_datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.services.datasets_enums import DatasetRole +from dataall.modules.s3_datasets.services.dataset_service import DatasetServiceInterface import logging diff --git a/backend/dataall/modules/dataset_sharing/services/share_item_service.py b/backend/dataall/modules/dataset_sharing/services/share_item_service.py index 67dc7c122..1049c5cc6 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_item_service.py +++ b/backend/dataall/modules/dataset_sharing/services/share_item_service.py @@ -30,8 +30,8 @@ LIST_ENVIRONMENT_SHARED_WITH_OBJECTS, APPROVE_SHARE_OBJECT, ) -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py b/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py index 51198d06e..94de76b78 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py +++ b/backend/dataall/modules/dataset_sharing/services/share_managers/lf_share_manager.py @@ -18,7 +18,7 @@ ShareItemActions, ShareItemHealthStatus, ) -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset from dataall.modules.dataset_sharing.services.dataset_sharing_alarm_service import DatasetSharingAlarmService from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject from dataall.modules.dataset_sharing.services.share_managers.share_manager_utils import ShareErrorFormatter diff --git a/backend/dataall/modules/dataset_sharing/services/share_managers/s3_access_point_share_manager.py b/backend/dataall/modules/dataset_sharing/services/share_managers/s3_access_point_share_manager.py index 2274a4aa3..46e3baf2e 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_managers/s3_access_point_share_manager.py +++ b/backend/dataall/modules/dataset_sharing/services/share_managers/s3_access_point_share_manager.py @@ -31,7 +31,7 @@ EMPTY_STATEMENT_SID, ) from dataall.modules.dataset_sharing.services.dataset_sharing_enums import PrincipalType -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset logger = logging.getLogger(__name__) ACCESS_POINT_CREATION_TIME = 30 diff --git a/backend/dataall/modules/dataset_sharing/services/share_managers/s3_bucket_share_manager.py b/backend/dataall/modules/dataset_sharing/services/share_managers/s3_bucket_share_manager.py index 7d41cf6b3..ef90f8bf6 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_managers/s3_bucket_share_manager.py +++ b/backend/dataall/modules/dataset_sharing/services/share_managers/s3_bucket_share_manager.py @@ -23,7 +23,7 @@ EMPTY_STATEMENT_SID, ) from dataall.modules.dataset_sharing.services.dataset_sharing_enums import PrincipalType -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetBucket from dataall.modules.dataset_sharing.db.share_object_repositories import ShareObjectRepository logger = logging.getLogger(__name__) diff --git a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py index 55c00a60b..38fe4f411 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py +++ b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py @@ -6,7 +6,7 @@ from dataall.core.tasks.db.task_models import Task from dataall.core.tasks.service_handlers import Worker from dataall.modules.dataset_sharing.db.share_object_models import ShareObject -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset from dataall.base.context import get_context from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareObjectStatus from dataall.modules.notifications.db.notification_repositories import NotificationRepository @@ -139,7 +139,7 @@ def _create_notification_task(self, subject, msg): - dataset.stewards - share.owner (person that opened the request) OR share.groupUri (if group_notifications=true) """ - share_notification_config = config.get_property('modules.datasets.features.share_notifications', default=None) + share_notification_config = config.get_property('modules.s3_datasets.features.share_notifications', default=None) if share_notification_config: for share_notification_config_type in share_notification_config.keys(): n_config = share_notification_config[share_notification_config_type] diff --git a/backend/dataall/modules/dataset_sharing/services/share_object_service.py b/backend/dataall/modules/dataset_sharing/services/share_object_service.py index 17d2602b9..e9188b970 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_object_service.py +++ b/backend/dataall/modules/dataset_sharing/services/share_object_service.py @@ -40,9 +40,9 @@ GET_SHARE_OBJECT, ) from dataall.modules.dataset_sharing.aws.glue_client import GlueClient -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset, DatasetStorageLocation -from dataall.modules.datasets.services.dataset_permissions import DATASET_TABLE_READ, DATASET_FOLDER_READ +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset, DatasetStorageLocation +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ, DATASET_FOLDER_READ from dataall.base.aws.iam import IAM import logging diff --git a/backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py b/backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py index 0bcacb66c..db59ddc59 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py +++ b/backend/dataall/modules/dataset_sharing/services/share_processors/lakeformation_process_share.py @@ -13,7 +13,7 @@ from dataall.modules.dataset_sharing.services.share_managers import LFShareManager from dataall.modules.dataset_sharing.aws.ram_client import RamClient from dataall.modules.dataset_sharing.services.share_object_service import ShareObjectService -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset from dataall.modules.dataset_sharing.db.share_object_models import ShareObject from dataall.modules.dataset_sharing.db.share_object_repositories import ShareObjectRepository, ShareItemSM from dataall.modules.dataset_sharing.services.share_managers.share_manager_utils import ShareErrorFormatter diff --git a/backend/dataall/modules/dataset_sharing/services/share_processors/s3_access_point_process_share.py b/backend/dataall/modules/dataset_sharing/services/share_processors/s3_access_point_process_share.py index 272684ec4..e6fe077a2 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_processors/s3_access_point_process_share.py +++ b/backend/dataall/modules/dataset_sharing/services/share_processors/s3_access_point_process_share.py @@ -5,7 +5,7 @@ from dataall.modules.dataset_sharing.services.share_exceptions import PrincipalRoleNotFound from dataall.modules.dataset_sharing.services.share_managers import S3AccessPointShareManager from dataall.modules.dataset_sharing.services.share_object_service import ShareObjectService -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ( ShareItemHealthStatus, ShareItemStatus, diff --git a/backend/dataall/modules/dataset_sharing/services/share_processors/s3_bucket_process_share.py b/backend/dataall/modules/dataset_sharing/services/share_processors/s3_bucket_process_share.py index 036f2919e..6aaa661e7 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_processors/s3_bucket_process_share.py +++ b/backend/dataall/modules/dataset_sharing/services/share_processors/s3_bucket_process_share.py @@ -5,7 +5,7 @@ from dataall.modules.dataset_sharing.services.share_exceptions import PrincipalRoleNotFound from dataall.modules.dataset_sharing.services.share_managers import S3BucketShareManager from dataall.modules.dataset_sharing.services.share_object_service import ShareObjectService -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetBucket from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ( ShareItemHealthStatus, ShareItemStatus, diff --git a/backend/dataall/modules/dataset_sharing/tasks/dataset_subscription_task.py b/backend/dataall/modules/dataset_sharing/tasks/dataset_subscription_task.py index 24ee2ad9e..26ce29b75 100644 --- a/backend/dataall/modules/dataset_sharing/tasks/dataset_subscription_task.py +++ b/backend/dataall/modules/dataset_sharing/tasks/dataset_subscription_task.py @@ -12,12 +12,12 @@ from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem from dataall.modules.dataset_sharing.db.share_object_repositories import ShareObjectRepository from dataall.modules.dataset_sharing.services.share_notification_service import ShareNotificationService -from dataall.modules.datasets.aws.sns_dataset_client import SnsDatasetClient -from dataall.modules.datasets.db.dataset_location_repositories import DatasetLocationRepository -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.aws.sns_dataset_client import SnsDatasetClient +from dataall.modules.s3_datasets.db.dataset_location_repositories import DatasetLocationRepository +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository from dataall.modules.dataset_sharing.tasks.subscriptions import poll_queues -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset root = logging.getLogger() root.setLevel(logging.INFO) diff --git a/backend/dataall/modules/datasets/api/dataset/__init__.py b/backend/dataall/modules/datasets/api/dataset/__init__.py deleted file mode 100644 index 5e6d35f34..000000000 --- a/backend/dataall/modules/datasets/api/dataset/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from dataall.modules.datasets.api.dataset import input_types, mutations, queries, resolvers, types - -__all__ = ['resolvers', 'types', 'input_types', 'queries', 'mutations'] diff --git a/backend/dataall/modules/datasets/__init__.py b/backend/dataall/modules/s3_datasets/__init__.py similarity index 74% rename from backend/dataall/modules/datasets/__init__.py rename to backend/dataall/modules/s3_datasets/__init__.py index 880203597..6e272f26b 100644 --- a/backend/dataall/modules/datasets/__init__.py +++ b/backend/dataall/modules/s3_datasets/__init__.py @@ -34,14 +34,14 @@ def __init__(self): from dataall.modules.feed.api.registry import FeedRegistry, FeedDefinition from dataall.modules.catalog.indexers.registry import GlossaryRegistry, GlossaryDefinition from dataall.core.environment.services.environment_resource_manager import EnvironmentResourceManager - from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer - from dataall.modules.datasets.indexers.location_indexer import DatasetLocationIndexer - from dataall.modules.datasets.indexers.table_indexer import DatasetTableIndexer + from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer + from dataall.modules.s3_datasets.indexers.location_indexer import DatasetLocationIndexer + from dataall.modules.s3_datasets.indexers.table_indexer import DatasetTableIndexer - import dataall.modules.datasets.api - from dataall.modules.datasets.services.dataset_permissions import GET_DATASET, UPDATE_DATASET - from dataall.modules.datasets.db.dataset_repositories import DatasetRepository - from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset + import dataall.modules.s3_datasets.api + from dataall.modules.s3_datasets.services.dataset_permissions import GET_DATASET, UPDATE_DATASET + from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository + from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset FeedRegistry.register(FeedDefinition('DatasetStorageLocation', DatasetStorageLocation)) FeedRegistry.register(FeedDefinition('DatasetTable', DatasetTable)) @@ -86,11 +86,11 @@ def is_supported(modes: Set[ImportMode]): return ImportMode.HANDLERS in modes def __init__(self): - import dataall.modules.datasets.handlers - import dataall.modules.datasets.db.dataset_models - import dataall.modules.datasets.db.dataset_repositories - import dataall.modules.datasets.services.dataset_permissions - import dataall.modules.datasets.services.datasets_enums + import dataall.modules.s3_datasets.handlers + import dataall.modules.s3_datasets.db.dataset_models + import dataall.modules.s3_datasets.db.dataset_repositories + import dataall.modules.s3_datasets.services.dataset_permissions + import dataall.modules.s3_datasets.services.datasets_enums log.info('Dataset handlers have been imported') @@ -103,10 +103,10 @@ def is_supported(modes: Set[ImportMode]): return ImportMode.CDK in modes def __init__(self): - import dataall.modules.datasets.cdk + import dataall.modules.s3_datasets.cdk from dataall.core.environment.cdk.environment_stack import EnvironmentSetup - from dataall.modules.datasets.cdk.dataset_glue_profiler_extension import DatasetGlueProfilerExtension - from dataall.modules.datasets.cdk.dataset_custom_resources_extension import DatasetCustomResourcesExtension + from dataall.modules.s3_datasets.cdk.dataset_glue_profiler_extension import DatasetGlueProfilerExtension + from dataall.modules.s3_datasets.cdk.dataset_custom_resources_extension import DatasetCustomResourcesExtension EnvironmentSetup.register(DatasetGlueProfilerExtension) EnvironmentSetup.register(DatasetCustomResourcesExtension) @@ -120,7 +120,7 @@ def is_supported(modes: Set[ImportMode]) -> bool: return ImportMode.STACK_UPDATER_TASK in modes def __init__(self): - from dataall.modules.datasets.tasks.dataset_stack_finder import DatasetStackFinder + from dataall.modules.s3_datasets.tasks.dataset_stack_finder import DatasetStackFinder DatasetStackFinder() log.info('Dataset stack updater task has been loaded') @@ -138,7 +138,7 @@ def depends_on() -> List[Type['ModuleInterface']]: return [CatalogIndexerModuleInterface] def __init__(self): - from dataall.modules.datasets.indexers.dataset_catalog_indexer import DatasetCatalogIndexer + from dataall.modules.s3_datasets.indexers.dataset_catalog_indexer import DatasetCatalogIndexer DatasetCatalogIndexer() log.info('Dataset catalog indexer task has been loaded') diff --git a/backend/dataall/modules/datasets/api/__init__.py b/backend/dataall/modules/s3_datasets/api/__init__.py similarity index 58% rename from backend/dataall/modules/datasets/api/__init__.py rename to backend/dataall/modules/s3_datasets/api/__init__.py index 3b4a43d07..bced7b8a2 100644 --- a/backend/dataall/modules/datasets/api/__init__.py +++ b/backend/dataall/modules/s3_datasets/api/__init__.py @@ -1,5 +1,5 @@ """The GraphQL schema of datasets and related functionality""" -from dataall.modules.datasets.api import table_column, profiling, storage_location, table, dataset +from dataall.modules.s3_datasets.api import table_column, profiling, storage_location, table, dataset __all__ = ['table_column', 'profiling', 'storage_location', 'table', 'dataset'] diff --git a/backend/dataall/modules/s3_datasets/api/dataset/__init__.py b/backend/dataall/modules/s3_datasets/api/dataset/__init__.py new file mode 100644 index 000000000..21cf79c47 --- /dev/null +++ b/backend/dataall/modules/s3_datasets/api/dataset/__init__.py @@ -0,0 +1,3 @@ +from dataall.modules.s3_datasets.api.dataset import input_types, mutations, queries, resolvers, types + +__all__ = ['resolvers', 'types', 'input_types', 'queries', 'mutations'] diff --git a/backend/dataall/modules/datasets/api/dataset/input_types.py b/backend/dataall/modules/s3_datasets/api/dataset/input_types.py similarity index 98% rename from backend/dataall/modules/datasets/api/dataset/input_types.py rename to backend/dataall/modules/s3_datasets/api/dataset/input_types.py index e82f6640f..a7571c426 100644 --- a/backend/dataall/modules/datasets/api/dataset/input_types.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/input_types.py @@ -1,6 +1,6 @@ from dataall.base.api import gql from dataall.base.api.constants import SortDirection -from dataall.modules.datasets.services.datasets_enums import DatasetSortField +from dataall.modules.s3_datasets.services.datasets_enums import DatasetSortField NewDatasetInput = gql.InputType( diff --git a/backend/dataall/modules/datasets/api/dataset/mutations.py b/backend/dataall/modules/s3_datasets/api/dataset/mutations.py similarity index 93% rename from backend/dataall/modules/datasets/api/dataset/mutations.py rename to backend/dataall/modules/s3_datasets/api/dataset/mutations.py index ed8f85b0a..d82f98194 100644 --- a/backend/dataall/modules/datasets/api/dataset/mutations.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/mutations.py @@ -1,10 +1,10 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.dataset.input_types import ( +from dataall.modules.s3_datasets.api.dataset.input_types import ( ModifyDatasetInput, NewDatasetInput, ImportDatasetInput, ) -from dataall.modules.datasets.api.dataset.resolvers import ( +from dataall.modules.s3_datasets.api.dataset.resolvers import ( create_dataset, update_dataset, generate_dataset_access_token, diff --git a/backend/dataall/modules/datasets/api/dataset/queries.py b/backend/dataall/modules/s3_datasets/api/dataset/queries.py similarity index 91% rename from backend/dataall/modules/datasets/api/dataset/queries.py rename to backend/dataall/modules/s3_datasets/api/dataset/queries.py index e892a734e..6e688bcd1 100644 --- a/backend/dataall/modules/datasets/api/dataset/queries.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/queries.py @@ -1,6 +1,6 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.dataset.input_types import DatasetFilter -from dataall.modules.datasets.api.dataset.resolvers import ( +from dataall.modules.s3_datasets.api.dataset.input_types import DatasetFilter +from dataall.modules.s3_datasets.api.dataset.resolvers import ( get_dataset, list_all_user_datasets, list_owned_datasets, @@ -9,7 +9,7 @@ list_datasets_owned_by_env_group, list_datasets_created_in_environment, ) -from dataall.modules.datasets.api.dataset.types import DatasetSearchResult +from dataall.modules.s3_datasets.api.dataset.types import DatasetSearchResult getDataset = gql.QueryField( name='getDataset', diff --git a/backend/dataall/modules/datasets/api/dataset/resolvers.py b/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py similarity index 92% rename from backend/dataall/modules/datasets/api/dataset/resolvers.py rename to backend/dataall/modules/s3_datasets/api/dataset/resolvers.py index 54ca62b6a..5b08f85a3 100644 --- a/backend/dataall/modules/datasets/api/dataset/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/resolvers.py @@ -7,9 +7,9 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.base.db.exceptions import RequiredParameter, InvalidInput -from dataall.modules.datasets.db.dataset_models import Dataset -from dataall.modules.datasets.services.datasets_enums import DatasetRole, ConfidentialityClassification -from dataall.modules.datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.services.datasets_enums import DatasetRole, ConfidentialityClassification +from dataall.modules.s3_datasets.services.dataset_service import DatasetService log = logging.getLogger(__name__) @@ -53,7 +53,7 @@ def resolve_user_role(context: Context, source: Dataset, **kwargs): return DatasetRole.NoPermission.value -@is_feature_enabled('modules.datasets.features.file_uploads') +@is_feature_enabled('modules.s3_datasets.features.file_uploads') def get_file_upload_presigned_url(context, source, datasetUri: str = None, input: dict = None): return DatasetService.get_file_upload_presigned_url(uri=datasetUri, data=input) @@ -122,17 +122,17 @@ def get_dataset_statistics(context: Context, source: Dataset, **kwargs): return DatasetService.get_dataset_statistics(source) -@is_feature_enabled('modules.datasets.features.aws_actions') +@is_feature_enabled('modules.s3_datasets.features.aws_actions') def get_dataset_assume_role_url(context: Context, source, datasetUri: str = None): return DatasetService.get_dataset_assume_role_url(uri=datasetUri) -@is_feature_enabled('modules.datasets.features.glue_crawler') +@is_feature_enabled('modules.s3_datasets.features.glue_crawler') def start_crawler(context: Context, source, datasetUri: str, input: dict = None): return DatasetService.start_crawler(uri=datasetUri, data=input) -@is_feature_enabled('modules.datasets.features.aws_actions') +@is_feature_enabled('modules.s3_datasets.features.aws_actions') def generate_dataset_access_token(context, source, datasetUri: str = None): return DatasetService.generate_dataset_access_token(uri=datasetUri) diff --git a/backend/dataall/modules/datasets/api/dataset/types.py b/backend/dataall/modules/s3_datasets/api/dataset/types.py similarity index 97% rename from backend/dataall/modules/datasets/api/dataset/types.py rename to backend/dataall/modules/s3_datasets/api/dataset/types.py index f432f4d30..863f896f7 100644 --- a/backend/dataall/modules/datasets/api/dataset/types.py +++ b/backend/dataall/modules/s3_datasets/api/dataset/types.py @@ -1,6 +1,6 @@ from dataall.base.api import gql -from dataall.modules.datasets.services.datasets_enums import DatasetRole -from dataall.modules.datasets.api.dataset.resolvers import ( +from dataall.modules.s3_datasets.services.datasets_enums import DatasetRole +from dataall.modules.s3_datasets.api.dataset.resolvers import ( get_dataset_environment, get_dataset_organization, get_dataset_owners_group, diff --git a/backend/dataall/modules/datasets/api/table_column/__init__.py b/backend/dataall/modules/s3_datasets/api/profiling/__init__.py similarity index 72% rename from backend/dataall/modules/datasets/api/table_column/__init__.py rename to backend/dataall/modules/s3_datasets/api/profiling/__init__.py index 214930e65..ec9acc44d 100644 --- a/backend/dataall/modules/datasets/api/table_column/__init__.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/__init__.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.api.table_column import ( +from dataall.modules.s3_datasets.api.profiling import ( input_types, mutations, queries, diff --git a/backend/dataall/modules/datasets/api/profiling/input_types.py b/backend/dataall/modules/s3_datasets/api/profiling/input_types.py similarity index 100% rename from backend/dataall/modules/datasets/api/profiling/input_types.py rename to backend/dataall/modules/s3_datasets/api/profiling/input_types.py diff --git a/backend/dataall/modules/datasets/api/profiling/mutations.py b/backend/dataall/modules/s3_datasets/api/profiling/mutations.py similarity index 76% rename from backend/dataall/modules/datasets/api/profiling/mutations.py rename to backend/dataall/modules/s3_datasets/api/profiling/mutations.py index 861f238cb..da6f7c480 100644 --- a/backend/dataall/modules/datasets/api/profiling/mutations.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/mutations.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.profiling.resolvers import start_profiling_run +from dataall.modules.s3_datasets.api.profiling.resolvers import start_profiling_run startDatasetProfilingRun = gql.MutationField( name='startDatasetProfilingRun', diff --git a/backend/dataall/modules/datasets/api/profiling/queries.py b/backend/dataall/modules/s3_datasets/api/profiling/queries.py similarity index 81% rename from backend/dataall/modules/datasets/api/profiling/queries.py rename to backend/dataall/modules/s3_datasets/api/profiling/queries.py index 0333de688..228dae28f 100644 --- a/backend/dataall/modules/datasets/api/profiling/queries.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/queries.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.profiling.resolvers import list_table_profiling_runs, get_dataset_table_profiling_run +from dataall.modules.s3_datasets.api.profiling.resolvers import list_table_profiling_runs, get_dataset_table_profiling_run listDatasetTableProfilingRuns = gql.QueryField( name='listDatasetTableProfilingRuns', diff --git a/backend/dataall/modules/datasets/api/profiling/resolvers.py b/backend/dataall/modules/s3_datasets/api/profiling/resolvers.py similarity index 84% rename from backend/dataall/modules/datasets/api/profiling/resolvers.py rename to backend/dataall/modules/s3_datasets/api/profiling/resolvers.py index bcfd161b1..b92b8d065 100644 --- a/backend/dataall/modules/datasets/api/profiling/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/resolvers.py @@ -3,9 +3,9 @@ from dataall.base.api.context import Context from dataall.base.db.exceptions import RequiredParameter -from dataall.modules.datasets.services.dataset_profiling_service import DatasetProfilingService -from dataall.modules.datasets.services.dataset_service import DatasetService -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun +from dataall.modules.s3_datasets.services.dataset_profiling_service import DatasetProfilingService +from dataall.modules.s3_datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/api/profiling/types.py b/backend/dataall/modules/s3_datasets/api/profiling/types.py similarity index 96% rename from backend/dataall/modules/datasets/api/profiling/types.py rename to backend/dataall/modules/s3_datasets/api/profiling/types.py index bc942eb7b..31eb2eaaa 100644 --- a/backend/dataall/modules/datasets/api/profiling/types.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/types.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.profiling.resolvers import ( +from dataall.modules.s3_datasets.api.profiling.resolvers import ( resolve_dataset, resolve_profiling_run_status, resolve_profiling_results, diff --git a/backend/dataall/modules/datasets/api/storage_location/__init__.py b/backend/dataall/modules/s3_datasets/api/storage_location/__init__.py similarity index 69% rename from backend/dataall/modules/datasets/api/storage_location/__init__.py rename to backend/dataall/modules/s3_datasets/api/storage_location/__init__.py index a2d3ea55f..9c8249432 100644 --- a/backend/dataall/modules/datasets/api/storage_location/__init__.py +++ b/backend/dataall/modules/s3_datasets/api/storage_location/__init__.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.api.storage_location import ( +from dataall.modules.s3_datasets.api.storage_location import ( input_types, mutations, queries, diff --git a/backend/dataall/modules/datasets/api/storage_location/input_types.py b/backend/dataall/modules/s3_datasets/api/storage_location/input_types.py similarity index 100% rename from backend/dataall/modules/datasets/api/storage_location/input_types.py rename to backend/dataall/modules/s3_datasets/api/storage_location/input_types.py diff --git a/backend/dataall/modules/datasets/api/storage_location/mutations.py b/backend/dataall/modules/s3_datasets/api/storage_location/mutations.py similarity index 82% rename from backend/dataall/modules/datasets/api/storage_location/mutations.py rename to backend/dataall/modules/s3_datasets/api/storage_location/mutations.py index 1185efabb..7680be07f 100644 --- a/backend/dataall/modules/datasets/api/storage_location/mutations.py +++ b/backend/dataall/modules/s3_datasets/api/storage_location/mutations.py @@ -1,14 +1,14 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.storage_location.input_types import ( +from dataall.modules.s3_datasets.api.storage_location.input_types import ( ModifyDatasetFolderInput, NewDatasetStorageLocationInput, ) -from dataall.modules.datasets.api.storage_location.resolvers import ( +from dataall.modules.s3_datasets.api.storage_location.resolvers import ( create_storage_location, update_storage_location, remove_storage_location, ) -from dataall.modules.datasets.api.storage_location.types import DatasetStorageLocation +from dataall.modules.s3_datasets.api.storage_location.types import DatasetStorageLocation createDatasetStorageLocation = gql.MutationField( name='createDatasetStorageLocation', diff --git a/backend/dataall/modules/datasets/api/storage_location/queries.py b/backend/dataall/modules/s3_datasets/api/storage_location/queries.py similarity index 75% rename from backend/dataall/modules/datasets/api/storage_location/queries.py rename to backend/dataall/modules/s3_datasets/api/storage_location/queries.py index e56630101..3ce9353a3 100644 --- a/backend/dataall/modules/datasets/api/storage_location/queries.py +++ b/backend/dataall/modules/s3_datasets/api/storage_location/queries.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.storage_location.resolvers import get_storage_location +from dataall.modules.s3_datasets.api.storage_location.resolvers import get_storage_location getDatasetStorageLocation = gql.QueryField( name='getDatasetStorageLocation', diff --git a/backend/dataall/modules/datasets/api/storage_location/resolvers.py b/backend/dataall/modules/s3_datasets/api/storage_location/resolvers.py similarity index 77% rename from backend/dataall/modules/datasets/api/storage_location/resolvers.py rename to backend/dataall/modules/s3_datasets/api/storage_location/resolvers.py index 41187827e..51f38da1e 100644 --- a/backend/dataall/modules/datasets/api/storage_location/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/storage_location/resolvers.py @@ -2,11 +2,11 @@ from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.base.db.exceptions import RequiredParameter from dataall.base.feature_toggle_checker import is_feature_enabled -from dataall.modules.datasets.services.dataset_location_service import DatasetLocationService -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.services.dataset_location_service import DatasetLocationService +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset -@is_feature_enabled('modules.datasets.features.file_actions') +@is_feature_enabled('modules.s3_datasets.features.file_actions') def create_storage_location(context, source, datasetUri: str = None, input: dict = None): if 'prefix' not in input: raise RequiredParameter('prefix') @@ -16,7 +16,7 @@ def create_storage_location(context, source, datasetUri: str = None, input: dict return DatasetLocationService.create_storage_location(uri=datasetUri, data=input) -@is_feature_enabled('modules.datasets.features.file_actions') +@is_feature_enabled('modules.s3_datasets.features.file_actions') def list_dataset_locations(context, source, filter: dict = None): if not source: return None @@ -25,17 +25,17 @@ def list_dataset_locations(context, source, filter: dict = None): return DatasetLocationService.list_dataset_locations(uri=source.datasetUri, filter=filter) -@is_feature_enabled('modules.datasets.features.file_actions') +@is_feature_enabled('modules.s3_datasets.features.file_actions') def get_storage_location(context, source, locationUri=None): return DatasetLocationService.get_storage_location(uri=locationUri) -@is_feature_enabled('modules.datasets.features.file_actions') +@is_feature_enabled('modules.s3_datasets.features.file_actions') def update_storage_location(context, source, locationUri: str = None, input: dict = None): return DatasetLocationService.update_storage_location(uri=locationUri, data=input) -@is_feature_enabled('modules.datasets.features.file_actions') +@is_feature_enabled('modules.s3_datasets.features.file_actions') def remove_storage_location(context, source, locationUri: str = None): return DatasetLocationService.remove_storage_location(uri=locationUri) diff --git a/backend/dataall/modules/datasets/api/storage_location/types.py b/backend/dataall/modules/s3_datasets/api/storage_location/types.py similarity index 96% rename from backend/dataall/modules/datasets/api/storage_location/types.py rename to backend/dataall/modules/s3_datasets/api/storage_location/types.py index 0a31c74d3..40070a287 100644 --- a/backend/dataall/modules/datasets/api/storage_location/types.py +++ b/backend/dataall/modules/s3_datasets/api/storage_location/types.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.storage_location.resolvers import resolve_glossary_terms, resolve_dataset +from dataall.modules.s3_datasets.api.storage_location.resolvers import resolve_glossary_terms, resolve_dataset DatasetStorageLocation = gql.ObjectType( name='DatasetStorageLocation', diff --git a/backend/dataall/modules/datasets/api/table/__init__.py b/backend/dataall/modules/s3_datasets/api/table/__init__.py similarity index 73% rename from backend/dataall/modules/datasets/api/table/__init__.py rename to backend/dataall/modules/s3_datasets/api/table/__init__.py index 951b4c211..6431898f2 100644 --- a/backend/dataall/modules/datasets/api/table/__init__.py +++ b/backend/dataall/modules/s3_datasets/api/table/__init__.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.api.table import ( +from dataall.modules.s3_datasets.api.table import ( input_types, mutations, queries, diff --git a/backend/dataall/modules/datasets/api/table/input_types.py b/backend/dataall/modules/s3_datasets/api/table/input_types.py similarity index 92% rename from backend/dataall/modules/datasets/api/table/input_types.py rename to backend/dataall/modules/s3_datasets/api/table/input_types.py index 5c538a2b9..fe9026518 100644 --- a/backend/dataall/modules/datasets/api/table/input_types.py +++ b/backend/dataall/modules/s3_datasets/api/table/input_types.py @@ -1,6 +1,6 @@ from dataall.base.api import gql from dataall.base.api.constants import SortDirection -from dataall.modules.datasets.services.datasets_enums import DatasetSortField +from dataall.modules.s3_datasets.services.datasets_enums import DatasetSortField ModifyDatasetTableInput = gql.InputType( diff --git a/backend/dataall/modules/datasets/api/table/mutations.py b/backend/dataall/modules/s3_datasets/api/table/mutations.py similarity index 84% rename from backend/dataall/modules/datasets/api/table/mutations.py rename to backend/dataall/modules/s3_datasets/api/table/mutations.py index 0caf689e3..08e601409 100644 --- a/backend/dataall/modules/datasets/api/table/mutations.py +++ b/backend/dataall/modules/s3_datasets/api/table/mutations.py @@ -1,6 +1,6 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.table.input_types import ModifyDatasetTableInput -from dataall.modules.datasets.api.table.resolvers import ( +from dataall.modules.s3_datasets.api.table.input_types import ModifyDatasetTableInput +from dataall.modules.s3_datasets.api.table.resolvers import ( update_table, delete_table, sync_tables, diff --git a/backend/dataall/modules/datasets/api/table/queries.py b/backend/dataall/modules/s3_datasets/api/table/queries.py similarity index 80% rename from backend/dataall/modules/datasets/api/table/queries.py rename to backend/dataall/modules/s3_datasets/api/table/queries.py index 04e3e64ad..af39a1cf7 100644 --- a/backend/dataall/modules/datasets/api/table/queries.py +++ b/backend/dataall/modules/s3_datasets/api/table/queries.py @@ -1,7 +1,7 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.table.input_types import DatasetTableFilter -from dataall.modules.datasets.api.table.resolvers import get_table, preview -from dataall.modules.datasets.api.table.types import ( +from dataall.modules.s3_datasets.api.table.input_types import DatasetTableFilter +from dataall.modules.s3_datasets.api.table.resolvers import get_table, preview +from dataall.modules.s3_datasets.api.table.types import ( DatasetTable, DatasetTableSearchResult, ) diff --git a/backend/dataall/modules/datasets/api/table/resolvers.py b/backend/dataall/modules/s3_datasets/api/table/resolvers.py similarity index 84% rename from backend/dataall/modules/datasets/api/table/resolvers.py rename to backend/dataall/modules/s3_datasets/api/table/resolvers.py index f6c03dd3b..5338e9853 100644 --- a/backend/dataall/modules/datasets/api/table/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/table/resolvers.py @@ -2,10 +2,10 @@ from dataall.base.feature_toggle_checker import is_feature_enabled from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository -from dataall.modules.datasets.api.dataset.resolvers import get_dataset +from dataall.modules.s3_datasets.api.dataset.resolvers import get_dataset from dataall.base.api.context import Context -from dataall.modules.datasets.services.dataset_table_service import DatasetTableService -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.services.dataset_table_service import DatasetTableService +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset log = logging.getLogger(__name__) @@ -24,7 +24,7 @@ def delete_table(context, source, tableUri: str = None): return DatasetTableService.delete_table(uri=tableUri) -@is_feature_enabled('modules.datasets.features.preview_data') +@is_feature_enabled('modules.s3_datasets.features.preview_data') def preview(context, source, tableUri: str = None): if not tableUri: return None diff --git a/backend/dataall/modules/datasets/api/table/types.py b/backend/dataall/modules/s3_datasets/api/table/types.py similarity index 95% rename from backend/dataall/modules/datasets/api/table/types.py rename to backend/dataall/modules/s3_datasets/api/table/types.py index 70fdd2347..20911734f 100644 --- a/backend/dataall/modules/datasets/api/table/types.py +++ b/backend/dataall/modules/s3_datasets/api/table/types.py @@ -1,6 +1,6 @@ -from dataall.modules.datasets.api.table_column.resolvers import list_table_columns +from dataall.modules.s3_datasets.api.table_column.resolvers import list_table_columns from dataall.base.api import gql -from dataall.modules.datasets.api.table.resolvers import ( +from dataall.modules.s3_datasets.api.table.resolvers import ( resolve_dataset, get_glue_table_properties, resolve_glossary_terms, diff --git a/backend/dataall/modules/datasets/api/profiling/__init__.py b/backend/dataall/modules/s3_datasets/api/table_column/__init__.py similarity index 71% rename from backend/dataall/modules/datasets/api/profiling/__init__.py rename to backend/dataall/modules/s3_datasets/api/table_column/__init__.py index 3706f9a1e..447896e8a 100644 --- a/backend/dataall/modules/datasets/api/profiling/__init__.py +++ b/backend/dataall/modules/s3_datasets/api/table_column/__init__.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.api.profiling import ( +from dataall.modules.s3_datasets.api.table_column import ( input_types, mutations, queries, diff --git a/backend/dataall/modules/datasets/api/table_column/input_types.py b/backend/dataall/modules/s3_datasets/api/table_column/input_types.py similarity index 100% rename from backend/dataall/modules/datasets/api/table_column/input_types.py rename to backend/dataall/modules/s3_datasets/api/table_column/input_types.py diff --git a/backend/dataall/modules/datasets/api/table_column/mutations.py b/backend/dataall/modules/s3_datasets/api/table_column/mutations.py similarity index 84% rename from backend/dataall/modules/datasets/api/table_column/mutations.py rename to backend/dataall/modules/s3_datasets/api/table_column/mutations.py index b921a79f0..d9ae99b6d 100644 --- a/backend/dataall/modules/datasets/api/table_column/mutations.py +++ b/backend/dataall/modules/s3_datasets/api/table_column/mutations.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.table_column.resolvers import sync_table_columns, update_table_column +from dataall.modules.s3_datasets.api.table_column.resolvers import sync_table_columns, update_table_column syncDatasetTableColumns = gql.MutationField( name='syncDatasetTableColumns', diff --git a/backend/dataall/modules/datasets/api/table_column/queries.py b/backend/dataall/modules/s3_datasets/api/table_column/queries.py similarity index 81% rename from backend/dataall/modules/datasets/api/table_column/queries.py rename to backend/dataall/modules/s3_datasets/api/table_column/queries.py index 5e2358cb7..dbdd26e15 100644 --- a/backend/dataall/modules/datasets/api/table_column/queries.py +++ b/backend/dataall/modules/s3_datasets/api/table_column/queries.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.table_column.resolvers import list_table_columns +from dataall.modules.s3_datasets.api.table_column.resolvers import list_table_columns listDatasetTableColumns = gql.QueryField( name='listDatasetTableColumns', diff --git a/backend/dataall/modules/datasets/api/table_column/resolvers.py b/backend/dataall/modules/s3_datasets/api/table_column/resolvers.py similarity index 87% rename from backend/dataall/modules/datasets/api/table_column/resolvers.py rename to backend/dataall/modules/s3_datasets/api/table_column/resolvers.py index 5cb1a8342..ab8a6f320 100644 --- a/backend/dataall/modules/datasets/api/table_column/resolvers.py +++ b/backend/dataall/modules/s3_datasets/api/table_column/resolvers.py @@ -1,8 +1,8 @@ from dataall.base.api.context import Context from dataall.modules.catalog.db.glossary_models import TermLink from dataall.base.db import paginate -from dataall.modules.datasets.services.dataset_column_service import DatasetColumnService -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn, DatasetTable +from dataall.modules.s3_datasets.services.dataset_column_service import DatasetColumnService +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn, DatasetTable def list_table_columns( diff --git a/backend/dataall/modules/datasets/api/table_column/types.py b/backend/dataall/modules/s3_datasets/api/table_column/types.py similarity index 95% rename from backend/dataall/modules/datasets/api/table_column/types.py rename to backend/dataall/modules/s3_datasets/api/table_column/types.py index db0b0de68..245b95446 100644 --- a/backend/dataall/modules/datasets/api/table_column/types.py +++ b/backend/dataall/modules/s3_datasets/api/table_column/types.py @@ -1,5 +1,5 @@ from dataall.base.api import gql -from dataall.modules.datasets.api.table_column.resolvers import resolve_terms +from dataall.modules.s3_datasets.api.table_column.resolvers import resolve_terms DatasetTableColumn = gql.ObjectType( diff --git a/backend/dataall/modules/datasets/aws/__init__.py b/backend/dataall/modules/s3_datasets/aws/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/aws/__init__.py rename to backend/dataall/modules/s3_datasets/aws/__init__.py diff --git a/backend/dataall/modules/datasets/aws/athena_table_client.py b/backend/dataall/modules/s3_datasets/aws/athena_table_client.py similarity index 96% rename from backend/dataall/modules/datasets/aws/athena_table_client.py rename to backend/dataall/modules/s3_datasets/aws/athena_table_client.py index 40764ded9..2dd843295 100644 --- a/backend/dataall/modules/datasets/aws/athena_table_client.py +++ b/backend/dataall/modules/s3_datasets/aws/athena_table_client.py @@ -6,7 +6,7 @@ from dataall.base.aws.sts import SessionHelper from dataall.core.environment.db.environment_models import Environment -from dataall.modules.datasets.db.dataset_models import DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable from dataall.base.utils import json_utils, sql_utils log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/glue_dataset_client.py b/backend/dataall/modules/s3_datasets/aws/glue_dataset_client.py similarity index 98% rename from backend/dataall/modules/datasets/aws/glue_dataset_client.py rename to backend/dataall/modules/s3_datasets/aws/glue_dataset_client.py index 77dd5a3d3..028e7e877 100644 --- a/backend/dataall/modules/datasets/aws/glue_dataset_client.py +++ b/backend/dataall/modules/s3_datasets/aws/glue_dataset_client.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/glue_profiler_client.py b/backend/dataall/modules/s3_datasets/aws/glue_profiler_client.py similarity index 90% rename from backend/dataall/modules/datasets/aws/glue_profiler_client.py rename to backend/dataall/modules/s3_datasets/aws/glue_profiler_client.py index 3aab6daf3..fd22b1c5a 100644 --- a/backend/dataall/modules/datasets/aws/glue_profiler_client.py +++ b/backend/dataall/modules/s3_datasets/aws/glue_profiler_client.py @@ -2,8 +2,8 @@ from botocore.exceptions import ClientError from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import Dataset -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun +from dataall.modules.s3_datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/glue_table_client.py b/backend/dataall/modules/s3_datasets/aws/glue_table_client.py similarity index 95% rename from backend/dataall/modules/datasets/aws/glue_table_client.py rename to backend/dataall/modules/s3_datasets/aws/glue_table_client.py index 627d07d93..f02ebb1ef 100644 --- a/backend/dataall/modules/datasets/aws/glue_table_client.py +++ b/backend/dataall/modules/s3_datasets/aws/glue_table_client.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError -from dataall.modules.datasets.db.dataset_models import DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/kms_dataset_client.py b/backend/dataall/modules/s3_datasets/aws/kms_dataset_client.py similarity index 100% rename from backend/dataall/modules/datasets/aws/kms_dataset_client.py rename to backend/dataall/modules/s3_datasets/aws/kms_dataset_client.py diff --git a/backend/dataall/modules/datasets/aws/lf_dataset_client.py b/backend/dataall/modules/s3_datasets/aws/lf_dataset_client.py similarity index 96% rename from backend/dataall/modules/datasets/aws/lf_dataset_client.py rename to backend/dataall/modules/s3_datasets/aws/lf_dataset_client.py index aaffc7787..9454e46d5 100644 --- a/backend/dataall/modules/datasets/aws/lf_dataset_client.py +++ b/backend/dataall/modules/s3_datasets/aws/lf_dataset_client.py @@ -3,7 +3,7 @@ from dataall.base.aws.sts import SessionHelper from dataall.core.environment.db.environment_models import Environment -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) PIVOT_ROLE_NAME_PREFIX = 'dataallPivotRole' diff --git a/backend/dataall/modules/datasets/aws/lf_table_client.py b/backend/dataall/modules/s3_datasets/aws/lf_table_client.py similarity index 97% rename from backend/dataall/modules/datasets/aws/lf_table_client.py rename to backend/dataall/modules/s3_datasets/aws/lf_table_client.py index 4a8d21337..884c8ca47 100644 --- a/backend/dataall/modules/datasets/aws/lf_table_client.py +++ b/backend/dataall/modules/s3_datasets/aws/lf_table_client.py @@ -2,7 +2,7 @@ from botocore.exceptions import ClientError from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/s3_dataset_bucket_policy_client.py b/backend/dataall/modules/s3_datasets/aws/s3_dataset_bucket_policy_client.py similarity index 97% rename from backend/dataall/modules/datasets/aws/s3_dataset_bucket_policy_client.py rename to backend/dataall/modules/s3_datasets/aws/s3_dataset_bucket_policy_client.py index 57a9dde43..25c9bc79a 100644 --- a/backend/dataall/modules/datasets/aws/s3_dataset_bucket_policy_client.py +++ b/backend/dataall/modules/s3_datasets/aws/s3_dataset_bucket_policy_client.py @@ -4,7 +4,7 @@ from botocore.exceptions import ClientError from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/s3_dataset_client.py b/backend/dataall/modules/s3_datasets/aws/s3_dataset_client.py similarity index 97% rename from backend/dataall/modules/datasets/aws/s3_dataset_client.py rename to backend/dataall/modules/s3_datasets/aws/s3_dataset_client.py index a96589a47..4ed6a0112 100644 --- a/backend/dataall/modules/datasets/aws/s3_dataset_client.py +++ b/backend/dataall/modules/s3_datasets/aws/s3_dataset_client.py @@ -5,7 +5,7 @@ from botocore.exceptions import ClientError from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/s3_location_client.py b/backend/dataall/modules/s3_datasets/aws/s3_location_client.py similarity index 93% rename from backend/dataall/modules/datasets/aws/s3_location_client.py rename to backend/dataall/modules/s3_datasets/aws/s3_location_client.py index 94b2d33a5..de7d9cab1 100644 --- a/backend/dataall/modules/datasets/aws/s3_location_client.py +++ b/backend/dataall/modules/s3_datasets/aws/s3_location_client.py @@ -1,7 +1,7 @@ import logging from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/aws/s3_profiler_client.py b/backend/dataall/modules/s3_datasets/aws/s3_profiler_client.py similarity index 100% rename from backend/dataall/modules/datasets/aws/s3_profiler_client.py rename to backend/dataall/modules/s3_datasets/aws/s3_profiler_client.py diff --git a/backend/dataall/modules/datasets/aws/sns_dataset_client.py b/backend/dataall/modules/s3_datasets/aws/sns_dataset_client.py similarity index 94% rename from backend/dataall/modules/datasets/aws/sns_dataset_client.py rename to backend/dataall/modules/s3_datasets/aws/sns_dataset_client.py index c2cce4e5a..deae637b7 100644 --- a/backend/dataall/modules/datasets/aws/sns_dataset_client.py +++ b/backend/dataall/modules/s3_datasets/aws/sns_dataset_client.py @@ -5,7 +5,7 @@ from dataall.base.aws.sts import SessionHelper from dataall.core.environment.db.environment_models import Environment -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/cdk/__init__.py b/backend/dataall/modules/s3_datasets/cdk/__init__.py similarity index 87% rename from backend/dataall/modules/datasets/cdk/__init__.py rename to backend/dataall/modules/s3_datasets/cdk/__init__.py index 88eea5d1f..a75388597 100644 --- a/backend/dataall/modules/datasets/cdk/__init__.py +++ b/backend/dataall/modules/s3_datasets/cdk/__init__.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.cdk import ( +from dataall.modules.s3_datasets.cdk import ( dataset_stack, env_role_dataset_databrew_policy, env_role_dataset_glue_policy, diff --git a/backend/dataall/modules/datasets/cdk/assets/gluedatabasecustomresource/__init__.py b/backend/dataall/modules/s3_datasets/cdk/assets/gluedatabasecustomresource/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/assets/gluedatabasecustomresource/__init__.py rename to backend/dataall/modules/s3_datasets/cdk/assets/gluedatabasecustomresource/__init__.py diff --git a/backend/dataall/modules/datasets/cdk/assets/gluedatabasecustomresource/index.py b/backend/dataall/modules/s3_datasets/cdk/assets/gluedatabasecustomresource/index.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/assets/gluedatabasecustomresource/index.py rename to backend/dataall/modules/s3_datasets/cdk/assets/gluedatabasecustomresource/index.py diff --git a/backend/dataall/modules/datasets/cdk/assets/glueprofilingjob/glue_script.py b/backend/dataall/modules/s3_datasets/cdk/assets/glueprofilingjob/glue_script.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/assets/glueprofilingjob/glue_script.py rename to backend/dataall/modules/s3_datasets/cdk/assets/glueprofilingjob/glue_script.py diff --git a/backend/dataall/modules/datasets/cdk/assets/lakeformationdefaultsettings/__init__.py b/backend/dataall/modules/s3_datasets/cdk/assets/lakeformationdefaultsettings/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/assets/lakeformationdefaultsettings/__init__.py rename to backend/dataall/modules/s3_datasets/cdk/assets/lakeformationdefaultsettings/__init__.py diff --git a/backend/dataall/modules/datasets/cdk/assets/lakeformationdefaultsettings/index.py b/backend/dataall/modules/s3_datasets/cdk/assets/lakeformationdefaultsettings/index.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/assets/lakeformationdefaultsettings/index.py rename to backend/dataall/modules/s3_datasets/cdk/assets/lakeformationdefaultsettings/index.py diff --git a/backend/dataall/modules/datasets/cdk/dataset_custom_resources_extension.py b/backend/dataall/modules/s3_datasets/cdk/dataset_custom_resources_extension.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/dataset_custom_resources_extension.py rename to backend/dataall/modules/s3_datasets/cdk/dataset_custom_resources_extension.py diff --git a/backend/dataall/modules/datasets/cdk/dataset_glue_profiler_extension.py b/backend/dataall/modules/s3_datasets/cdk/dataset_glue_profiler_extension.py similarity index 100% rename from backend/dataall/modules/datasets/cdk/dataset_glue_profiler_extension.py rename to backend/dataall/modules/s3_datasets/cdk/dataset_glue_profiler_extension.py diff --git a/backend/dataall/modules/datasets/cdk/dataset_stack.py b/backend/dataall/modules/s3_datasets/cdk/dataset_stack.py similarity index 98% rename from backend/dataall/modules/datasets/cdk/dataset_stack.py rename to backend/dataall/modules/s3_datasets/cdk/dataset_stack.py index abd428ad1..f1e3188e7 100644 --- a/backend/dataall/modules/datasets/cdk/dataset_stack.py +++ b/backend/dataall/modules/s3_datasets/cdk/dataset_stack.py @@ -22,8 +22,8 @@ from dataall.base.cdkproxy.stacks.manager import stack from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup from dataall.core.stacks.services.runtime_stacks_tagging import TagsUtil -from dataall.modules.datasets.aws.lf_dataset_client import LakeFormationDatasetClient -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.aws.lf_dataset_client import LakeFormationDatasetClient +from dataall.modules.s3_datasets.db.dataset_models import Dataset from dataall.base.utils.cdk_nag_utils import CDKNagUtil from dataall.base.config import config @@ -501,7 +501,7 @@ def __init__(self, scope, id, target_uri: str = None, **kwargs): ) trigger.node.add_dependency(job) - if config.get_property('modules.datasets.features.confidentiality_dropdown', False): + if config.get_property('modules.s3_datasets.features.confidentiality_dropdown', False): Tags.of(self).add('Classification', dataset.confidentiality) TagsUtil.add_tags(stack=self, model=Dataset, target_type='dataset') diff --git a/backend/dataall/modules/datasets/cdk/env_role_dataset_databrew_policy.py b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_databrew_policy.py similarity index 95% rename from backend/dataall/modules/datasets/cdk/env_role_dataset_databrew_policy.py rename to backend/dataall/modules/s3_datasets/cdk/env_role_dataset_databrew_policy.py index eaedf7336..aab4e017d 100644 --- a/backend/dataall/modules/datasets/cdk/env_role_dataset_databrew_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_databrew_policy.py @@ -1,7 +1,7 @@ from dataall.core.environment.cdk.env_role_core_policies.service_policy import ServicePolicy from aws_cdk import aws_iam as iam -from dataall.modules.datasets.services.dataset_permissions import CREATE_DATASET +from dataall.modules.s3_datasets.services.dataset_permissions import CREATE_DATASET class DatasetDatabrewServicePolicy(ServicePolicy): diff --git a/backend/dataall/modules/datasets/cdk/env_role_dataset_glue_policy.py b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_glue_policy.py similarity index 98% rename from backend/dataall/modules/datasets/cdk/env_role_dataset_glue_policy.py rename to backend/dataall/modules/s3_datasets/cdk/env_role_dataset_glue_policy.py index 02a020a05..ca7604e27 100644 --- a/backend/dataall/modules/datasets/cdk/env_role_dataset_glue_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_glue_policy.py @@ -1,7 +1,7 @@ from dataall.core.environment.cdk.env_role_core_policies.service_policy import ServicePolicy from aws_cdk import aws_iam as iam -from dataall.modules.datasets.services.dataset_permissions import CREATE_DATASET +from dataall.modules.s3_datasets.services.dataset_permissions import CREATE_DATASET class DatasetGlueCatalogServicePolicy(ServicePolicy): diff --git a/backend/dataall/modules/datasets/cdk/env_role_dataset_s3_policy.py b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_s3_policy.py similarity index 96% rename from backend/dataall/modules/datasets/cdk/env_role_dataset_s3_policy.py rename to backend/dataall/modules/s3_datasets/cdk/env_role_dataset_s3_policy.py index 2fab433f5..49c19cc29 100644 --- a/backend/dataall/modules/datasets/cdk/env_role_dataset_s3_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/env_role_dataset_s3_policy.py @@ -2,8 +2,8 @@ from aws_cdk import aws_iam as iam from dataall.core.environment.cdk.env_role_core_policies.data_policy import S3Policy -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset class DatasetS3Policy(S3Policy): diff --git a/backend/dataall/modules/datasets/cdk/pivot_role_datasets_policy.py b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py similarity index 98% rename from backend/dataall/modules/datasets/cdk/pivot_role_datasets_policy.py rename to backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py index 72f16de6e..8e5e00905 100644 --- a/backend/dataall/modules/datasets/cdk/pivot_role_datasets_policy.py +++ b/backend/dataall/modules/s3_datasets/cdk/pivot_role_datasets_policy.py @@ -5,8 +5,8 @@ split_policy_with_mutiple_value_condition_in_statements, ) from dataall.core.environment.cdk.pivot_role_stack import PivotRoleStatementSet -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset from aws_cdk import aws_iam as iam diff --git a/backend/dataall/modules/datasets/db/__init__.py b/backend/dataall/modules/s3_datasets/db/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/db/__init__.py rename to backend/dataall/modules/s3_datasets/db/__init__.py diff --git a/backend/dataall/modules/datasets/db/dataset_bucket_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_bucket_repositories.py similarity index 92% rename from backend/dataall/modules/datasets/db/dataset_bucket_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_bucket_repositories.py index ba1da41e3..056d404ec 100644 --- a/backend/dataall/modules/datasets/db/dataset_bucket_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_bucket_repositories.py @@ -1,6 +1,6 @@ import logging -from dataall.modules.datasets.db.dataset_models import DatasetBucket, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetBucket, Dataset logger = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/db/dataset_column_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_column_repositories.py similarity index 94% rename from backend/dataall/modules/datasets/db/dataset_column_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_column_repositories.py index c147d5f77..438cace9b 100644 --- a/backend/dataall/modules/datasets/db/dataset_column_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_column_repositories.py @@ -2,7 +2,7 @@ from dataall.base.db import paginate from dataall.base.db.exceptions import ObjectNotFound -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn class DatasetColumnRepository: diff --git a/backend/dataall/modules/datasets/db/dataset_location_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_location_repositories.py similarity index 97% rename from backend/dataall/modules/datasets/db/dataset_location_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_location_repositories.py index 8cb8c1e45..791aeeded 100644 --- a/backend/dataall/modules/datasets/db/dataset_location_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_location_repositories.py @@ -3,7 +3,7 @@ from sqlalchemy import and_, or_ from dataall.base.db import paginate, exceptions -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset logger = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/db/dataset_models.py b/backend/dataall/modules/s3_datasets/db/dataset_models.py similarity index 98% rename from backend/dataall/modules/datasets/db/dataset_models.py rename to backend/dataall/modules/s3_datasets/db/dataset_models.py index e0fb247a8..87fe21328 100644 --- a/backend/dataall/modules/datasets/db/dataset_models.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_models.py @@ -2,7 +2,7 @@ from sqlalchemy.dialects.postgresql import JSON, ARRAY from sqlalchemy.orm import query_expression from dataall.base.db import Base, Resource, utils -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language class DatasetTableColumn(Resource, Base): diff --git a/backend/dataall/modules/datasets/db/dataset_profiling_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py similarity index 97% rename from backend/dataall/modules/datasets/db/dataset_profiling_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py index 000c364cb..001fcb1b6 100644 --- a/backend/dataall/modules/datasets/db/dataset_profiling_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_profiling_repositories.py @@ -1,7 +1,7 @@ from sqlalchemy import and_ from dataall.base.db import paginate -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun, DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun, DatasetTable class DatasetProfilingRepository: diff --git a/backend/dataall/modules/datasets/db/dataset_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_repositories.py similarity index 98% rename from backend/dataall/modules/datasets/db/dataset_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_repositories.py index 6de362c97..1a22b00db 100644 --- a/backend/dataall/modules/datasets/db/dataset_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_repositories.py @@ -7,9 +7,9 @@ from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.base.db import paginate from dataall.base.db.exceptions import ObjectNotFound -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language from dataall.core.environment.services.environment_resource_manager import EnvironmentResource -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset, DatasetLock +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset, DatasetLock from dataall.base.utils.naming_convention import ( NamingConventionService, NamingConventionPattern, diff --git a/backend/dataall/modules/datasets/db/dataset_table_repositories.py b/backend/dataall/modules/s3_datasets/db/dataset_table_repositories.py similarity index 98% rename from backend/dataall/modules/datasets/db/dataset_table_repositories.py rename to backend/dataall/modules/s3_datasets/db/dataset_table_repositories.py index 6dd3f1325..3625edd8b 100644 --- a/backend/dataall/modules/datasets/db/dataset_table_repositories.py +++ b/backend/dataall/modules/s3_datasets/db/dataset_table_repositories.py @@ -4,7 +4,7 @@ from sqlalchemy.sql import and_ from dataall.base.db import exceptions -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn, DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn, DatasetTable, Dataset from dataall.base.utils import json_utils logger = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/handlers/__init__.py b/backend/dataall/modules/s3_datasets/handlers/__init__.py similarity index 62% rename from backend/dataall/modules/datasets/handlers/__init__.py rename to backend/dataall/modules/s3_datasets/handlers/__init__.py index 5cd2d168f..800da88b7 100644 --- a/backend/dataall/modules/datasets/handlers/__init__.py +++ b/backend/dataall/modules/s3_datasets/handlers/__init__.py @@ -3,6 +3,6 @@ processing in a separate lambda function """ -from dataall.modules.datasets.handlers import glue_table_sync_handler, glue_profiling_handler, glue_dataset_handler +from dataall.modules.s3_datasets.handlers import glue_table_sync_handler, glue_profiling_handler, glue_dataset_handler __all__ = ['glue_table_sync_handler', 'glue_profiling_handler', 'glue_dataset_handler'] diff --git a/backend/dataall/modules/datasets/handlers/glue_dataset_handler.py b/backend/dataall/modules/s3_datasets/handlers/glue_dataset_handler.py similarity index 75% rename from backend/dataall/modules/datasets/handlers/glue_dataset_handler.py rename to backend/dataall/modules/s3_datasets/handlers/glue_dataset_handler.py index a255926af..8d1427c38 100644 --- a/backend/dataall/modules/datasets/handlers/glue_dataset_handler.py +++ b/backend/dataall/modules/s3_datasets/handlers/glue_dataset_handler.py @@ -2,9 +2,9 @@ from dataall.core.tasks.service_handlers import Worker from dataall.core.tasks.db.task_models import Task -from dataall.modules.datasets.aws.glue_dataset_client import DatasetCrawler -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/handlers/glue_profiling_handler.py b/backend/dataall/modules/s3_datasets/handlers/glue_profiling_handler.py similarity index 70% rename from backend/dataall/modules/datasets/handlers/glue_profiling_handler.py rename to backend/dataall/modules/s3_datasets/handlers/glue_profiling_handler.py index a94221796..b7fb41721 100644 --- a/backend/dataall/modules/datasets/handlers/glue_profiling_handler.py +++ b/backend/dataall/modules/s3_datasets/handlers/glue_profiling_handler.py @@ -2,10 +2,10 @@ from dataall.core.tasks.service_handlers import Worker from dataall.core.tasks.db.task_models import Task -from dataall.modules.datasets.aws.glue_profiler_client import GlueDatasetProfilerClient -from dataall.modules.datasets.db.dataset_profiling_repositories import DatasetProfilingRepository -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun, Dataset +from dataall.modules.s3_datasets.aws.glue_profiler_client import GlueDatasetProfilerClient +from dataall.modules.s3_datasets.db.dataset_profiling_repositories import DatasetProfilingRepository +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun, Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/handlers/glue_table_sync_handler.py b/backend/dataall/modules/s3_datasets/handlers/glue_table_sync_handler.py similarity index 88% rename from backend/dataall/modules/datasets/handlers/glue_table_sync_handler.py rename to backend/dataall/modules/s3_datasets/handlers/glue_table_sync_handler.py index 3588f3c02..a9251e716 100644 --- a/backend/dataall/modules/datasets/handlers/glue_table_sync_handler.py +++ b/backend/dataall/modules/s3_datasets/handlers/glue_table_sync_handler.py @@ -3,9 +3,9 @@ from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sts import SessionHelper from dataall.core.tasks.db.task_models import Task -from dataall.modules.datasets.aws.glue_table_client import GlueTableClient -from dataall.modules.datasets.aws.lf_table_client import LakeFormationTableClient -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn, DatasetTable +from dataall.modules.s3_datasets.aws.glue_table_client import GlueTableClient +from dataall.modules.s3_datasets.aws.lf_table_client import LakeFormationTableClient +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn, DatasetTable log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/indexers/__init__.py b/backend/dataall/modules/s3_datasets/indexers/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/indexers/__init__.py rename to backend/dataall/modules/s3_datasets/indexers/__init__.py diff --git a/backend/dataall/modules/datasets/indexers/dataset_catalog_indexer.py b/backend/dataall/modules/s3_datasets/indexers/dataset_catalog_indexer.py similarity index 69% rename from backend/dataall/modules/datasets/indexers/dataset_catalog_indexer.py rename to backend/dataall/modules/s3_datasets/indexers/dataset_catalog_indexer.py index 015476467..5b6a4d5c8 100644 --- a/backend/dataall/modules/datasets/indexers/dataset_catalog_indexer.py +++ b/backend/dataall/modules/s3_datasets/indexers/dataset_catalog_indexer.py @@ -1,10 +1,10 @@ import logging -from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer -from dataall.modules.datasets.indexers.location_indexer import DatasetLocationIndexer -from dataall.modules.datasets.indexers.table_indexer import DatasetTableIndexer -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer +from dataall.modules.s3_datasets.indexers.location_indexer import DatasetLocationIndexer +from dataall.modules.s3_datasets.indexers.table_indexer import DatasetTableIndexer +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset from dataall.modules.catalog.indexers.catalog_indexer import CatalogIndexer log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/indexers/dataset_indexer.py b/backend/dataall/modules/s3_datasets/indexers/dataset_indexer.py similarity index 93% rename from backend/dataall/modules/datasets/indexers/dataset_indexer.py rename to backend/dataall/modules/s3_datasets/indexers/dataset_indexer.py index e2764a82b..e245bcea1 100644 --- a/backend/dataall/modules/datasets/indexers/dataset_indexer.py +++ b/backend/dataall/modules/s3_datasets/indexers/dataset_indexer.py @@ -5,8 +5,8 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.organizations.db.organization_repositories import OrganizationRepository from dataall.modules.vote.db.vote_repositories import VoteRepository -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_location_repositories import DatasetLocationRepository +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_location_repositories import DatasetLocationRepository from dataall.modules.catalog.indexers.base_indexer import BaseIndexer diff --git a/backend/dataall/modules/datasets/indexers/location_indexer.py b/backend/dataall/modules/s3_datasets/indexers/location_indexer.py similarity index 90% rename from backend/dataall/modules/datasets/indexers/location_indexer.py rename to backend/dataall/modules/s3_datasets/indexers/location_indexer.py index 3ee140182..b5216dacb 100644 --- a/backend/dataall/modules/datasets/indexers/location_indexer.py +++ b/backend/dataall/modules/s3_datasets/indexers/location_indexer.py @@ -4,9 +4,9 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.organizations.db.organization_repositories import OrganizationRepository -from dataall.modules.datasets.db.dataset_location_repositories import DatasetLocationRepository -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer +from dataall.modules.s3_datasets.db.dataset_location_repositories import DatasetLocationRepository +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer from dataall.modules.catalog.indexers.base_indexer import BaseIndexer diff --git a/backend/dataall/modules/datasets/indexers/table_indexer.py b/backend/dataall/modules/s3_datasets/indexers/table_indexer.py similarity index 91% rename from backend/dataall/modules/datasets/indexers/table_indexer.py rename to backend/dataall/modules/s3_datasets/indexers/table_indexer.py index 4f4f4c8c3..15b6320c1 100644 --- a/backend/dataall/modules/datasets/indexers/table_indexer.py +++ b/backend/dataall/modules/s3_datasets/indexers/table_indexer.py @@ -4,9 +4,9 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.organizations.db.organization_repositories import OrganizationRepository -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer from dataall.modules.catalog.indexers.base_indexer import BaseIndexer diff --git a/backend/dataall/modules/datasets/services/__init__.py b/backend/dataall/modules/s3_datasets/services/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/services/__init__.py rename to backend/dataall/modules/s3_datasets/services/__init__.py diff --git a/backend/dataall/modules/datasets/services/dataset_alarm_service.py b/backend/dataall/modules/s3_datasets/services/dataset_alarm_service.py similarity index 94% rename from backend/dataall/modules/datasets/services/dataset_alarm_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_alarm_service.py index 35c2940b7..70369d603 100644 --- a/backend/dataall/modules/datasets/services/dataset_alarm_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_alarm_service.py @@ -2,7 +2,7 @@ from datetime import datetime -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset from dataall.base.utils.alarm_service import AlarmService log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/services/dataset_column_service.py b/backend/dataall/modules/s3_datasets/services/dataset_column_service.py similarity index 81% rename from backend/dataall/modules/datasets/services/dataset_column_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_column_service.py index 387b930be..eba100f0f 100644 --- a/backend/dataall/modules/datasets/services/dataset_column_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_column_service.py @@ -3,14 +3,14 @@ from dataall.base.aws.sts import SessionHelper from dataall.base.context import get_context from dataall.core.tasks.db.task_models import Task -from dataall.modules.datasets.aws.glue_table_client import GlueTableClient -from dataall.modules.datasets.db.dataset_column_repositories import DatasetColumnRepository -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository -from dataall.modules.datasets.services.dataset_permissions import UPDATE_DATASET_TABLE -from dataall.modules.datasets.db.dataset_models import DatasetTable, DatasetTableColumn -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification -from dataall.modules.datasets.services.dataset_permissions import PREVIEW_DATASET_TABLE +from dataall.modules.s3_datasets.aws.glue_table_client import GlueTableClient +from dataall.modules.s3_datasets.db.dataset_column_repositories import DatasetColumnRepository +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.services.dataset_permissions import UPDATE_DATASET_TABLE +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, DatasetTableColumn +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification +from dataall.modules.s3_datasets.services.dataset_permissions import PREVIEW_DATASET_TABLE class DatasetColumnService: diff --git a/backend/dataall/modules/datasets/services/dataset_location_service.py b/backend/dataall/modules/s3_datasets/services/dataset_location_service.py similarity index 89% rename from backend/dataall/modules/datasets/services/dataset_location_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_location_service.py index 8e65635a5..870bb03a8 100644 --- a/backend/dataall/modules/datasets/services/dataset_location_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_location_service.py @@ -3,20 +3,20 @@ from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.base.db.exceptions import ResourceShared, ResourceAlreadyExists -from dataall.modules.datasets.services.dataset_service import DatasetService -from dataall.modules.datasets.aws.s3_location_client import S3LocationClient -from dataall.modules.datasets.db.dataset_location_repositories import DatasetLocationRepository -from dataall.modules.datasets.indexers.location_indexer import DatasetLocationIndexer -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.aws.s3_location_client import S3LocationClient +from dataall.modules.s3_datasets.db.dataset_location_repositories import DatasetLocationRepository +from dataall.modules.s3_datasets.indexers.location_indexer import DatasetLocationIndexer +from dataall.modules.s3_datasets.services.dataset_permissions import ( UPDATE_DATASET_FOLDER, MANAGE_DATASETS, CREATE_DATASET_FOLDER, LIST_DATASET_FOLDERS, DELETE_DATASET_FOLDER, ) -from dataall.modules.datasets.services.dataset_permissions import DATASET_FOLDER_READ, GET_DATASET_FOLDER -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_FOLDER_READ, GET_DATASET_FOLDER +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset class DatasetLocationService: diff --git a/backend/dataall/modules/datasets/services/dataset_permissions.py b/backend/dataall/modules/s3_datasets/services/dataset_permissions.py similarity index 100% rename from backend/dataall/modules/datasets/services/dataset_permissions.py rename to backend/dataall/modules/s3_datasets/services/dataset_permissions.py diff --git a/backend/dataall/modules/datasets/services/dataset_profiling_service.py b/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py similarity index 85% rename from backend/dataall/modules/datasets/services/dataset_profiling_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py index 3ccb2c5ac..183db0b45 100644 --- a/backend/dataall/modules/datasets/services/dataset_profiling_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_profiling_service.py @@ -7,15 +7,15 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.tasks.db.task_models import Task from dataall.base.db.exceptions import ObjectNotFound -from dataall.modules.datasets.aws.glue_profiler_client import GlueDatasetProfilerClient -from dataall.modules.datasets.aws.s3_profiler_client import S3ProfilerClient -from dataall.modules.datasets.db.dataset_profiling_repositories import DatasetProfilingRepository -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository -from dataall.modules.datasets.services.dataset_permissions import PROFILE_DATASET_TABLE, GET_DATASET -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun, DatasetTable -from dataall.modules.datasets.services.dataset_permissions import PREVIEW_DATASET_TABLE +from dataall.modules.s3_datasets.aws.glue_profiler_client import GlueDatasetProfilerClient +from dataall.modules.s3_datasets.aws.s3_profiler_client import S3ProfilerClient +from dataall.modules.s3_datasets.db.dataset_profiling_repositories import DatasetProfilingRepository +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.services.dataset_permissions import PROFILE_DATASET_TABLE, GET_DATASET +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun, DatasetTable +from dataall.modules.s3_datasets.services.dataset_permissions import PREVIEW_DATASET_TABLE class DatasetProfilingService: diff --git a/backend/dataall/modules/datasets/services/dataset_service.py b/backend/dataall/modules/s3_datasets/services/dataset_service.py similarity index 96% rename from backend/dataall/modules/datasets/services/dataset_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_service.py index a1b13867d..fbf574a51 100644 --- a/backend/dataall/modules/datasets/services/dataset_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_service.py @@ -11,7 +11,7 @@ from dataall.core.stacks.services.stack_service import StackService from dataall.core.tasks.service_handlers import Worker from dataall.base.aws.sts import SessionHelper -from dataall.modules.datasets.aws.kms_dataset_client import KmsClient +from dataall.modules.s3_datasets.aws.kms_dataset_client import KmsClient from dataall.base.context import get_context from dataall.core.permissions.services.group_policy_service import GroupPolicyService from dataall.core.environment.services.environment_service import EnvironmentService @@ -20,14 +20,14 @@ from dataall.core.stacks.db.stack_models import Stack from dataall.core.tasks.db.task_models import Task from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository -from dataall.modules.datasets.db.dataset_bucket_repositories import DatasetBucketRepository +from dataall.modules.s3_datasets.db.dataset_bucket_repositories import DatasetBucketRepository from dataall.modules.vote.db.vote_repositories import VoteRepository -from dataall.modules.datasets.aws.glue_dataset_client import DatasetCrawler -from dataall.modules.datasets.aws.s3_dataset_client import S3DatasetClient -from dataall.modules.datasets.db.dataset_location_repositories import DatasetLocationRepository -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository -from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler +from dataall.modules.s3_datasets.aws.s3_dataset_client import S3DatasetClient +from dataall.modules.s3_datasets.db.dataset_location_repositories import DatasetLocationRepository +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer +from dataall.modules.s3_datasets.services.dataset_permissions import ( CREDENTIALS_DATASET, CRAWL_DATASET, DELETE_DATASET, @@ -39,10 +39,10 @@ DATASET_READ, IMPORT_DATASET, ) -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.datasets_enums import DatasetRole -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetTable -from dataall.modules.datasets.services.dataset_permissions import DATASET_TABLE_READ +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.datasets_enums import DatasetRole +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetTable +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/services/dataset_table_service.py b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py similarity index 90% rename from backend/dataall/modules/datasets/services/dataset_table_service.py rename to backend/dataall/modules/s3_datasets/services/dataset_table_service.py index ddbd71a5a..dd758150d 100644 --- a/backend/dataall/modules/datasets/services/dataset_table_service.py +++ b/backend/dataall/modules/s3_datasets/services/dataset_table_service.py @@ -5,25 +5,25 @@ from dataall.core.permissions.services.tenant_policy_service import TenantPolicyService from dataall.modules.catalog.db.glossary_repositories import GlossaryRepository from dataall.core.environment.services.environment_service import EnvironmentService -from dataall.modules.datasets.aws.athena_table_client import AthenaTableClient -from dataall.modules.datasets.aws.glue_dataset_client import DatasetCrawler -from dataall.modules.datasets.db.dataset_table_repositories import DatasetTableRepository -from dataall.modules.datasets.indexers.table_indexer import DatasetTableIndexer -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.aws.athena_table_client import AthenaTableClient +from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler +from dataall.modules.s3_datasets.db.dataset_table_repositories import DatasetTableRepository +from dataall.modules.s3_datasets.indexers.table_indexer import DatasetTableIndexer +from dataall.modules.s3_datasets.services.dataset_permissions import ( UPDATE_DATASET_TABLE, MANAGE_DATASETS, DELETE_DATASET_TABLE, SYNC_DATASET, ) -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.services.dataset_permissions import ( PREVIEW_DATASET_TABLE, DATASET_TABLE_READ, GET_DATASET_TABLE, ) -from dataall.modules.datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.services.dataset_service import DatasetService from dataall.base.utils import json_utils log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/services/datasets_enums.py b/backend/dataall/modules/s3_datasets/services/datasets_enums.py similarity index 89% rename from backend/dataall/modules/datasets/services/datasets_enums.py rename to backend/dataall/modules/s3_datasets/services/datasets_enums.py index abf44bd2c..fb6201d5f 100644 --- a/backend/dataall/modules/datasets/services/datasets_enums.py +++ b/backend/dataall/modules/s3_datasets/services/datasets_enums.py @@ -2,7 +2,7 @@ from dataall.base.config import config from dataall.base.db.exceptions import InvalidInput -custom_confidentiality_mapping = config.get_property('modules.datasets.features.custom_confidentiality_mapping', {}) +custom_confidentiality_mapping = config.get_property('modules.s3_datasets.features.custom_confidentiality_mapping', {}) class DatasetRole(GraphQLEnumMapper): @@ -36,7 +36,7 @@ def get_confidentiality_level(confidentiality): @staticmethod def validate_confidentiality_level(confidentiality): - if config.get_property('modules.datasets.features.confidentiality_dropdown', False): + if config.get_property('modules.s3_datasets.features.confidentiality_dropdown', False): confidentiality = ConfidentialityClassification.get_confidentiality_level(confidentiality) if confidentiality not in [item.value for item in list(ConfidentialityClassification)]: raise InvalidInput( diff --git a/backend/dataall/modules/datasets/tasks/__init__.py b/backend/dataall/modules/s3_datasets/tasks/__init__.py similarity index 100% rename from backend/dataall/modules/datasets/tasks/__init__.py rename to backend/dataall/modules/s3_datasets/tasks/__init__.py diff --git a/backend/dataall/modules/datasets/tasks/dataset_stack_finder.py b/backend/dataall/modules/s3_datasets/tasks/dataset_stack_finder.py similarity index 80% rename from backend/dataall/modules/datasets/tasks/dataset_stack_finder.py rename to backend/dataall/modules/s3_datasets/tasks/dataset_stack_finder.py index 684dbf245..94ff6f8c7 100644 --- a/backend/dataall/modules/datasets/tasks/dataset_stack_finder.py +++ b/backend/dataall/modules/s3_datasets/tasks/dataset_stack_finder.py @@ -2,8 +2,8 @@ from typing import List from dataall.core.environment.tasks.env_stack_finder import StackFinder -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import Dataset log = logging.getLogger(__name__) diff --git a/backend/dataall/modules/datasets/tasks/tables_syncer.py b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py similarity index 85% rename from backend/dataall/modules/datasets/tasks/tables_syncer.py rename to backend/dataall/modules/s3_datasets/tasks/tables_syncer.py index 2199ad34a..502e564eb 100644 --- a/backend/dataall/modules/datasets/tasks/tables_syncer.py +++ b/backend/dataall/modules/s3_datasets/tasks/tables_syncer.py @@ -7,13 +7,13 @@ from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db import get_engine -from dataall.modules.datasets.aws.glue_dataset_client import DatasetCrawler -from dataall.modules.datasets.aws.lf_table_client import LakeFormationTableClient -from dataall.modules.datasets.services.dataset_table_service import DatasetTableService -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset -from dataall.modules.datasets.indexers.table_indexer import DatasetTableIndexer -from dataall.modules.datasets.services.dataset_alarm_service import DatasetAlarmService +from dataall.modules.s3_datasets.aws.glue_dataset_client import DatasetCrawler +from dataall.modules.s3_datasets.aws.lf_table_client import LakeFormationTableClient +from dataall.modules.s3_datasets.services.dataset_table_service import DatasetTableService +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.indexers.table_indexer import DatasetTableIndexer +from dataall.modules.s3_datasets.services.dataset_alarm_service import DatasetAlarmService root = logging.getLogger() root.setLevel(logging.INFO) diff --git a/backend/migrations/versions/5e5c84138af7_backfill_confidentiality.py b/backend/migrations/versions/5e5c84138af7_backfill_confidentiality.py index cd811fae7..c2590d457 100644 --- a/backend/migrations/versions/5e5c84138af7_backfill_confidentiality.py +++ b/backend/migrations/versions/5e5c84138af7_backfill_confidentiality.py @@ -14,7 +14,7 @@ from sqlalchemy.ext.declarative import declarative_base from dataall.base.db import utils, Resource -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language revision = '5e5c84138af7' diff --git a/backend/migrations/versions/72b8a90b6ee8__share_request_purpose.py b/backend/migrations/versions/72b8a90b6ee8__share_request_purpose.py index 54f6de292..c5c5d703a 100644 --- a/backend/migrations/versions/72b8a90b6ee8__share_request_purpose.py +++ b/backend/migrations/versions/72b8a90b6ee8__share_request_purpose.py @@ -14,7 +14,7 @@ from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService from dataall.modules.dataset_sharing.db.share_object_models import ShareObject from dataall.modules.dataset_sharing.services.share_permissions import SHARE_OBJECT_APPROVER, SHARE_OBJECT_REQUESTER -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository # revision identifiers, used by Alembic. revision = '72b8a90b6ee8' diff --git a/backend/migrations/versions/8c79fb896983_add_table_for_buckets.py b/backend/migrations/versions/8c79fb896983_add_table_for_buckets.py index c54ed81f7..589896927 100644 --- a/backend/migrations/versions/8c79fb896983_add_table_for_buckets.py +++ b/backend/migrations/versions/8c79fb896983_add_table_for_buckets.py @@ -19,7 +19,7 @@ from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareObjectStatus from datetime import datetime -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language # revision identifiers, used by Alembic. diff --git a/backend/migrations/versions/97050ec09354_release_3_7_8.py b/backend/migrations/versions/97050ec09354_release_3_7_8.py index f4bcad6dd..882af867e 100644 --- a/backend/migrations/versions/97050ec09354_release_3_7_8.py +++ b/backend/migrations/versions/97050ec09354_release_3_7_8.py @@ -12,7 +12,7 @@ from sqlalchemy.ext.declarative import declarative_base from dataall.base.db import utils, Resource -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language # revision identifiers, used by Alembic. diff --git a/backend/migrations/versions/c6d01930179d_add_backfill_read_folder_permissions.py b/backend/migrations/versions/c6d01930179d_add_backfill_read_folder_permissions.py index 028775dd4..0c3fc64b4 100644 --- a/backend/migrations/versions/c6d01930179d_add_backfill_read_folder_permissions.py +++ b/backend/migrations/versions/c6d01930179d_add_backfill_read_folder_permissions.py @@ -12,8 +12,8 @@ from dataall.core.permissions.api.enums import PermissionType from dataall.core.permissions.services.permission_service import PermissionService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.modules.datasets.services.dataset_permissions import DATASET_FOLDER_READ, GET_DATASET_FOLDER -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_FOLDER_READ, GET_DATASET_FOLDER +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset from dataall.modules.dataset_sharing.db.share_object_models import ShareObject, ShareObjectItem from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareItemStatus, ShareableType diff --git a/backend/migrations/versions/d05f9a5b215e_backfill_dataset_table_permissions.py b/backend/migrations/versions/d05f9a5b215e_backfill_dataset_table_permissions.py index 17cbca3b0..c80a098da 100644 --- a/backend/migrations/versions/d05f9a5b215e_backfill_dataset_table_permissions.py +++ b/backend/migrations/versions/d05f9a5b215e_backfill_dataset_table_permissions.py @@ -25,8 +25,8 @@ ShareItemStatus, ) from dataall.modules.dataset_sharing.db.share_object_repositories import ShareObjectRepository -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.services.dataset_permissions import DATASET_TABLE_READ +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ # revision identifiers, used by Alembic. revision = 'd05f9a5b215e' diff --git a/backend/migrations/versions/e1cd4927482b_rename_imported_dataset_aws_resources.py b/backend/migrations/versions/e1cd4927482b_rename_imported_dataset_aws_resources.py index 12096b123..30adbdf89 100644 --- a/backend/migrations/versions/e1cd4927482b_rename_imported_dataset_aws_resources.py +++ b/backend/migrations/versions/e1cd4927482b_rename_imported_dataset_aws_resources.py @@ -16,7 +16,7 @@ NamingConventionService, NamingConventionPattern, ) -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification, Language +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification, Language # revision identifiers, used by Alembic. revision = 'e1cd4927482b' diff --git a/deploy/stacks/backend_stack.py b/deploy/stacks/backend_stack.py index 61c5a1ee8..42206bc48 100644 --- a/deploy/stacks/backend_stack.py +++ b/deploy/stacks/backend_stack.py @@ -403,7 +403,7 @@ def __init__( internet_facing=internet_facing, ) - @run_if(['modules.datasets.features.share_notifications.email.active']) + @run_if(['modules.s3_datasets.features.share_notifications.email.active']) def create_ses_stack(self, custom_domain, envname, kwargs, resource_prefix): if custom_domain is None or None in [ custom_domain.get('hosted_zone_name', None), diff --git a/deploy/stacks/container.py b/deploy/stacks/container.py index a8dd7e27c..b4a1a7543 100644 --- a/deploy/stacks/container.py +++ b/deploy/stacks/container.py @@ -178,7 +178,7 @@ def __init__( self.add_share_verifier_task() self.add_share_reapplier_task() - @run_if(['modules.datasets.active', 'modules.dashboards.active']) + @run_if(['modules.s3_datasets.active', 'modules.dashboards.active']) def add_catalog_indexer_task(self): catalog_indexer_task, catalog_indexer_task_def = self.set_scheduled_task( cluster=self.ecs_cluster, @@ -199,7 +199,7 @@ def add_catalog_indexer_task(self): self.ecs_task_definitions_families.append(catalog_indexer_task.task_definition.family) - @run_if(['modules.datasets.active']) + @run_if(['modules.s3_datasets.active']) def add_share_management_task(self): share_management_task_definition = ecs.FargateTaskDefinition( self, @@ -239,7 +239,7 @@ def add_share_management_task(self): ) self.ecs_task_definitions_families.append(share_management_task_definition.family) - @run_if(['modules.datasets.active']) + @run_if(['modules.s3_datasets.active']) def add_share_verifier_task(self): verify_shares_task, verify_shares_task_def = self.set_scheduled_task( cluster=self.ecs_cluster, @@ -259,7 +259,7 @@ def add_share_verifier_task(self): ) self.ecs_task_definitions_families.append(verify_shares_task.task_definition.family) - @run_if(['modules.datasets.active']) + @run_if(['modules.s3_datasets.active']) def add_share_reapplier_task(self): share_reapplier_task_definition = ecs.FargateTaskDefinition( self, @@ -285,7 +285,7 @@ def add_share_reapplier_task(self): ) self.ecs_task_definitions_families.append(share_reapplier_task_definition.family) - @run_if(['modules.datasets.active']) + @run_if(['modules.s3_datasets.active']) def add_subscription_task(self): subscriptions_task, subscription_task_def = self.set_scheduled_task( cluster=self.ecs_cluster, @@ -309,11 +309,11 @@ def add_subscription_task(self): ) self.ecs_task_definitions_families.append(subscriptions_task.task_definition.family) - @run_if(['modules.datasets.active']) + @run_if(['modules.s3_datasets.active']) def add_sync_dataset_table_task(self): sync_tables_task, sync_tables_task_def = self.set_scheduled_task( cluster=self.ecs_cluster, - command=['python3.9', '-m', 'dataall.modules.datasets.tasks.tables_syncer'], + command=['python3.9', '-m', 'dataall.modules.s3_datasets.tasks.tables_syncer'], container_id='container', ecr_repository=self._ecr_repository, environment=self._create_env('INFO'), diff --git a/tests/conftest.py b/tests/conftest.py index 44e7d3006..c0c78991d 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -185,7 +185,7 @@ def mock_aws_client(module_mocker): session = MagicMock() # there can be other mocker clients - module_mocker.patch('dataall.modules.datasets.aws.s3_dataset_client.SessionHelper', session_helper) + module_mocker.patch('dataall.modules.s3_datasets.aws.s3_dataset_client.SessionHelper', session_helper) module_mocker.patch('dataall.modules.dataset_sharing.aws.kms_client.SessionHelper', session_helper) diff --git a/tests/modules/datasets/conftest.py b/tests/modules/datasets/conftest.py index 5a507b48b..55b1747ce 100644 --- a/tests/modules/datasets/conftest.py +++ b/tests/modules/datasets/conftest.py @@ -9,37 +9,37 @@ from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareableType, PrincipalType from dataall.modules.dataset_sharing.db.share_object_models import ShareObject, ShareObjectItem from dataall.modules.dataset_sharing.services.share_permissions import SHARE_OBJECT_REQUESTER, SHARE_OBJECT_APPROVER -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification -from dataall.modules.datasets.services.dataset_permissions import DATASET_TABLE_READ -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetTable, DatasetStorageLocation -from dataall.modules.datasets.services.dataset_permissions import DATASET_ALL +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetTable, DatasetStorageLocation +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_ALL @pytest.fixture(scope='module', autouse=True) def patch_dataset_methods(module_mocker): module_mocker.patch( - 'dataall.modules.datasets.services.dataset_service.DatasetService.check_dataset_account', return_value=True + 'dataall.modules.s3_datasets.services.dataset_service.DatasetService.check_dataset_account', return_value=True ) module_mocker.patch( - 'dataall.modules.datasets.services.dataset_service.DatasetService._deploy_dataset_stack', return_value=True + 'dataall.modules.s3_datasets.services.dataset_service.DatasetService._deploy_dataset_stack', return_value=True ) s3_mock_client = MagicMock() glue_mock_client = MagicMock() - module_mocker.patch('dataall.modules.datasets.services.dataset_profiling_service.S3ProfilerClient', s3_mock_client) + module_mocker.patch('dataall.modules.s3_datasets.services.dataset_profiling_service.S3ProfilerClient', s3_mock_client) module_mocker.patch( - 'dataall.modules.datasets.services.dataset_profiling_service.GlueDatasetProfilerClient', glue_mock_client + 'dataall.modules.s3_datasets.services.dataset_profiling_service.GlueDatasetProfilerClient', glue_mock_client ) s3_mock_client().get_profiling_results_from_s3.return_value = '{"results": "yes"}' glue_mock_client().run_job.return_value = True module_mocker.patch( - 'dataall.modules.datasets.services.datasets_enums.ConfidentialityClassification.validate_confidentiality_level', + 'dataall.modules.s3_datasets.services.datasets_enums.ConfidentialityClassification.validate_confidentiality_level', return_value=True, ) confidentiality_classification_mocker = MagicMock() module_mocker.patch( - 'dataall.modules.datasets.services.datasets_enums.ConfidentialityClassification', + 'dataall.modules.s3_datasets.services.datasets_enums.ConfidentialityClassification', return_value=confidentiality_classification_mocker, ) # Return the input when mocking. This mock avoids checking the custom_confidentiality_mapping value in the actual function and just returns whatever confidentiality value is supplied for pytests diff --git a/tests/modules/datasets/tasks/conftest.py b/tests/modules/datasets/tasks/conftest.py index 373cbc450..657bc583d 100644 --- a/tests/modules/datasets/tasks/conftest.py +++ b/tests/modules/datasets/tasks/conftest.py @@ -9,7 +9,7 @@ PrincipalType, ) from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetBucket @pytest.fixture(scope='module') diff --git a/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py b/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py index f58bdc2ad..4414c9abf 100644 --- a/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py +++ b/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py @@ -1,7 +1,7 @@ import pytest from dataall.modules.catalog.tasks.catalog_indexer_task import index_objects -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset @pytest.fixture(scope='module', autouse=True) @@ -48,8 +48,8 @@ def table(org, env, db, sync_dataset): def test_catalog_indexer(db, org, env, sync_dataset, table, mocker): - mocker.patch('dataall.modules.datasets.indexers.table_indexer.DatasetTableIndexer.upsert_all', return_value=[table]) - mocker.patch('dataall.modules.datasets.indexers.dataset_indexer.DatasetIndexer.upsert', return_value=sync_dataset) + mocker.patch('dataall.modules.s3_datasets.indexers.table_indexer.DatasetTableIndexer.upsert_all', return_value=[table]) + mocker.patch('dataall.modules.s3_datasets.indexers.dataset_indexer.DatasetIndexer.upsert', return_value=sync_dataset) indexed_objects_counter = index_objects(engine=db) # Count should be One table + One Dataset = 2 assert indexed_objects_counter == 2 diff --git a/tests/modules/datasets/tasks/test_dataset_subscriptions.py b/tests/modules/datasets/tasks/test_dataset_subscriptions.py index 2e5cdecee..395fc1d4e 100644 --- a/tests/modules/datasets/tasks/test_dataset_subscriptions.py +++ b/tests/modules/datasets/tasks/test_dataset_subscriptions.py @@ -11,7 +11,7 @@ PrincipalType, ) from dataall.modules.dataset_sharing.db.share_object_models import ShareObjectItem, ShareObject -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset from dataall.modules.dataset_sharing.tasks.dataset_subscription_task import DatasetSubscriptionService from dataall.core.environment.api.enums import EnvironmentPermission diff --git a/tests/modules/datasets/tasks/test_dataset_tables_sync.py b/tests/modules/datasets/tasks/test_dataset_tables_sync.py index 49755daa1..8dc6fb367 100644 --- a/tests/modules/datasets/tasks/test_dataset_tables_sync.py +++ b/tests/modules/datasets/tasks/test_dataset_tables_sync.py @@ -1,8 +1,8 @@ from unittest.mock import MagicMock import pytest -from dataall.modules.datasets.db.dataset_models import DatasetTable -from dataall.modules.datasets.tasks.tables_syncer import sync_tables +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable +from dataall.modules.s3_datasets.tasks.tables_syncer import sync_tables @pytest.fixture(scope='module', autouse=True) @@ -31,7 +31,7 @@ def table_fixture(org, env, db, sync_dataset): def test_tables_sync(db, org, env, sync_dataset, table_fixture, mocker): mock_crawler = MagicMock() - mocker.patch('dataall.modules.datasets.tasks.tables_syncer.DatasetCrawler', mock_crawler) + mocker.patch('dataall.modules.s3_datasets.tasks.tables_syncer.DatasetCrawler', mock_crawler) mocker.patch( 'dataall.base.aws.sts.SessionHelper.get_delegation_role_arn', return_value='arn:role', @@ -88,10 +88,10 @@ def test_tables_sync(db, org, env, sync_dataset, table_fixture, mocker): }, ] - mocker.patch('dataall.modules.datasets.tasks.tables_syncer.is_assumable_pivot_role', return_value=True) + mocker.patch('dataall.modules.s3_datasets.tasks.tables_syncer.is_assumable_pivot_role', return_value=True) mock_client = MagicMock() - mocker.patch('dataall.modules.datasets.tasks.tables_syncer.LakeFormationTableClient', mock_client) + mocker.patch('dataall.modules.s3_datasets.tasks.tables_syncer.LakeFormationTableClient', mock_client) mock_client.grant_principals_all_table_permissions = True processed_tables = sync_tables(engine=db) diff --git a/tests/modules/datasets/tasks/test_lf_share_manager.py b/tests/modules/datasets/tasks/test_lf_share_manager.py index 43934dd76..3e17dd3be 100644 --- a/tests/modules/datasets/tasks/test_lf_share_manager.py +++ b/tests/modules/datasets/tasks/test_lf_share_manager.py @@ -16,7 +16,7 @@ from dataall.core.environment.db.environment_models import Environment, EnvironmentGroup from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareItemStatus from dataall.modules.dataset_sharing.db.share_object_models import ShareObject, ShareObjectItem -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset from dataall.modules.dataset_sharing.services.dataset_sharing_alarm_service import DatasetSharingAlarmService from dataall.modules.dataset_sharing.services.share_processors.lakeformation_process_share import ( ProcessLakeFormationShare, diff --git a/tests/modules/datasets/tasks/test_s3_access_point_share_manager.py b/tests/modules/datasets/tasks/test_s3_access_point_share_manager.py index fbfe2020e..e57d0be45 100644 --- a/tests/modules/datasets/tasks/test_s3_access_point_share_manager.py +++ b/tests/modules/datasets/tasks/test_s3_access_point_share_manager.py @@ -12,7 +12,7 @@ from dataall.modules.dataset_sharing.db.share_object_models import ShareObject, ShareObjectItem from dataall.modules.dataset_sharing.services.managed_share_policy_service import SharePolicyService from dataall.modules.dataset_sharing.services.share_managers import S3AccessPointShareManager -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, Dataset SOURCE_ENV_ACCOUNT = '111111111111' SOURCE_ENV_ROLE_NAME = 'dataall-ProducerEnvironment-i6v1v1c2' diff --git a/tests/modules/datasets/tasks/test_s3_bucket_share_manager.py b/tests/modules/datasets/tasks/test_s3_bucket_share_manager.py index 48a2561e3..4f61dee78 100644 --- a/tests/modules/datasets/tasks/test_s3_bucket_share_manager.py +++ b/tests/modules/datasets/tasks/test_s3_bucket_share_manager.py @@ -10,7 +10,7 @@ from dataall.modules.dataset_sharing.db.share_object_models import ShareObject from dataall.modules.dataset_sharing.services.share_managers import S3BucketShareManager from dataall.modules.dataset_sharing.services.managed_share_policy_service import SharePolicyService -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetBucket +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetBucket SOURCE_ENV_ACCOUNT = '111111111111' SOURCE_ENV_ROLE_NAME = 'dataall-ProducerEnvironment-i6v1v1c2' diff --git a/tests/modules/datasets/tasks/test_stacks_updater_with_datasets.py b/tests/modules/datasets/tasks/test_stacks_updater_with_datasets.py index 68380aa04..416b5c95b 100644 --- a/tests/modules/datasets/tasks/test_stacks_updater_with_datasets.py +++ b/tests/modules/datasets/tasks/test_stacks_updater_with_datasets.py @@ -1,5 +1,5 @@ import pytest -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset from dataall.core.environment.tasks.env_stacks_updater import update_stacks diff --git a/tests/modules/datasets/test_dataset.py b/tests/modules/datasets/test_dataset.py index 34fd6a69b..8340bb366 100644 --- a/tests/modules/datasets/test_dataset.py +++ b/tests/modules/datasets/test_dataset.py @@ -6,11 +6,11 @@ from dataall.base.config import config from dataall.core.environment.db.environment_models import Environment from dataall.core.organizations.db.organization_models import Organization -from dataall.modules.datasets.db.dataset_repositories import DatasetRepository -from dataall.modules.datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetLock +from dataall.modules.s3_datasets.db.dataset_repositories import DatasetRepository +from dataall.modules.s3_datasets.db.dataset_models import DatasetStorageLocation, DatasetTable, Dataset, DatasetLock from tests.core.stacks.test_stack import update_stack_query -from dataall.modules.datasets.services.datasets_enums import ConfidentialityClassification +from dataall.modules.s3_datasets.services.datasets_enums import ConfidentialityClassification mocked_key_id = 'some_key' @@ -19,7 +19,7 @@ @pytest.fixture(scope='module', autouse=True) def mock_s3_client(module_mocker): s3_client = MagicMock() - module_mocker.patch('dataall.modules.datasets.services.dataset_service.S3DatasetClient', s3_client) + module_mocker.patch('dataall.modules.s3_datasets.services.dataset_service.S3DatasetClient', s3_client) s3_client().get_bucket_encryption.return_value = ('aws:kms', mocked_key_id) yield s3_client @@ -34,7 +34,7 @@ def dataset1( group, ) -> Dataset: kms_client = MagicMock() - module_mocker.patch('dataall.modules.datasets.services.dataset_service.KmsClient', kms_client) + module_mocker.patch('dataall.modules.s3_datasets.services.dataset_service.KmsClient', kms_client) kms_client().get_key_id.return_value = mocked_key_id @@ -166,10 +166,10 @@ def test_update_dataset(dataset1, client, group, group2, module_mocker): @pytest.mark.skipif( - not config.get_property('modules.datasets.features.glue_crawler'), reason='Feature Disabled by Config' + not config.get_property('modules.s3_datasets.features.glue_crawler'), reason='Feature Disabled by Config' ) def test_start_crawler(org_fixture, env_fixture, dataset1, client, group, module_mocker): - module_mocker.patch('dataall.modules.datasets.services.dataset_service.DatasetCrawler', MagicMock()) + module_mocker.patch('dataall.modules.s3_datasets.services.dataset_service.DatasetCrawler', MagicMock()) mutation = """ mutation StartGlueCrawler($datasetUri:String, $input:CrawlerInput){ startGlueCrawler(datasetUri:$datasetUri,input:$input){ diff --git a/tests/modules/datasets/test_dataset_feed.py b/tests/modules/datasets/test_dataset_feed.py index 5f9b85b4b..3236d4634 100644 --- a/tests/modules/datasets/test_dataset_feed.py +++ b/tests/modules/datasets/test_dataset_feed.py @@ -1,5 +1,5 @@ from dataall.modules.feed.api.registry import FeedRegistry -from dataall.modules.datasets.db.dataset_models import DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable def test_dataset_registered(): diff --git a/tests/modules/datasets/test_dataset_glossary.py b/tests/modules/datasets/test_dataset_glossary.py index aaac934b5..543d20f91 100644 --- a/tests/modules/datasets/test_dataset_glossary.py +++ b/tests/modules/datasets/test_dataset_glossary.py @@ -1,7 +1,7 @@ from typing import List from dataall.modules.catalog.db.glossary_models import TermLink -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn from tests.modules.catalog.test_glossary import * diff --git a/tests/modules/datasets/test_dataset_indexers.py b/tests/modules/datasets/test_dataset_indexers.py index 21fd8b3bb..cf45b9819 100644 --- a/tests/modules/datasets/test_dataset_indexers.py +++ b/tests/modules/datasets/test_dataset_indexers.py @@ -1,6 +1,6 @@ -from dataall.modules.datasets.indexers.location_indexer import DatasetLocationIndexer -from dataall.modules.datasets.indexers.table_indexer import DatasetTableIndexer -from dataall.modules.datasets.indexers.dataset_indexer import DatasetIndexer +from dataall.modules.s3_datasets.indexers.location_indexer import DatasetLocationIndexer +from dataall.modules.s3_datasets.indexers.table_indexer import DatasetTableIndexer +from dataall.modules.s3_datasets.indexers.dataset_indexer import DatasetIndexer def test_es_request(): diff --git a/tests/modules/datasets/test_dataset_location.py b/tests/modules/datasets/test_dataset_location.py index 8fc3af77e..9a3e5e73b 100644 --- a/tests/modules/datasets/test_dataset_location.py +++ b/tests/modules/datasets/test_dataset_location.py @@ -2,7 +2,7 @@ import pytest from dataall.base.config import config -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset @pytest.fixture(scope='module') @@ -11,11 +11,11 @@ def dataset1(env_fixture, org_fixture, dataset, group) -> Dataset: @pytest.mark.skipif( - not config.get_property('modules.datasets.features.file_actions'), reason='Feature Disabled by Config' + not config.get_property('modules.s3_datasets.features.file_actions'), reason='Feature Disabled by Config' ) def test_create_location(client, dataset1, user, group, patch_es, module_mocker): mock_client = MagicMock() - module_mocker.patch('dataall.modules.datasets.services.dataset_location_service.S3LocationClient', mock_client) + module_mocker.patch('dataall.modules.s3_datasets.services.dataset_location_service.S3LocationClient', mock_client) response = client.query( """ mutation createDatasetStorageLocation($datasetUri:String!, $input:NewDatasetStorageLocationInput!){ @@ -43,7 +43,7 @@ def test_create_location(client, dataset1, user, group, patch_es, module_mocker) @pytest.mark.skipif( - not config.get_property('modules.datasets.features.file_actions'), reason='Feature Disabled by Config' + not config.get_property('modules.s3_datasets.features.file_actions'), reason='Feature Disabled by Config' ) def test_manage_dataset_location(client, dataset1, user, group): response = client.query( diff --git a/tests/modules/datasets/test_dataset_permissions.py b/tests/modules/datasets/test_dataset_permissions.py index 19c21ad97..46a80cccc 100644 --- a/tests/modules/datasets/test_dataset_permissions.py +++ b/tests/modules/datasets/test_dataset_permissions.py @@ -2,15 +2,15 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.base.db.exceptions import ResourceUnauthorized from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.modules.datasets.services.dataset_permissions import ( +from dataall.modules.s3_datasets.services.dataset_permissions import ( DATASET_WRITE, UPDATE_DATASET, MANAGE_DATASETS, DATASET_READ, ) -from dataall.modules.datasets.services.dataset_service import DatasetService -from dataall.modules.datasets.db.dataset_models import Dataset -from dataall.modules.datasets.services.dataset_permissions import DATASET_TABLE_READ +from dataall.modules.s3_datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.services.dataset_permissions import DATASET_TABLE_READ from tests.core.permissions.test_permission import * from dataall.core.organizations.services.organization_service import OrganizationService diff --git a/tests/modules/datasets/test_dataset_profiling.py b/tests/modules/datasets/test_dataset_profiling.py index e0e5a6430..653ef8321 100644 --- a/tests/modules/datasets/test_dataset_profiling.py +++ b/tests/modules/datasets/test_dataset_profiling.py @@ -2,7 +2,7 @@ import pytest -from dataall.modules.datasets.db.dataset_models import DatasetProfilingRun, Dataset, DatasetTable +from dataall.modules.s3_datasets.db.dataset_models import DatasetProfilingRun, Dataset, DatasetTable @pytest.fixture(scope='module', autouse=True) diff --git a/tests/modules/datasets/test_dataset_resource_found.py b/tests/modules/datasets/test_dataset_resource_found.py index ce9c02c0f..e75fffcb4 100644 --- a/tests/modules/datasets/test_dataset_resource_found.py +++ b/tests/modules/datasets/test_dataset_resource_found.py @@ -1,5 +1,5 @@ -from dataall.modules.datasets.db.dataset_models import Dataset, DatasetLock -from dataall.modules.datasets.services.dataset_permissions import CREATE_DATASET +from dataall.modules.s3_datasets.db.dataset_models import Dataset, DatasetLock +from dataall.modules.s3_datasets.services.dataset_permissions import CREATE_DATASET def get_env(client, env_fixture, group): diff --git a/tests/modules/datasets/test_dataset_stack.py b/tests/modules/datasets/test_dataset_stack.py index 22edbd396..c3e204fe8 100644 --- a/tests/modules/datasets/test_dataset_stack.py +++ b/tests/modules/datasets/test_dataset_stack.py @@ -5,8 +5,8 @@ from aws_cdk import App from dataall.core.environment.db.environment_models import Environment -from dataall.modules.datasets.cdk.dataset_stack import DatasetStack -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.cdk.dataset_stack import DatasetStack +from dataall.modules.s3_datasets.db.dataset_models import Dataset @pytest.fixture(scope='module', autouse=True) @@ -38,15 +38,15 @@ def dataset(db, env_fixture: Environment) -> Dataset: @pytest.fixture(scope='function', autouse=True) def patch_methods(mocker, db, dataset, env_fixture, org_fixture): - mocker.patch('dataall.modules.datasets.cdk.dataset_stack.DatasetStack.get_engine', return_value=db) - mocker.patch('dataall.modules.datasets.cdk.dataset_stack.DatasetStack.get_target', return_value=dataset) + mocker.patch('dataall.modules.s3_datasets.cdk.dataset_stack.DatasetStack.get_engine', return_value=db) + mocker.patch('dataall.modules.s3_datasets.cdk.dataset_stack.DatasetStack.get_target', return_value=dataset) mocker.patch( 'dataall.base.aws.sts.SessionHelper.get_delegation_role_name', return_value='dataall-pivot-role-name-pytest', ) lf_client = MagicMock() mocker.patch( - 'dataall.modules.datasets.cdk.dataset_stack.LakeFormationDatasetClient', + 'dataall.modules.s3_datasets.cdk.dataset_stack.LakeFormationDatasetClient', return_value=lf_client, ) lf_client.return_value.check_existing_lf_registered_location = False diff --git a/tests/modules/datasets/test_dataset_table.py b/tests/modules/datasets/test_dataset_table.py index 50d21e83e..062393350 100644 --- a/tests/modules/datasets/test_dataset_table.py +++ b/tests/modules/datasets/test_dataset_table.py @@ -1,5 +1,5 @@ -from dataall.modules.datasets.services.dataset_table_service import DatasetTableService -from dataall.modules.datasets.db.dataset_models import DatasetTableColumn, DatasetTable, Dataset +from dataall.modules.s3_datasets.services.dataset_table_service import DatasetTableService +from dataall.modules.s3_datasets.db.dataset_models import DatasetTableColumn, DatasetTable, Dataset def test_add_tables(table, dataset_fixture, db): diff --git a/tests/modules/datasets/test_environment_stack_with_dataset.py b/tests/modules/datasets/test_environment_stack_with_dataset.py index f96f808f3..174895438 100644 --- a/tests/modules/datasets/test_environment_stack_with_dataset.py +++ b/tests/modules/datasets/test_environment_stack_with_dataset.py @@ -4,7 +4,7 @@ from dataall.core.environment.cdk.environment_stack import EnvironmentSetup from dataall.core.environment.db.environment_models import EnvironmentGroup -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.db.dataset_models import Dataset @pytest.fixture(scope='function', autouse=True) diff --git a/tests/modules/datasets/test_glue_profiler.py b/tests/modules/datasets/test_glue_profiler.py index 838ecac9f..1d0242983 100644 --- a/tests/modules/datasets/test_glue_profiler.py +++ b/tests/modules/datasets/test_glue_profiler.py @@ -1,4 +1,4 @@ -from dataall.modules.datasets.cdk.dataset_glue_profiler_extension import DatasetGlueProfilerExtension +from dataall.modules.s3_datasets.cdk.dataset_glue_profiler_extension import DatasetGlueProfilerExtension from pathlib import Path diff --git a/tests/modules/datasets/test_import_dataset_check_unit.py b/tests/modules/datasets/test_import_dataset_check_unit.py index 402bdbd60..9b4718621 100644 --- a/tests/modules/datasets/test_import_dataset_check_unit.py +++ b/tests/modules/datasets/test_import_dataset_check_unit.py @@ -4,8 +4,8 @@ import pytest from dataall.base.db.exceptions import RequiredParameter, InvalidInput, UnauthorizedOperation, AWSResourceNotFound -from dataall.modules.datasets.services.dataset_service import DatasetService -from dataall.modules.datasets.db.dataset_models import Dataset +from dataall.modules.s3_datasets.services.dataset_service import DatasetService +from dataall.modules.s3_datasets.db.dataset_models import Dataset def test_s3_managed_bucket_import(mock_aws_client): diff --git a/tests/modules/datasets/test_share.py b/tests/modules/datasets/test_share.py index dbf621956..99ff2b8b0 100644 --- a/tests/modules/datasets/test_share.py +++ b/tests/modules/datasets/test_share.py @@ -22,7 +22,7 @@ ShareObjectSM, ) from dataall.modules.dataset_sharing.services.share_object_service import ShareObjectService -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset @pytest.fixture(scope='function') From aa992579be10bdfbbed07554a73d059bf19e2791 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 6 May 2024 14:52:27 +0200 Subject: [PATCH 02/29] Rename datasets to s3_datasets --- backend/dataall/modules/dataset_sharing/__init__.py | 4 ++-- backend/dataall/modules/worksheets/__init__.py | 4 ++-- ...move_dataset_table_read_permissions_from_env_admins.py | 2 +- frontend/src/modules/Catalog/views/Catalog.js | 8 ++++---- frontend/src/modules/Datasets/views/DatasetCreateForm.js | 8 ++++---- frontend/src/modules/Datasets/views/DatasetEditForm.js | 8 ++++---- frontend/src/modules/Datasets/views/DatasetImportForm.js | 8 ++++---- 7 files changed, 21 insertions(+), 21 deletions(-) diff --git a/backend/dataall/modules/dataset_sharing/__init__.py b/backend/dataall/modules/dataset_sharing/__init__.py index 4364702cc..9912b1849 100644 --- a/backend/dataall/modules/dataset_sharing/__init__.py +++ b/backend/dataall/modules/dataset_sharing/__init__.py @@ -17,7 +17,7 @@ def is_supported(modes: Set[ImportMode]) -> bool: @staticmethod def depends_on() -> List[Type['ModuleInterface']]: from dataall.modules.notifications import NotificationsModuleInterface - from dataall.modules.datasets import DatasetApiModuleInterface + from dataall.modules.s3_datasets import DatasetApiModuleInterface return [DatasetApiModuleInterface, NotificationsModuleInterface] @@ -42,7 +42,7 @@ def is_supported(modes: List[ImportMode]): @staticmethod def depends_on() -> List[Type['ModuleInterface']]: from dataall.modules.notifications import NotificationsModuleInterface - from dataall.modules.datasets import DatasetAsyncHandlersModuleInterface + from dataall.modules.s3_datasets import DatasetAsyncHandlersModuleInterface return [DatasetAsyncHandlersModuleInterface, NotificationsModuleInterface] diff --git a/backend/dataall/modules/worksheets/__init__.py b/backend/dataall/modules/worksheets/__init__.py index 94fdb048a..7ff91ad6c 100644 --- a/backend/dataall/modules/worksheets/__init__.py +++ b/backend/dataall/modules/worksheets/__init__.py @@ -26,7 +26,7 @@ def __init__(self): @staticmethod def depends_on() -> List[Type['ModuleInterface']]: - from dataall.modules.datasets import DatasetApiModuleInterface + from dataall.modules.s3_datasets import DatasetApiModuleInterface return [DatasetApiModuleInterface] @@ -45,6 +45,6 @@ def __init__(self): @staticmethod def depends_on() -> List[Type['ModuleInterface']]: - from dataall.modules.datasets import DatasetCdkModuleInterface + from dataall.modules.s3_datasets import DatasetCdkModuleInterface return [DatasetCdkModuleInterface] diff --git a/backend/migrations/versions/458572580709_remove_dataset_table_read_permissions_from_env_admins.py b/backend/migrations/versions/458572580709_remove_dataset_table_read_permissions_from_env_admins.py index c63196db3..3df8a26a1 100644 --- a/backend/migrations/versions/458572580709_remove_dataset_table_read_permissions_from_env_admins.py +++ b/backend/migrations/versions/458572580709_remove_dataset_table_read_permissions_from_env_admins.py @@ -13,7 +13,7 @@ from dataall.core.environment.services.environment_service import EnvironmentService from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService -from dataall.modules.datasets.db.dataset_models import DatasetTable, Dataset +from dataall.modules.s3_datasets.db.dataset_models import DatasetTable, Dataset from dataall.modules.dataset_sharing.db.share_object_models import ShareObject, ShareObjectItem from dataall.modules.dataset_sharing.services.dataset_sharing_enums import ShareItemStatus, ShareableType, PrincipalType diff --git a/frontend/src/modules/Catalog/views/Catalog.js b/frontend/src/modules/Catalog/views/Catalog.js index b9664c004..69d147bd1 100644 --- a/frontend/src/modules/Catalog/views/Catalog.js +++ b/frontend/src/modules/Catalog/views/Catalog.js @@ -174,9 +174,9 @@ const Catalog = () => { const [openMenu, setOpenMenu] = useState(false); const dataFieldList = ['label', 'name', 'description', 'region', 'tags']; - if (config.modules.datasets.features.topics_dropdown === true) + if (config.modules.s3_datasets.features.topics_dropdown === true) dataFieldList.push('topics'); - if (config.modules.datasets.features.confidentiality_dropdown === true) + if (config.modules.s3_datasets.features.confidentiality_dropdown === true) dataFieldList.push('classification'); const filterItemsInit = [ @@ -200,14 +200,14 @@ const Catalog = () => { } ]; - if (config.modules.datasets.features.topics_dropdown === true) + if (config.modules.s3_datasets.features.topics_dropdown === true) filterItemsInit.push({ title: 'Topics', dataField: 'topics', componentId: 'TopicSensor', filterLabel: 'Topics' }); - if (config.modules.datasets.features.confidentiality_dropdown === true) + if (config.modules.s3_datasets.features.confidentiality_dropdown === true) filterItemsInit.push({ title: 'Classification', dataField: 'classification', diff --git a/frontend/src/modules/Datasets/views/DatasetCreateForm.js b/frontend/src/modules/Datasets/views/DatasetCreateForm.js index b5485fae5..d1285d43b 100644 --- a/frontend/src/modules/Datasets/views/DatasetCreateForm.js +++ b/frontend/src/modules/Datasets/views/DatasetCreateForm.js @@ -51,10 +51,10 @@ const DatasetCreateForm = (props) => { const [groupOptions, setGroupOptions] = useState([]); const [environmentOptions, setEnvironmentOptions] = useState([]); const [confidentialityOptions] = useState( - config.modules.datasets.features.confidentiality_dropdown === true && - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.confidentiality_dropdown === true && + config.modules.s3_datasets.features.custom_confidentiality_mapping ? Object.keys( - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.custom_confidentiality_mapping ) : ConfidentialityList ); @@ -396,7 +396,7 @@ const DatasetCreateForm = (props) => { - {config.modules.datasets.features + {config.modules.s3_datasets.features .auto_approval_for_confidentiality_level[ values.confidentiality ] === true && ( diff --git a/frontend/src/modules/Datasets/views/DatasetEditForm.js b/frontend/src/modules/Datasets/views/DatasetEditForm.js index c778f00d6..80e9dc92e 100644 --- a/frontend/src/modules/Datasets/views/DatasetEditForm.js +++ b/frontend/src/modules/Datasets/views/DatasetEditForm.js @@ -55,10 +55,10 @@ const DatasetEditForm = (props) => { const [selectableTerms, setSelectableTerms] = useState([]); const [tableTerms, setTableTerms] = useState([]); const [confidentialityOptions] = useState( - config.modules.datasets.features.confidentiality_dropdown === true && - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.confidentiality_dropdown === true && + config.modules.s3_datasets.features.custom_confidentiality_mapping ? Object.keys( - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.custom_confidentiality_mapping ) : ConfidentialityList ); @@ -482,7 +482,7 @@ const DatasetEditForm = (props) => { - {config.modules.datasets.features + {config.modules.s3_datasets.features .auto_approval_for_confidentiality_level[ values.confidentiality ] === true && ( diff --git a/frontend/src/modules/Datasets/views/DatasetImportForm.js b/frontend/src/modules/Datasets/views/DatasetImportForm.js index afe64ac0c..a82e04774 100644 --- a/frontend/src/modules/Datasets/views/DatasetImportForm.js +++ b/frontend/src/modules/Datasets/views/DatasetImportForm.js @@ -51,10 +51,10 @@ const DatasetImportForm = (props) => { const [groupOptions, setGroupOptions] = useState([]); const [environmentOptions, setEnvironmentOptions] = useState([]); const [confidentialityOptions] = useState( - config.modules.datasets.features.confidentiality_dropdown === true && - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.confidentiality_dropdown === true && + config.modules.s3_datasets.features.custom_confidentiality_mapping ? Object.keys( - config.modules.datasets.features.custom_confidentiality_mapping + config.modules.s3_datasets.features.custom_confidentiality_mapping ) : ConfidentialityList ); @@ -409,7 +409,7 @@ const DatasetImportForm = (props) => { - {config.modules.datasets.features + {config.modules.s3_datasets.features .auto_approval_for_confidentiality_level[ values.confidentiality ] === true && ( From 8ea5ca234dd51d4ee78216604fd208a34c789786 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 6 May 2024 14:56:45 +0200 Subject: [PATCH 03/29] Rename datasets to s3_datasets --- .../services/share_notification_service.py | 4 +++- .../dataall/modules/s3_datasets/api/profiling/queries.py | 5 ++++- config.json | 2 +- tests/modules/datasets/conftest.py | 4 +++- .../datasets/tasks/test_dataset_catalog_indexer.py | 8 ++++++-- 5 files changed, 17 insertions(+), 6 deletions(-) diff --git a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py index 38fe4f411..5f5410122 100644 --- a/backend/dataall/modules/dataset_sharing/services/share_notification_service.py +++ b/backend/dataall/modules/dataset_sharing/services/share_notification_service.py @@ -139,7 +139,9 @@ def _create_notification_task(self, subject, msg): - dataset.stewards - share.owner (person that opened the request) OR share.groupUri (if group_notifications=true) """ - share_notification_config = config.get_property('modules.s3_datasets.features.share_notifications', default=None) + share_notification_config = config.get_property( + 'modules.s3_datasets.features.share_notifications', default=None + ) if share_notification_config: for share_notification_config_type in share_notification_config.keys(): n_config = share_notification_config[share_notification_config_type] diff --git a/backend/dataall/modules/s3_datasets/api/profiling/queries.py b/backend/dataall/modules/s3_datasets/api/profiling/queries.py index 228dae28f..d7093a81e 100644 --- a/backend/dataall/modules/s3_datasets/api/profiling/queries.py +++ b/backend/dataall/modules/s3_datasets/api/profiling/queries.py @@ -1,5 +1,8 @@ from dataall.base.api import gql -from dataall.modules.s3_datasets.api.profiling.resolvers import list_table_profiling_runs, get_dataset_table_profiling_run +from dataall.modules.s3_datasets.api.profiling.resolvers import ( + list_table_profiling_runs, + get_dataset_table_profiling_run, +) listDatasetTableProfilingRuns = gql.QueryField( name='listDatasetTableProfilingRuns', diff --git a/config.json b/config.json index 5e32ed272..2e7ca12f8 100644 --- a/config.json +++ b/config.json @@ -9,7 +9,7 @@ "datapipelines": { "active": true }, - "datasets": { + "s3_datasets": { "active": true, "features": { "file_uploads": true, diff --git a/tests/modules/datasets/conftest.py b/tests/modules/datasets/conftest.py index 55b1747ce..4cc1902ce 100644 --- a/tests/modules/datasets/conftest.py +++ b/tests/modules/datasets/conftest.py @@ -25,7 +25,9 @@ def patch_dataset_methods(module_mocker): ) s3_mock_client = MagicMock() glue_mock_client = MagicMock() - module_mocker.patch('dataall.modules.s3_datasets.services.dataset_profiling_service.S3ProfilerClient', s3_mock_client) + module_mocker.patch( + 'dataall.modules.s3_datasets.services.dataset_profiling_service.S3ProfilerClient', s3_mock_client + ) module_mocker.patch( 'dataall.modules.s3_datasets.services.dataset_profiling_service.GlueDatasetProfilerClient', glue_mock_client ) diff --git a/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py b/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py index 4414c9abf..5a1a975b0 100644 --- a/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py +++ b/tests/modules/datasets/tasks/test_dataset_catalog_indexer.py @@ -48,8 +48,12 @@ def table(org, env, db, sync_dataset): def test_catalog_indexer(db, org, env, sync_dataset, table, mocker): - mocker.patch('dataall.modules.s3_datasets.indexers.table_indexer.DatasetTableIndexer.upsert_all', return_value=[table]) - mocker.patch('dataall.modules.s3_datasets.indexers.dataset_indexer.DatasetIndexer.upsert', return_value=sync_dataset) + mocker.patch( + 'dataall.modules.s3_datasets.indexers.table_indexer.DatasetTableIndexer.upsert_all', return_value=[table] + ) + mocker.patch( + 'dataall.modules.s3_datasets.indexers.dataset_indexer.DatasetIndexer.upsert', return_value=sync_dataset + ) indexed_objects_counter = index_objects(engine=db) # Count should be One table + One Dataset = 2 assert indexed_objects_counter == 2 From 2ad4a2e65035053f83e6302766ce9cd9b873e980 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 6 May 2024 15:42:40 +0200 Subject: [PATCH 04/29] Fix references to config in frontend --- frontend/src/modules/Datasets/components/DatasetData.js | 2 +- frontend/src/modules/Datasets/components/DatasetGovernance.js | 4 ++-- frontend/src/modules/Datasets/components/DatasetTables.js | 2 +- frontend/src/modules/Datasets/views/DatasetCreateForm.js | 2 +- frontend/src/modules/Datasets/views/DatasetEditForm.js | 2 +- frontend/src/modules/Datasets/views/DatasetImportForm.js | 2 +- frontend/src/modules/Datasets/views/DatasetView.js | 2 +- frontend/src/modules/Folders/views/FolderView.js | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/frontend/src/modules/Datasets/components/DatasetData.js b/frontend/src/modules/Datasets/components/DatasetData.js index ab2318654..abef82756 100644 --- a/frontend/src/modules/Datasets/components/DatasetData.js +++ b/frontend/src/modules/Datasets/components/DatasetData.js @@ -13,7 +13,7 @@ export const DatasetData = (props) => { - {isFeatureEnabled('datasets', 'file_actions') && ( + {isFeatureEnabled('s3_datasets', 'file_actions') && ( diff --git a/frontend/src/modules/Datasets/components/DatasetGovernance.js b/frontend/src/modules/Datasets/components/DatasetGovernance.js index f4b1ea6cb..a32e2b285 100644 --- a/frontend/src/modules/Datasets/components/DatasetGovernance.js +++ b/frontend/src/modules/Datasets/components/DatasetGovernance.js @@ -49,7 +49,7 @@ export const DatasetGovernance = (props) => { - {isFeatureEnabled('datasets', 'confidentiality_dropdown') && ( + {isFeatureEnabled('s3_datasets', 'confidentiality_dropdown') && ( Classification @@ -59,7 +59,7 @@ export const DatasetGovernance = (props) => { )} - {isFeatureEnabled('datasets', 'topics_dropdown') && ( + {isFeatureEnabled('s3_datasets', 'topics_dropdown') && ( Topics diff --git a/frontend/src/modules/Datasets/components/DatasetTables.js b/frontend/src/modules/Datasets/components/DatasetTables.js index ce8d48934..0d427610b 100644 --- a/frontend/src/modules/Datasets/components/DatasetTables.js +++ b/frontend/src/modules/Datasets/components/DatasetTables.js @@ -217,7 +217,7 @@ export const DatasetTables = (props) => { Synchronize - {isFeatureEnabled('datasets', 'glue_crawler') && ( + {isFeatureEnabled('s3_datasets', 'glue_crawler') && ( { )} - {isFeatureEnabled('datasets', 'topics_dropdown') && ( + {isFeatureEnabled('s3_datasets', 'topics_dropdown') && ( { )} - {isFeatureEnabled('datasets', 'topics_dropdown') && ( + {isFeatureEnabled('s3_datasets', 'topics_dropdown') && ( { )} - {isFeatureEnabled('datasets', 'topics_dropdown') && ( + {isFeatureEnabled('s3_datasets', 'topics_dropdown') && ( { > Chat - {isFeatureEnabled('datasets', 'aws_actions') && ( + {isFeatureEnabled('s3_datasets', 'aws_actions') && ( )} - diff --git a/frontend/src/modules/S3_Datasets/services/listDatasets.js b/frontend/src/modules/S3_Datasets/services/listDatasets.js index a56503237..648f936a0 100644 --- a/frontend/src/modules/S3_Datasets/services/listDatasets.js +++ b/frontend/src/modules/S3_Datasets/services/listDatasets.js @@ -37,6 +37,7 @@ export const listDatasets = ({ filter }) => ({ stack { status } + datasetType } } } From f39296fff086142b4a8fc6ded89ed6cb252919ba Mon Sep 17 00:00:00 2001 From: dlpzx Date: Tue, 21 May 2024 14:56:06 +0200 Subject: [PATCH 29/29] Better typing hints --- .../dataall/modules/datasets_base/db/dataset_repositories.py | 5 +++-- .../modules/datasets_base/services/dataset_list_service.py | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/backend/dataall/modules/datasets_base/db/dataset_repositories.py b/backend/dataall/modules/datasets_base/db/dataset_repositories.py index 102dd1a53..15f4d16dd 100644 --- a/backend/dataall/modules/datasets_base/db/dataset_repositories.py +++ b/backend/dataall/modules/datasets_base/db/dataset_repositories.py @@ -1,4 +1,5 @@ import logging +from typing import List from sqlalchemy import and_, or_ from sqlalchemy.orm import Query from dataall.base.db import paginate @@ -41,7 +42,7 @@ class DatasetListRepository: """DAO layer for Listing Datasets in Environments""" @staticmethod - def paginated_all_user_datasets(session, username, groups, all_subqueries, data=None) -> dict: + def paginated_all_user_datasets(session, username, groups, all_subqueries: List[Query], data=None) -> dict: return paginate( query=DatasetListRepository._query_all_user_datasets(session, username, groups, all_subqueries, data), page=data.get('page', 1), @@ -49,7 +50,7 @@ def paginated_all_user_datasets(session, username, groups, all_subqueries, data= ).to_dict() @staticmethod - def _query_all_user_datasets(session, username, groups, all_subqueries, filter) -> Query: + def _query_all_user_datasets(session, username, groups, all_subqueries: List[Query], filter: dict = None) -> Query: query = session.query(DatasetBase).filter( or_( DatasetBase.owner == username, diff --git a/backend/dataall/modules/datasets_base/services/dataset_list_service.py b/backend/dataall/modules/datasets_base/services/dataset_list_service.py index c8d42204c..b4d82f300 100644 --- a/backend/dataall/modules/datasets_base/services/dataset_list_service.py +++ b/backend/dataall/modules/datasets_base/services/dataset_list_service.py @@ -1,4 +1,5 @@ import logging +from sqlalchemy.orm import Query from typing import List from dataall.modules.datasets_base.services.dataset_service_interface import DatasetServiceInterface from dataall.base.context import get_context @@ -15,7 +16,7 @@ def register(cls, interface: DatasetServiceInterface): cls._interfaces.append(interface) @classmethod - def _list_all_user_interface_datasets(cls, session, username, groups) -> List: + def _list_all_user_interface_datasets(cls, session, username, groups) -> List[Query]: """All list_datasets from other modules that need to be appended to the list of datasets""" return [ query