Merge branch 'main' into data-dot-allgh-884-IAM-policy-splitting

# Conflicts:
#	backend/dataall/modules/s3_datasets_shares/services/share_managers/s3_access_point_share_manager.py
#	backend/requirements.txt
#	docker-compose.yaml

.checkov.baseline

@@ -430,12 +430,6 @@
                         "CKV_AWS_111"
                     ]
                 },
-                {
-                    "resource": "AWS::IAM::Policy.dataallmaincdkpipelinePipelineRoleDefaultPolicy98FFDB2A",
-                    "check_ids": [
-                        "CKV_AWS_111"
-                    ]
-                },
                 {
                     "resource": "AWS::Lambda::Function.CustomS3AutoDeleteObjectsCustomResourceProviderHandler9D90184F",
                     "check_ids": [
@@ -502,7 +496,7 @@
                     ]
                 },
                 {
-                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy7BF536CC5",
+                    "resource": "AWS::IAM::ManagedPolicy.dataallanothergroup111111servicespolicy5A19E75CA",
                     "check_ids": [
                         "CKV_AWS_109"
                     ]
@@ -538,7 +532,7 @@
                     ]
                 },
                 {
-                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy7DCF428B2",
+                    "resource": "AWS::IAM::ManagedPolicy.dataalltestadmins111111servicespolicy56D7DC525",
                     "check_ids": [
                         "CKV_AWS_109"
                     ]
@@ -606,25 +600,6 @@
                 }
             ]
         },
-        {
-            "file": "/checkov_pipeline_synth.json",
-            "findings": [
-                {
-                    "resource": "AWS::IAM::Role.PipelineRoleDCFDBB91",
-                    "check_ids": [
-                        "CKV_AWS_107",
-                        "CKV_AWS_108",
-                        "CKV_AWS_111"
-                    ]
-                },
-                {
-                    "resource": "AWS::S3::Bucket.thistableartifactsbucketDB1C8C64",
-                    "check_ids": [
-                        "CKV_AWS_18"
-                    ]
-                }
-            ]
-        },
         {
             "file": "/checkov_smstudio_extension_synth.json",
             "findings": [

.github/workflows/bandit.yml

@@ -28,6 +28,7 @@ jobs:
       - name: Install
         run: |
           python -m pip install --upgrade pip
+          python -m pip install typing-extensions
           python -m pip install bandit
       - name: Bandit
         run: bandit -r -lll -ii .

backend/dataall/base/cdkproxy/requirements.txt

@@ -1,17 +1,12 @@
-aws-cdk-lib==2.99.0
-boto3==1.34.119
-boto3-stubs==1.34.119
-botocore==1.34.119
+aws-cdk-lib==2.160.0
+boto3==1.35.26
+boto3-stubs==1.35.26
 cdk-nag==2.7.2
-constructs==10.0.73
-starlette==0.36.3
-fastapi == 0.109.2
-Flask==2.3.2
+fastapi == 0.115.0
 PyYAML==6.0
 requests==2.32.2
 tabulate==0.8.9
 uvicorn==0.15.0
 werkzeug==3.0.3
-constructs>=10.0.0,<11.0.0
 git-remote-codecommit==1.16
 aws-ddk-core==1.3.0

backend/dataall/base/context.py

@@ -4,7 +4,7 @@
 that in the request scope
 
 The class uses Flask's approach to handle request: ThreadLocal
-That approach should work fine for AWS Lambdas and local server that uses Flask app
+That approach should work fine for AWS Lambdas and local server that uses FastApi app
 """
 
 from dataclasses import dataclass

backend/dataall/base/feature_toggle_checker.py

@@ -2,6 +2,8 @@
 Contains decorators that check if a feature has been enabled or not
 """
 
+from typing import List, Any, Optional, Callable
+
 from dataall.base.config import config
 from dataall.base.utils.decorator_utls import process_func
 
@@ -19,3 +21,35 @@ def decorated(*args, **kwargs):
         return fn_decorator(decorated)
 
     return decorator
+
+
+def is_feature_enabled_for_allowed_values(
+    allowed_values: List[Any],
+    enabled_values: List[Any],
+    default_value: Any,
+    resolve_property: Optional[Callable] = None,
+    config_property: Optional[str] = None,
+):
+    def decorator(f):
+        fn, fn_decorator = process_func(f)
+
+        def decorated(*args, **kwargs):
+            config_property_value = None
+            if config_property is None and resolve_property is None:
+                raise Exception('Config property not provided')
+            if resolve_property:
+                config_property_value = resolve_property(*args, **kwargs)
+            if config_property:
+                config_property_value = config_property
+            value = config.get_property(config_property_value, default_value)
+            if value not in allowed_values:
+                raise Exception(
+                    f'Disabled since incorrect values in config {config_property_value}. Correct config values {allowed_values}'
+                )
+            if value not in enabled_values:
+                raise Exception(f'Disabled by config: {value}. Enable config value(s): {", ".join(enabled_values)}')
+            return fn(*args, **kwargs)
+
+        return fn_decorator(decorated)
+
+    return decorator

backend/dataall/base/utils/expiration_util.py

@@ -9,11 +9,21 @@ class ExpirationUtils:
     def calculate_expiry_date(expirationPeriod, expirySetting):
         currentDate = date.today()
         if expirySetting == Expiration.Quartely.value:
-            quarterlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod * 3 - 1)
+            if currentDate < datetime(currentDate.year, currentDate.month, 15).date():
+                # First half of the month - extend 2.X months
+                quarterlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod * 3 - 1)
+            else:
+                # Second half of the month - extend 3.X months
+                quarterlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod * 3)
             day = calendar.monthrange(quarterlyCalculatedDate.year, quarterlyCalculatedDate.month)[1]
             shareExpiryDate = datetime(quarterlyCalculatedDate.year, quarterlyCalculatedDate.month, day)
         elif expirySetting == Expiration.Monthly.value:
-            monthlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod - 1)
+            if currentDate < datetime(currentDate.year, currentDate.month, 15).date():
+                # First half of the month - extend until end of month
+                monthlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod - 1)
+            else:
+                # Second half of the month - extend until end of next month
+                monthlyCalculatedDate = currentDate + relativedelta(months=expirationPeriod)
             monthEndDay = calendar.monthrange(monthlyCalculatedDate.year, monthlyCalculatedDate.month)[1]
             shareExpiryDate = datetime(monthlyCalculatedDate.year, monthlyCalculatedDate.month, monthEndDay)
         else:

backend/dataall/core/environment/cdk/environment_stack.py

@@ -583,9 +583,17 @@ def create_integration_tests_role(self):
                     's3:PutEncryptionConfiguration',
                     's3:GetObject*',
                     's3:DeleteObject',
+                    's3:DeleteObjectVersion',
                 ],
                 effect=iam.Effect.ALLOW,
-                resources=['arn:aws:s3:::dataalltesting*', 'arn:aws:s3:::dataalltesting*/*'],
+                resources=[
+                    'arn:aws:s3:::dataalltesting*',
+                    'arn:aws:s3:::dataalltesting*/*',
+                    'arn:aws:s3:::dataall-session*',
+                    'arn:aws:s3:::dataall-session*/*',
+                    'arn:aws:s3:::dataall-temp*',
+                    'arn:aws:s3:::dataall-temp*/*',
+                ],
             )
         )
         self.test_role.add_to_policy(
@@ -618,6 +626,8 @@ def create_integration_tests_role(self):
                     'kms:DescribeKey',
                     's3:GetBucketVersioning',
                     's3:List*',
+                    's3:ListAccessPoints',
+                    's3:DeleteAccessPoint',
                 ],
                 effect=iam.Effect.ALLOW,
                 resources=['*'],
@@ -654,3 +664,27 @@ def create_integration_tests_role(self):
                 resources=[f'arn:aws:cloudformation:*:{self.account}:stack/*/*'],
             ),
         )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=[
+                    'iam:GetRole',
+                    'iam:CreateRole',
+                    'iam:DeleteRole',
+                    'iam:PutRolePolicy',
+                    'iam:DeleteRolePolicy',
+                ],
+                effect=iam.Effect.ALLOW,
+                resources=[f'arn:aws:iam::{self.account}:role/dataall-test-*'],
+            ),
+        )
+
+        self.test_role.add_to_policy(
+            iam.PolicyStatement(
+                actions=[
+                    'quicksight:DescribeAccountSubscription',
+                ],
+                effect=iam.Effect.ALLOW,
+                resources=[f'arn:aws:quicksight:*:{self.account}:*'],
+            ),
+        )

backend/dataall/core/environment/cdk/pivot_role_core_policies/s3.py

@@ -17,6 +17,7 @@ def get_statements(self):
                 effect=iam.Effect.ALLOW,
                 actions=[
                     's3:ListAllMyBuckets',
+                    's3:ListAccessPoints',
                     's3:GetBucketLocation',
                     's3:PutBucketTagging',
                     's3:GetEncryptionConfiguration',

backend/dataall/core/environment/db/environment_repositories.py

@@ -310,3 +310,16 @@ def query_all_active_environments(session) -> List[Environment]:
     @staticmethod
     def query_environment_groups(session, uri):
         return session.query(EnvironmentGroup).filter(EnvironmentGroup.environmentUri == uri).all()
+
+    @staticmethod
+    def get_environment_consumption_role_by_name(session, uri, IAMRoleName):
+        return (
+            session.query(ConsumptionRole)
+            .filter(
+                and_(
+                    ConsumptionRole.environmentUri == uri,
+                    ConsumptionRole.IAMRoleName == IAMRoleName,
+                )
+            )
+            .first()
+        )

backend/dataall/core/environment/services/environment_service.py

@@ -729,6 +729,11 @@ def paginated_user_environment_groups(uri, data=None) -> dict:
     def get_all_environment_groups(session, uri, filter) -> Query:
         return EnvironmentRepository.query_all_environment_groups(session, uri, filter)
 
+    @staticmethod
+    def list_all_environment_groups(uri, data=None) -> [str]:
+        with get_context().db_engine.scoped_session() as session:
+            return [g.groupUri for g in EnvironmentRepository.query_all_environment_groups(session, uri, data).all()]
+
     @staticmethod
     @ResourcePolicyService.has_resource_permission(environment_permissions.LIST_ENVIRONMENT_GROUPS)
     def paginated_all_environment_groups(uri, data=None) -> dict:
@@ -1147,3 +1152,9 @@ def resolve_consumption_role_policies(uri, IAMRoleName):
             region=environment.region,
             resource_prefix=environment.resourcePrefix,
         ).get_all_policies()
+
+    @staticmethod
+    @ResourcePolicyService.has_resource_permission(environment_permissions.GET_ENVIRONMENT)
+    def get_consumption_role_by_name(uri, IAMRoleName):
+        with get_context().db_engine.scoped_session() as session:
+            return EnvironmentRepository.get_environment_consumption_role_by_name(session, uri, IAMRoleName)

backend/dataall/core/permissions/services/resource_policy_service.py

@@ -217,6 +217,7 @@ def get_resource_policy_permissions(session, group_uri, resource_uri) -> List[Re
             raise exceptions.RequiredParameter(param_name='group_uri')
         if not resource_uri:
             raise exceptions.RequiredParameter(param_name='resource_uri')
+
         policy = ResourcePolicyRepository.find_resource_policy(
             session=session,
             group_uri=group_uri,

backend/dataall/core/stacks/api/resolvers.py

@@ -6,7 +6,7 @@
 from dataall.base.api.context import Context
 from dataall.core.environment.services.environment_service import EnvironmentService
 from dataall.core.stacks.services.keyvaluetag_service import KeyValueTagService
-from dataall.core.stacks.services.stack_service import StackService
+from dataall.core.stacks.services.stack_service import StackService, map_target_type_to_log_config_path
 from dataall.core.stacks.db.stack_models import Stack
 from dataall.core.stacks.aws.cloudwatch import CloudWatch
 from dataall.base.utils import Parameter
@@ -57,6 +57,16 @@ def resolve_events(context, source: Stack, **kwargs):
     return json.dumps(source.events or {})
 
 
+def resolve_stack_visibility(context, source: Stack, **kwargs):
+    if not source:
+        return False
+    try:
+        return StackService.check_if_user_allowed_view_logs(target_type=source.stack, target_uri=source.targetUri)
+    except Exception as e:
+        log.error(f'Failed to check if the user is allowed to view stack logs due to: {e}')
+        return False
+
+
 def resolve_task_id(context, source: Stack, **kwargs):
     if not source:
         return None

backend/dataall/core/stacks/api/types.py

@@ -6,6 +6,7 @@
     resolve_events,
     resolve_task_id,
     resolve_error,
+    resolve_stack_visibility,
 )
 
 Stack = gql.ObjectType(
@@ -20,13 +21,15 @@
         gql.Field(name='region', type=gql.NonNullableType(gql.String)),
         gql.Field(name='status', type=gql.String),
         gql.Field(name='stackid', type=gql.String),
+        gql.Field(name='updated', type=gql.AWSDateTime),
         gql.Field(name='link', type=gql.String, resolver=resolve_link),
         gql.Field(name='outputs', type=gql.String, resolver=resolve_outputs),
         gql.Field(name='resources', type=gql.String, resolver=resolve_resources),
         gql.Field(name='error', type=gql.String, resolver=resolve_error),
         gql.Field(name='events', type=gql.String, resolver=resolve_events),
         gql.Field(name='EcsTaskArn', type=gql.String),
         gql.Field(name='EcsTaskId', type=gql.String, resolver=resolve_task_id),
+        gql.Field(name='canViewLogs', type=gql.Boolean, resolver=resolve_stack_visibility),
     ],
 )
 

backend/dataall/core/stacks/services/stack_service.py

@@ -3,6 +3,8 @@
 import requests
 import logging
 
+from dataall.base.db import exceptions
+from dataall.base.feature_toggle_checker import is_feature_enabled_for_allowed_values
 from dataall.core.permissions.services.resource_policy_service import ResourcePolicyService
 from dataall.core.stacks.aws.cloudformation import CloudFormation
 from dataall.core.stacks.services.keyvaluetag_service import KeyValueTagService
@@ -44,6 +46,22 @@ def verify_target_type_and_uri(target_type, target_uri):
             raise RequiredParameter('targetType')
 
 
+def map_target_type_to_log_config_path(**kwargs):
+    target_type = kwargs.get('target_type')
+    if target_type == 'environment':
+        return 'core.features.show_stack_logs'
+    elif target_type == 'dataset':
+        return 'modules.s3_datasets.features.show_stack_logs'
+    elif target_type == 'mlstudio':
+        return 'modules.mlstudio.features.show_stack_logs'
+    elif target_type == 'notebooks':
+        return 'modules.notebooks.features.show_stack_logs'
+    elif target_type == 'datapipelines':
+        return 'modules.datapipelines.features.show_stack_logs'
+    else:
+        return 'Invalid Config'
+
+
 class StackService:
     @staticmethod
     def resolve_parent_obj_stack(targetUri: str, environmentUri: str):
@@ -188,6 +206,7 @@ def update_stack_tags(input):
     @staticmethod
     def get_stack_logs(target_uri, target_type):
         context = get_context()
+        StackService.check_if_user_allowed_view_logs(target_type=target_type, target_uri=target_uri)
         StackRequestVerifier.verify_target_type_and_uri(target_uri, target_type)
 
         with context.db_engine.scoped_session() as session:
@@ -213,3 +232,21 @@ def get_stack_logs(target_uri, target_type):
                     | filter @logStream like "{stack.EcsTaskArn.split('/')[-1]}"
                     """
         return query
+
+    @staticmethod
+    @is_feature_enabled_for_allowed_values(
+        allowed_values=['admin-only', 'enabled', 'disabled'],
+        enabled_values=['admin-only', 'enabled'],
+        default_value='enabled',
+        resolve_property=map_target_type_to_log_config_path,
+    )
+    def check_if_user_allowed_view_logs(target_type: str, target_uri: str):
+        context = get_context()
+        config_value = config.get_property(map_target_type_to_log_config_path(target_type=target_type), 'enabled')
+        if config_value == 'admin-only' and 'DAAdministrators' not in context.groups:
+            raise exceptions.ResourceUnauthorized(
+                username=context.username,
+                action='View Stack logs',
+                resource_uri=f'{target_uri} ( Resource type: {target_type} )',
+            )
+        return True

backend/dataall/modules/datapipelines/cdk/blueprints/data_pipeline_blueprint/requirements.txt

@@ -1,3 +1,2 @@
-aws-cdk-lib==2.103.1
-constructs>=10.0.0,<11.0.0
+aws-cdk-lib==2.160.0
 aws-ddk-core==1.3.0