Main v2 (#683)

A new PR without conflicts 

By submitting this pull request, I confirm that my contribution is made
under the terms of the Apache 2.0 license.

---------

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Amr Saber <amr.m.saber.mail@gmail.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
Co-authored-by: dlpzx <71252798+dlpzx@users.noreply.github.com>
Co-authored-by: wolanlu <101870655+wolanlu@users.noreply.github.com>
Co-authored-by: dbalintx <132444646+dbalintx@users.noreply.github.com>
Co-authored-by: Noah Paige <69586985+noah-paige@users.noreply.github.com>
Co-authored-by: kukushking <kukushkin.anton@gmail.com>
Co-authored-by: Dariusz Osiennik <osiend@amazon.com>
Co-authored-by: Dennis Goldner <107395339+degoldner@users.noreply.github.com>
Co-authored-by: Abdulrahman Kaitoua <abdulrahman.kaitoua@polimi.it>
Co-authored-by: akaitoua-sa <126820454+akaitoua-sa@users.noreply.github.com>
Co-authored-by: Gezim Musliaj <102723839+gmuslia@users.noreply.github.com>
Co-authored-by: Rick Bernotas <97474536+rbernotas@users.noreply.github.com>
Co-authored-by: David Mutune Kimengu <57294718+kimengu-david@users.noreply.github.com>
Co-authored-by: Mohit Arora <1666133+blitzmohit@users.noreply.github.com>
Co-authored-by: Balint David <dbalintx@amazon.com>
Co-authored-by: chamcca <40579012+chamcca@users.noreply.github.com>
Co-authored-by: Dhruba <117375130+marjet26@users.noreply.github.com>
Co-authored-by: Srinivas Reddy <srinivasreddych@outlook.com>
Co-authored-by: mourya-33 <134511711+mourya-33@users.noreply.github.com>
Co-authored-by: dlpzx <dlpzx@amazon.com>
Co-authored-by: Noah Paige <noahpaig@amazon.com>
Co-authored-by: Mo <136477937+itsmo-amzn@users.noreply.github.com>
Co-authored-by: Raul Cajias <cajias@users.noreply.github.com>
Co-authored-by: Maryam Khidir <maryamolaide95@gmail.com>
Co-authored-by: Jorge Iglesias Garcia <44316552+jorgeig-space@users.noreply.github.com>
Co-authored-by: Jorge Iglesias Garcia <jorgeig@amazon.de>

.coveragerc

@@ -1,2 +1,2 @@
 [run]
-omit = backend/dataall/cdkproxy/assets/*, backend/dataall/aws/*
+omit = backend/dataall/base/cdkproxy/assets/*, backend/dataall/aws/*

.git-blame-ignore-revs

@@ -0,0 +1,5 @@
+# Run this command to always ignore formatting commits in `git blame`
+# git config blame.ignoreRevsFile .git-blame-ignore-revs
+
+# Enforce frontend styling and remove dead code
+fefc45c2cdc4d3107369c4d70210894d098a775c

.github/workflows/coverage.yml

@@ -6,6 +6,7 @@ on:
     branches:
     - main
     - release/*
+    - main-v2
 
 jobs:
   run-tests:

.github/workflows/minimal-security-check.yml

@@ -6,6 +6,7 @@ on:
     branches:
     - main
     - release/*
+    - main-v2
 
 jobs:
   build:

.github/workflows/static-checking.yml

@@ -6,6 +6,7 @@ on:
     branches:
       - main
       - release/*
+      - main-v2
 
 jobs:
   Check:

.github/workflows/validate-db-schema.yml

@@ -6,6 +6,7 @@ on:
     branches:
     - main
     - release/*
+    - main-v2
 
 env:
   envname: local

.gitignore

@@ -44,7 +44,7 @@ cobertura.xml
 /node_modules/
 *venv*
 *pgdata*
-backend/dataall/cdkproxy/cdk.out/
+backend/dataall/base/cdkproxy/cdk.out/
 **.zip
 #Frontend
 /node_modules
@@ -70,5 +70,3 @@ npm-debug.log*
 yarn-debug.log*
 yarn-error.log*
 .idea
-/backend/dataall/cdkproxy/assets/gluedataqualityjob/datahubdq.zip
-/backend/dataall/cdkproxy/assets/glueprofilingjob/datahubdq.zip

Makefile

@@ -28,7 +28,7 @@ install-backend:
 	pip install -r backend/requirements.txt
 
 install-cdkproxy:
-	pip install -r backend/dataall/cdkproxy/requirements.txt
+	pip install -r backend/dataall/base/cdkproxy/requirements.txt
 
 install-tests:
 	pip install -r tests/requirements.txt

backend/api_handler.py

@@ -9,11 +9,15 @@
     graphql_sync,
 )
 
-from dataall.api.Objects import bootstrap as bootstrap_schema, get_executable_schema
-from dataall.aws.handlers.service_handlers import Worker
-from dataall.aws.handlers.sqs import SqsQueue
-from dataall.db import init_permissions, get_engine, api, permissions
-from dataall.searchproxy import connect
+from dataall.base.api import bootstrap as bootstrap_schema, get_executable_schema
+from dataall.core.tasks.service_handlers import Worker
+from dataall.base.aws.sqs import SqsQueue
+from dataall.base.context import set_context, dispose_context, RequestContext
+from dataall.core.permissions.db import save_permissions_with_tenant
+from dataall.core.permissions.db.tenant_policy_repositories import TenantPolicy
+from dataall.base.db import get_engine
+from dataall.core.permissions import permissions
+from dataall.base.loader import load_modules, ImportMode
 
 logger = logging.getLogger()
 logger.setLevel(os.environ.get('LOG_LEVEL', 'INFO'))
@@ -23,14 +27,14 @@
 for name in ['boto3', 's3transfer', 'botocore', 'boto']:
     logging.getLogger(name).setLevel(logging.ERROR)
 
+load_modules(modes={ImportMode.API})
 SCHEMA = bootstrap_schema()
 TYPE_DEFS = gql(SCHEMA.gql(with_directives=False))
 ENVNAME = os.getenv('envname', 'local')
 ENGINE = get_engine(envname=ENVNAME)
-ES = connect(envname=ENVNAME)
 Worker.queue = SqsQueue.send
 
-init_permissions(ENGINE)
+save_permissions_with_tenant(ENGINE)
 
 
 def resolver_adapter(resolver):
@@ -42,7 +46,6 @@ def adapted(obj, info, **kwargs):
                 username=info.context['username'],
                 groups=info.context['groups'],
                 schema=info.context['schema'],
-                cdkproxyurl=info.context['cdkproxyurl'],
             ),
             source=obj or None,
             **kwargs,
@@ -97,7 +100,6 @@ def handler(event, context):
 
     log.info('Lambda Event %s', event)
     log.debug('Env name %s', ENVNAME)
-    log.debug('ElasticSearch %s', ES)
     log.debug('Engine %s', ENGINE.engine.url)
 
     if event['httpMethod'] == 'OPTIONS':
@@ -117,14 +119,14 @@ def handler(event, context):
             groups = get_groups(event['requestContext']['authorizer']['claims'])
             with ENGINE.scoped_session() as session:
                 for group in groups:
-                    policy = api.TenantPolicy.find_tenant_policy(
+                    policy = TenantPolicy.find_tenant_policy(
                         session, group, 'dataall'
                     )
                     if not policy:
                         print(
                             f'No policy found for Team {group}. Attaching TENANT_ALL permissions'
                         )
-                        api.TenantPolicy.attach_group_tenant_policy(
+                        TenantPolicy.attach_group_tenant_policy(
                             session=session,
                             group=group,
                             permissions=permissions.TENANT_ALL,
@@ -135,21 +137,24 @@ def handler(event, context):
             print(f'Error managing groups due to: {e}')
             groups = []
 
+        set_context(RequestContext(ENGINE, username, groups))
+
         app_context = {
             'engine': ENGINE,
-            'es': ES,
             'username': username,
             'groups': groups,
             'schema': SCHEMA,
-            'cdkproxyurl': None,
         }
+
     else:
         raise Exception(f'Could not initialize user context from event {event}')
 
     query = json.loads(event.get('body'))
     success, response = graphql_sync(
         schema=executable_schema, data=query, context_value=app_context
     )
+
+    dispose_context()
     response = json.dumps(response)
 
     log.info('Lambda Response %s', response)

backend/aws_handler.py

@@ -2,8 +2,9 @@
 import logging
 import os
 
-from dataall.aws.handlers.service_handlers import Worker
-from dataall.db import get_engine
+from dataall.core.tasks.service_handlers import Worker
+from dataall.base.db import get_engine
+from dataall.base.loader import load_modules, ImportMode
 
 logger = logging.getLogger()
 logger.setLevel(os.environ.get('LOG_LEVEL'))
@@ -13,6 +14,8 @@
 
 engine = get_engine(envname=ENVNAME)
 
+load_modules(modes={ImportMode.HANDLERS})
+
 
 def handler(event, context=None):
     """Processes  messages received from sqs"""

backend/cdkproxymain.py

@@ -7,9 +7,12 @@
 from botocore.exceptions import ClientError
 from fastapi import FastAPI, BackgroundTasks, status, Response
 
-import dataall.cdkproxy.cdk_cli_wrapper as wrapper
-from dataall.cdkproxy.stacks import StackManager
-from dataall import db
+import dataall.base.cdkproxy.cdk_cli_wrapper as wrapper
+from dataall.base import db
+from dataall.base.loader import load_modules, ImportMode
+from dataall.base.cdkproxy.stacks import StackManager
+from dataall.core.organizations.db.organization_models import Organization
+from dataall.core.stacks.db.stack_models import Stack
 
 print('\n'.join(sys.path))
 
@@ -20,7 +23,7 @@
     f"Application started for envname= `{ENVNAME}` DH_DOCKER_VERSION:{os.environ.get('DH_DOCKER_VERSION')}"
 )
 
-
+load_modules(modes={ImportMode.CDK})
 StackManager.registered_stacks()
 
 
@@ -29,7 +32,7 @@ def connect():
     try:
         engine = db.get_engine(envname=ENVNAME)
         with engine.scoped_session() as session:
-            orgs = session.query(db.models.Organization).all()
+            orgs = session.query(Organization).all()
         return engine
     except Exception as e:
         raise Exception('Connection Error')
@@ -151,7 +154,7 @@ async def create_stack(
 
     for stackid in stack_ids:
         with engine.scoped_session() as session:
-            stack: db.models.Stack = session.query(db.models.Stack).get(stackid)
+            stack: Stack = session.query(Stack).get(stackid)
             if not stack:
                 logger.warning(f'Could not find stack with stackUri `{stackid}`')
                 response.status_code = status.HTTP_302_FOUND
@@ -194,7 +197,7 @@ async def delete_stack(
             'message': f'Failed to connect to database for environment `{ENVNAME}`',
         }
     with engine.scoped_session() as session:
-        stack: db.models.Stack = session.query(db.models.Stack).get(stackid)
+        stack: Stack = session.query(Stack).get(stackid)
         if not stack:
             logger.warning(f'Could not find stack with stackUri `{stackid}`')
             response.status_code = status.HTTP_302_FOUND
@@ -232,7 +235,7 @@ def get_stack(stackid: str, response: Response):
             'message': f'Failed to connect to database for environment `{ENVNAME}`',
         }
     with engine.scoped_session() as session:
-        stack: db.models.Stack = session.query(db.models.Stack).get(stackid)
+        stack: Stack = session.query(Stack).get(stackid)
         if not stack:
             logger.warning(f'Could not find stack with stackUri `{stackid}`')
             response.status_code = status.HTTP_404_NOT_FOUND

backend/dataall/__init__.py

@@ -1 +1,2 @@
-from . import utils, db, aws, api, searchproxy, tasks, version
+from . import core, version
+from .base import utils, db, api