From 1f6682824544415ef4438c33f68fbbeadca5ca0a Mon Sep 17 00:00:00 2001 From: Yuri Liang Date: Fri, 7 Oct 2022 18:39:58 +0800 Subject: [PATCH 01/11] add Cognito Group resolver --- backend/dataall/api/Objects/Group/queries.py | 8 +++++- .../dataall/api/Objects/Group/resolvers.py | 18 +++++++++++++ backend/dataall/api/Objects/Group/schema.py | 7 ++++++ backend/dataall/aws/handlers/cognito.py | 25 +++++++++++++++++++ frontend/src/api/Groups/listCognitoGroups.js | 14 +++++++++++ 5 files changed, 71 insertions(+), 1 deletion(-) create mode 100644 backend/dataall/aws/handlers/cognito.py create mode 100644 frontend/src/api/Groups/listCognitoGroups.js diff --git a/backend/dataall/api/Objects/Group/queries.py b/backend/dataall/api/Objects/Group/queries.py index 4d85f0fb4..0af6a0a7f 100644 --- a/backend/dataall/api/Objects/Group/queries.py +++ b/backend/dataall/api/Objects/Group/queries.py @@ -1,5 +1,5 @@ from ... import gql -from .resolvers import get_group, list_datasets_owned_by_env_group, list_data_items_shared_with_env_group +from .resolvers import get_group, list_datasets_owned_by_env_group, list_data_items_shared_with_env_group, list_cognito_groups getGroup = gql.QueryField( name='getGroup', @@ -33,3 +33,9 @@ type=gql.Ref('EnvironmentPublishedItemSearchResults'), test_scope='Dataset', ) + +listCognitoGroups = gql.QueryField( + name='listCognitoGroups', + type=gql.Ref('CognitoGroup'), + resolver=list_cognito_groups +) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index bc9c97815..f8ce604a1 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -1,7 +1,11 @@ +import os from .... import db from ....db import exceptions from ....db.models import Group from ...constants import * +from ....aws.handlers.parameter_store import ParameterStoreManager +from ....aws.handlers.sts import SessionHelper +from ....aws.handlers.cognito import Cognito def resolve_group_environment_permissions(context, source, environmentUri): @@ -70,3 +74,17 @@ def list_data_items_shared_with_env_group( data=filter, check_perm=True, ) + +def list_cognito_groups(context, source): + current_account = SessionHelper.get_account() + current_region = os.getenv('AWS_REGION', 'eu-west-1') + envname = os.getenv('envname', 'local') + if envname in ['local', 'dkrcompose']: + return ['DAAdministrators'] + parameter_path = f'/dataall/{envname}/cognito/userpool' + user_pool_id = ParameterStoreManager.get_parameter_value(current_account, current_region, parameter_path) + groups = Cognito.list_cognito_groups(current_account, current_region, user_pool_id) + res = [] + for group in groups: + res.append(group['GroupName']) + return res diff --git a/backend/dataall/api/Objects/Group/schema.py b/backend/dataall/api/Objects/Group/schema.py index 624f81db8..75f5350a5 100644 --- a/backend/dataall/api/Objects/Group/schema.py +++ b/backend/dataall/api/Objects/Group/schema.py @@ -46,3 +46,10 @@ gql.Field(name='nodes', type=gql.ArrayType(Group)), ], ) + +CognitoGroup = gql.ObjectType( + name='CognitoGroup', + fields=[ + gql.Field(name='groupName', type=gql.String), + ], +) diff --git a/backend/dataall/aws/handlers/cognito.py b/backend/dataall/aws/handlers/cognito.py new file mode 100644 index 000000000..860bf9f61 --- /dev/null +++ b/backend/dataall/aws/handlers/cognito.py @@ -0,0 +1,25 @@ +import logging + +from .sts import SessionHelper + + +log = logging.getLogger(__name__) + + +class Cognito: + @staticmethod + def client(account_id: str, region_name: str, client_type: str): + session = SessionHelper.remote_session(account_id) + return session.client(client_type, region_name=region_name) + + @staticmethod + def list_cognito_groups(account_id: str, region: str, user_pool_id: str): + try: + cognitoCli = Cognito.client(account_id, region, "cognito-idp") + response = cognitoCli.list_groups(UsePoolId=user_pool_id) + except Exception as e: + log.error( + f'Failed to list groups of user pool {user_pool_id} due to {e}' + ) + else: + return response['Groups'] diff --git a/frontend/src/api/Groups/listCognitoGroups.js b/frontend/src/api/Groups/listCognitoGroups.js new file mode 100644 index 000000000..6188fbb95 --- /dev/null +++ b/frontend/src/api/Groups/listCognitoGroups.js @@ -0,0 +1,14 @@ +import { gql } from 'apollo-boost'; + +const listCognitoGroups = () => ({ + variables: { + }, + query: gql` + query listCognitoGroups() { + listCognitoGroups() { + } + } + ` +}); + +export default listCognitoGroups; From e5fc455a6a35f790a51289e588c9e17c84d50952 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Sun, 9 Oct 2022 09:04:16 +0200 Subject: [PATCH 02/11] Modified output to array of groups and list of dictionaries - backend --- backend/dataall/api/Objects/Group/queries.py | 2 +- backend/dataall/api/Objects/Group/resolvers.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/backend/dataall/api/Objects/Group/queries.py b/backend/dataall/api/Objects/Group/queries.py index 0af6a0a7f..6ad2c1ddf 100644 --- a/backend/dataall/api/Objects/Group/queries.py +++ b/backend/dataall/api/Objects/Group/queries.py @@ -36,6 +36,6 @@ listCognitoGroups = gql.QueryField( name='listCognitoGroups', - type=gql.Ref('CognitoGroup'), + type=gql.ArrayType(gql.Ref('CognitoGroup')), resolver=list_cognito_groups ) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index f8ce604a1..7c0221751 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -86,5 +86,6 @@ def list_cognito_groups(context, source): groups = Cognito.list_cognito_groups(current_account, current_region, user_pool_id) res = [] for group in groups: - res.append(group['GroupName']) + res.append({"groupName": group['GroupName']}) + return res From 12198f6b59ef79038ba26f7e0f78377c517dd2db Mon Sep 17 00:00:00 2001 From: dlpzx Date: Sun, 9 Oct 2022 09:08:21 +0200 Subject: [PATCH 03/11] api call in frontend view --- .../src/views/Environments/EnvironmentTeamInviteForm.js | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js index a9082d583..35663ea05 100644 --- a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js +++ b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js @@ -28,6 +28,7 @@ import useClient from '../../hooks/useClient'; import listEnvironmentGroupInvitationPermissions from '../../api/Environment/listEnvironmentPermissions'; import inviteGroupOnEnvironment from '../../api/Environment/inviteGroup'; import listEnvironmentNotInvitedGroups from '../../api/Environment/listNotInvitedGroups'; +import listCognitoGroups from '../../api/Groups/listCognitoGroups'; const EnvironmentTeamInviteForm = (props) => { const { environment, onClose, open, reloadTeams, ...other } = props; @@ -45,16 +46,16 @@ const EnvironmentTeamInviteForm = (props) => { try { setLoadingGroups(true); const response = await client.query( - listEnvironmentNotInvitedGroups({ + listCognitoGroups({ environmentUri: environment.environmentUri }) ); if (!response.errors) { setGroupOptions( - response.data.listEnvironmentNotInvitedGroups.nodes.map((g) => ({ + response.data.listCognitoGroups.map((g) => ({ ...g, - value: g.groupUri, - label: g.groupUri + value: g.groupName, + label: g.groupName })) ); } else { From bf2a7c04185a3c31bccbef5e13be1e2f79545a7f Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 10 Oct 2022 08:50:17 +0200 Subject: [PATCH 04/11] api call in frontend view + fix input output in resolver local + fix api src definition --- backend/dataall/api/Objects/Group/resolvers.py | 2 +- frontend/src/api/Groups/listCognitoGroups.js | 7 +++---- .../src/views/Environments/EnvironmentTeamInviteForm.js | 8 +++----- 3 files changed, 7 insertions(+), 10 deletions(-) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index 7c0221751..035ae61cd 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -80,7 +80,7 @@ def list_cognito_groups(context, source): current_region = os.getenv('AWS_REGION', 'eu-west-1') envname = os.getenv('envname', 'local') if envname in ['local', 'dkrcompose']: - return ['DAAdministrators'] + return [{"groupName": 'DAAdministrators'}, {"groupName": 'Engineers'}, {"groupName": 'Scientists'}] parameter_path = f'/dataall/{envname}/cognito/userpool' user_pool_id = ParameterStoreManager.get_parameter_value(current_account, current_region, parameter_path) groups = Cognito.list_cognito_groups(current_account, current_region, user_pool_id) diff --git a/frontend/src/api/Groups/listCognitoGroups.js b/frontend/src/api/Groups/listCognitoGroups.js index 6188fbb95..ede8a6cdf 100644 --- a/frontend/src/api/Groups/listCognitoGroups.js +++ b/frontend/src/api/Groups/listCognitoGroups.js @@ -1,11 +1,10 @@ import { gql } from 'apollo-boost'; const listCognitoGroups = () => ({ - variables: { - }, query: gql` - query listCognitoGroups() { - listCognitoGroups() { + query listCognitoGroups { + listCognitoGroups{ + groupName } } ` diff --git a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js index 35663ea05..a4767af15 100644 --- a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js +++ b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js @@ -45,11 +45,9 @@ const EnvironmentTeamInviteForm = (props) => { const fetchGroups = useCallback(async () => { try { setLoadingGroups(true); - const response = await client.query( - listCognitoGroups({ - environmentUri: environment.environmentUri - }) - ); + console.log("fetchgroups") + const response = await client.query(listCognitoGroups()); + console.log(response) if (!response.errors) { setGroupOptions( response.data.listCognitoGroups.map((g) => ({ From 056b545eedfa4e9cc1e9a226e03a0119bcf02d45 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 10 Oct 2022 08:53:19 +0200 Subject: [PATCH 05/11] flake --- backend/dataall/api/Objects/Group/resolvers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index 035ae61cd..d0f7294be 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -75,6 +75,7 @@ def list_data_items_shared_with_env_group( check_perm=True, ) + def list_cognito_groups(context, source): current_account = SessionHelper.get_account() current_region = os.getenv('AWS_REGION', 'eu-west-1') From 3cdcd12777b8bf8d8599ebdcc18310a61b3ab1b8 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Thu, 13 Oct 2022 14:31:10 +0200 Subject: [PATCH 06/11] Added orga views and filter for orga or environment --- backend/dataall/api/Objects/Group/input_types.py | 8 ++++++++ backend/dataall/api/Objects/Group/queries.py | 3 +++ backend/dataall/api/Objects/Group/resolvers.py | 5 ++++- frontend/src/api/Groups/listCognitoGroups.js | 13 ++++++++++--- .../views/Environments/EnvironmentTeamInviteForm.js | 5 +---- .../Organizations/OrganizationTeamInviteForm.js | 10 +++------- 6 files changed, 29 insertions(+), 15 deletions(-) diff --git a/backend/dataall/api/Objects/Group/input_types.py b/backend/dataall/api/Objects/Group/input_types.py index 6ba08c2f0..9cccb014c 100644 --- a/backend/dataall/api/Objects/Group/input_types.py +++ b/backend/dataall/api/Objects/Group/input_types.py @@ -8,3 +8,11 @@ gql.Argument(name='pageSize', type=gql.Integer), ], ) + +CognitoGroupFilter = gql.InputType( + name='CognitoGroupFilter', + arguments=[ + gql.Argument(name='type', type=gql.String), + gql.Argument(name='uri', type=gql.String), + ], +) diff --git a/backend/dataall/api/Objects/Group/queries.py b/backend/dataall/api/Objects/Group/queries.py index 6ad2c1ddf..5cbf484ff 100644 --- a/backend/dataall/api/Objects/Group/queries.py +++ b/backend/dataall/api/Objects/Group/queries.py @@ -36,6 +36,9 @@ listCognitoGroups = gql.QueryField( name='listCognitoGroups', + args=[ + gql.Argument(name='filter', type=gql.Ref('CognitoGroupFilter')), + ], type=gql.ArrayType(gql.Ref('CognitoGroup')), resolver=list_cognito_groups ) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index d0f7294be..b47e0ea33 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -76,7 +76,10 @@ def list_data_items_shared_with_env_group( ) -def list_cognito_groups(context, source): +def list_cognito_groups(context, source, filter: dict = None): + # filter: + # filter.get("type") = 'organization' or 'environment' + # filter.get("uri") = 'organizationUri' or 'environmentUri' correspondingly current_account = SessionHelper.get_account() current_region = os.getenv('AWS_REGION', 'eu-west-1') envname = os.getenv('envname', 'local') diff --git a/frontend/src/api/Groups/listCognitoGroups.js b/frontend/src/api/Groups/listCognitoGroups.js index ede8a6cdf..1128849b1 100644 --- a/frontend/src/api/Groups/listCognitoGroups.js +++ b/frontend/src/api/Groups/listCognitoGroups.js @@ -1,9 +1,16 @@ import { gql } from 'apollo-boost'; -const listCognitoGroups = () => ({ +const listCognitoGroups ({ filter }) => ({ + variables: { + filter + }, query: gql` - query listCognitoGroups { - listCognitoGroups{ + query listCognitoGroups ( + $filter: CognitoGroupFilter + ) { + listCognitoGroups ( + filter: $filter + ){ groupName } } diff --git a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js index a4767af15..0191c3c65 100644 --- a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js +++ b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js @@ -27,7 +27,6 @@ import { useDispatch } from '../../store'; import useClient from '../../hooks/useClient'; import listEnvironmentGroupInvitationPermissions from '../../api/Environment/listEnvironmentPermissions'; import inviteGroupOnEnvironment from '../../api/Environment/inviteGroup'; -import listEnvironmentNotInvitedGroups from '../../api/Environment/listNotInvitedGroups'; import listCognitoGroups from '../../api/Groups/listCognitoGroups'; const EnvironmentTeamInviteForm = (props) => { @@ -45,9 +44,7 @@ const EnvironmentTeamInviteForm = (props) => { const fetchGroups = useCallback(async () => { try { setLoadingGroups(true); - console.log("fetchgroups") - const response = await client.query(listCognitoGroups()); - console.log(response) + const response = await client.query(listCognitoGroups({type: "environment", uri: environment.environmentUri})); if (!response.errors) { setGroupOptions( response.data.listCognitoGroups.map((g) => ({ diff --git a/frontend/src/views/Organizations/OrganizationTeamInviteForm.js b/frontend/src/views/Organizations/OrganizationTeamInviteForm.js index 3b93f822a..23ca855dd 100644 --- a/frontend/src/views/Organizations/OrganizationTeamInviteForm.js +++ b/frontend/src/views/Organizations/OrganizationTeamInviteForm.js @@ -25,7 +25,7 @@ import { SET_ERROR } from '../../store/errorReducer'; import { useDispatch } from '../../store'; import useClient from '../../hooks/useClient'; import inviteGroupToOrganization from '../../api/Organization/inviteGroup'; -import listOrganizationNotInvitedGroups from '../../api/Organization/listNotInvitedGroups'; +import listCognitoGroups from '../../api/Groups/listCognitoGroups'; const OrganizationTeamInviteForm = (props) => { const { organization, onClose, open, reloadTeams, ...other } = props; @@ -40,14 +40,10 @@ const OrganizationTeamInviteForm = (props) => { const fetchGroups = useCallback(async () => { try { setLoadingGroups(true); - const response = await client.query( - listOrganizationNotInvitedGroups({ - organizationUri: organization.organizationUri - }) - ); + const response = await client.query(listCognitoGroups({type: "organization", uri: organization.organizationUri})); if (!response.errors) { setGroupOptions( - response.data.listOrganizationNotInvitedGroups.nodes.map((g) => ({ + response.data.listCognitoGroups.map((g) => ({ ...g, value: g.groupUri, label: g.groupUri From 2754739bd90c8b311cbda81c626c739c2b8b5002 Mon Sep 17 00:00:00 2001 From: Yuri Liang Date: Mon, 17 Oct 2022 16:00:55 +0800 Subject: [PATCH 07/11] Show only not invited groups and remove two unused apis --- .../api/Objects/Environment/queries.py | 10 ----- .../api/Objects/Environment/resolvers.py | 16 ------- .../dataall/api/Objects/Group/resolvers.py | 43 +++++++++++++------ .../api/Objects/Organization/queries.py | 10 ----- .../api/Objects/Organization/resolvers.py | 16 ------- backend/dataall/aws/handlers/cognito.py | 2 +- backend/dataall/db/api/environment.py | 19 -------- backend/dataall/db/api/organization.py | 20 --------- deploy/stacks/lambda_api.py | 1 + .../api/Environment/listNotInvitedGroups.js | 30 ------------- frontend/src/api/Groups/listCognitoGroups.js | 2 +- .../api/Organization/listNotInvitedGroups.js | 30 ------------- .../Environments/EnvironmentTeamInviteForm.js | 7 ++- .../OrganizationTeamInviteForm.js | 11 +++-- tests/api/test_environment.py | 40 ----------------- tests/api/test_organization.py | 40 ----------------- 16 files changed, 48 insertions(+), 249 deletions(-) delete mode 100644 frontend/src/api/Environment/listNotInvitedGroups.js delete mode 100644 frontend/src/api/Organization/listNotInvitedGroups.js diff --git a/backend/dataall/api/Objects/Environment/queries.py b/backend/dataall/api/Objects/Environment/queries.py index acc8fefd7..9143cae1b 100644 --- a/backend/dataall/api/Objects/Environment/queries.py +++ b/backend/dataall/api/Objects/Environment/queries.py @@ -116,16 +116,6 @@ resolver=list_environment_invited_groups, ) -listEnvironmentNotInvitedGroups = gql.QueryField( - name='listEnvironmentNotInvitedGroups', - type=gql.Ref('GroupSearchResult'), - args=[ - gql.Argument(name='environmentUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='filter', type=gql.Ref('GroupFilter')), - ], - resolver=list_environment_not_invited_groups, -) - listEnvironmentGroups = gql.QueryField( name='listEnvironmentGroups', type=gql.Ref('GroupSearchResult'), diff --git a/backend/dataall/api/Objects/Environment/resolvers.py b/backend/dataall/api/Objects/Environment/resolvers.py index cb16aa79a..5ef2d7a4d 100644 --- a/backend/dataall/api/Objects/Environment/resolvers.py +++ b/backend/dataall/api/Objects/Environment/resolvers.py @@ -180,22 +180,6 @@ def list_environment_invited_groups( ) -def list_environment_not_invited_groups( - context: Context, source, environmentUri=None, filter=None -): - if filter is None: - filter = {} - with context.engine.scoped_session() as session: - return db.api.Environment.not_environment_groups( - session=session, - username=context.username, - groups=context.groups, - uri=environmentUri, - data=filter, - check_perm=True, - ) - - def list_environment_groups(context: Context, source, environmentUri=None, filter=None): if filter is None: filter = {} diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index b47e0ea33..2eaea725d 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -1,13 +1,14 @@ import os +import logging +import boto3 from .... import db from ....db import exceptions from ....db.models import Group from ...constants import * -from ....aws.handlers.parameter_store import ParameterStoreManager -from ....aws.handlers.sts import SessionHelper -from ....aws.handlers.cognito import Cognito +log = logging.getLogger() + def resolve_group_environment_permissions(context, source, environmentUri): if not source: return None @@ -77,19 +78,37 @@ def list_data_items_shared_with_env_group( def list_cognito_groups(context, source, filter: dict = None): - # filter: - # filter.get("type") = 'organization' or 'environment' - # filter.get("uri") = 'organizationUri' or 'environmentUri' correspondingly - current_account = SessionHelper.get_account() - current_region = os.getenv('AWS_REGION', 'eu-west-1') envname = os.getenv('envname', 'local') if envname in ['local', 'dkrcompose']: return [{"groupName": 'DAAdministrators'}, {"groupName": 'Engineers'}, {"groupName": 'Scientists'}] + current_region = os.getenv('AWS_REGION', 'eu-west-1') parameter_path = f'/dataall/{envname}/cognito/userpool' - user_pool_id = ParameterStoreManager.get_parameter_value(current_account, current_region, parameter_path) - groups = Cognito.list_cognito_groups(current_account, current_region, user_pool_id) + ssm = boto3.client('ssm', region_name=current_region) + cognito = boto3.client('cognito-idp', region_name=current_region) + user_pool_id = ssm.get_parameter(Name=parameter_path)['Parameter']['Value'] + groups = cognito.list_groups(UserPoolId=user_pool_id)['Groups'] + category, category_uri = filter.get("type"), filter.get("uri") + if category and category_uri: + if category == 'environment': + with context.engine.scoped_session() as session: + invited_groups = db.api.Environment.query_all_environment_groups( + session=session, + username=context.username, + groups=context.groups, + uri=category_uri, + filter=None, + ).all() + if category == 'organization': + with context.engine.scoped_session() as session: + organization = db.api.Organization.get_organization_by_uri(session, category_uri) + invited_groups = db.api.Organization.query_organization_groups( + session=session, + uri=organization.organizationUri, + filter=None, + ).all() + invited_group_uris = [item.groupUri for item in invited_groups] res = [] for group in groups: - res.append({"groupName": group['GroupName']}) - + if group['GroupName'] not in invited_group_uris: + res.append({"groupName": group['GroupName']}) return res diff --git a/backend/dataall/api/Objects/Organization/queries.py b/backend/dataall/api/Objects/Organization/queries.py index a9a2453c9..3f47e88b0 100644 --- a/backend/dataall/api/Objects/Organization/queries.py +++ b/backend/dataall/api/Objects/Organization/queries.py @@ -33,16 +33,6 @@ resolver=list_organization_invited_groups, ) -listOrganizationNotInvitedGroups = gql.QueryField( - name='listOrganizationNotInvitedGroups', - type=gql.Ref('GroupSearchResult'), - args=[ - gql.Argument(name='organizationUri', type=gql.NonNullableType(gql.String)), - gql.Argument(name='filter', type=gql.Ref('GroupFilter')), - ], - resolver=list_organization_not_invited_groups, -) - listOrganizationGroups = gql.QueryField( name='listOrganizationGroups', type=gql.Ref('GroupSearchResult'), diff --git a/backend/dataall/api/Objects/Organization/resolvers.py b/backend/dataall/api/Objects/Organization/resolvers.py index c7b55e699..f97f2849c 100644 --- a/backend/dataall/api/Objects/Organization/resolvers.py +++ b/backend/dataall/api/Objects/Organization/resolvers.py @@ -161,22 +161,6 @@ def list_organization_invited_groups( ) -def list_organization_not_invited_groups( - context: Context, source, organizationUri=None, filter=None -): - if filter is None: - filter = {} - with context.engine.scoped_session() as session: - return db.api.Organization.not_organization_groups( - session=session, - username=context.username, - groups=context.groups, - uri=organizationUri, - data=filter, - check_perm=True, - ) - - def list_organization_groups( context: Context, source, organizationUri=None, filter=None ): diff --git a/backend/dataall/aws/handlers/cognito.py b/backend/dataall/aws/handlers/cognito.py index 860bf9f61..eb0c23f62 100644 --- a/backend/dataall/aws/handlers/cognito.py +++ b/backend/dataall/aws/handlers/cognito.py @@ -16,7 +16,7 @@ def client(account_id: str, region_name: str, client_type: str): def list_cognito_groups(account_id: str, region: str, user_pool_id: str): try: cognitoCli = Cognito.client(account_id, region, "cognito-idp") - response = cognitoCli.list_groups(UsePoolId=user_pool_id) + response = cognitoCli.list_groups(UserPoolId=user_pool_id) except Exception as e: log.error( f'Failed to list groups of user pool {user_pool_id} due to {e}' diff --git a/backend/dataall/db/api/environment.py b/backend/dataall/db/api/environment.py index 79954c862..1e7fff7b3 100644 --- a/backend/dataall/db/api/environment.py +++ b/backend/dataall/db/api/environment.py @@ -630,25 +630,6 @@ def list_environment_invited_groups( session, username, groups, uri, data ).all() - @staticmethod - @has_resource_perm(permissions.LIST_ENVIRONMENT_GROUPS) - def not_environment_groups( - session, username, groups, uri, data=None, check_perm=None - ) -> dict: - environment_groups: [] = ( - session.query(models.EnvironmentGroup).filter( - and_( - models.EnvironmentGroup.groupUri.in_(groups), - models.EnvironmentGroup.environmentUri == uri, - ), - ) - ).all() - environment_groups = [g.groupUri for g in environment_groups] - not_invited_groups = [ - {'groupUri': group} for group in groups if group not in environment_groups - ] - return Page(not_invited_groups, 1, 1000, len(not_invited_groups)).to_dict() - @staticmethod def query_environment_datasets(session, username, groups, uri, filter) -> Query: query = session.query(models.Dataset).filter( diff --git a/backend/dataall/db/api/organization.py b/backend/dataall/db/api/organization.py index e1ce8fdec..320519e2e 100644 --- a/backend/dataall/db/api/organization.py +++ b/backend/dataall/db/api/organization.py @@ -374,26 +374,6 @@ def paginated_organization_invited_groups( page_size=data.get('pageSize', 10), ).to_dict() - @staticmethod - @has_tenant_perm(permissions.MANAGE_ORGANIZATIONS) - @has_resource_perm(permissions.GET_ORGANIZATION) - def not_organization_groups( - session, username, groups, uri, data=None, check_perm=False - ) -> dict: - org_groups: [] = ( - session.query(models.OrganizationGroup).filter( - and_( - models.OrganizationGroup.groupUri.in_(groups), - models.OrganizationGroup.organizationUri == uri, - ), - ) - ).all() - org_groups = [g.groupUri for g in org_groups] - not_invited_groups = [ - {'groupUri': group} for group in groups if group not in org_groups - ] - return Page(not_invited_groups, 1, 1000, len(not_invited_groups)).to_dict() - @staticmethod def count_organization_invited_groups(session, uri, group) -> int: groups = ( diff --git a/deploy/stacks/lambda_api.py b/deploy/stacks/lambda_api.py index e562ee376..300397446 100644 --- a/deploy/stacks/lambda_api.py +++ b/deploy/stacks/lambda_api.py @@ -240,6 +240,7 @@ def create_function_role(self, envname, resource_prefix, fn_name): 'xray:GetSamplingRules', 'xray:GetSamplingTargets', 'xray:GetSamplingStatisticSummaries', + 'cognito-idp:ListGroups' ], resources=['*'], ), diff --git a/frontend/src/api/Environment/listNotInvitedGroups.js b/frontend/src/api/Environment/listNotInvitedGroups.js deleted file mode 100644 index 88a33c245..000000000 --- a/frontend/src/api/Environment/listNotInvitedGroups.js +++ /dev/null @@ -1,30 +0,0 @@ -import { gql } from 'apollo-boost'; - -const listEnvironmentNotInvitedGroups = ({ filter, environmentUri }) => ({ - variables: { - environmentUri, - filter - }, - query: gql` - query listEnvironmentNotInvitedGroups( - $filter: GroupFilter - $environmentUri: String - ) { - listEnvironmentNotInvitedGroups( - environmentUri: $environmentUri - filter: $filter - ) { - count - page - pages - hasNext - hasPrevious - nodes { - groupUri - } - } - } - ` -}); - -export default listEnvironmentNotInvitedGroups; diff --git a/frontend/src/api/Groups/listCognitoGroups.js b/frontend/src/api/Groups/listCognitoGroups.js index 1128849b1..49d473a50 100644 --- a/frontend/src/api/Groups/listCognitoGroups.js +++ b/frontend/src/api/Groups/listCognitoGroups.js @@ -1,6 +1,6 @@ import { gql } from 'apollo-boost'; -const listCognitoGroups ({ filter }) => ({ +const listCognitoGroups = ({ filter }) => ({ variables: { filter }, diff --git a/frontend/src/api/Organization/listNotInvitedGroups.js b/frontend/src/api/Organization/listNotInvitedGroups.js deleted file mode 100644 index 72138643e..000000000 --- a/frontend/src/api/Organization/listNotInvitedGroups.js +++ /dev/null @@ -1,30 +0,0 @@ -import { gql } from 'apollo-boost'; - -const listOrganizationNotInvitedGroups = ({ filter, organizationUri }) => ({ - variables: { - organizationUri, - filter - }, - query: gql` - query listOrganizationNotInvitedGroups( - $filter: GroupFilter - $organizationUri: String - ) { - listOrganizationNotInvitedGroups( - organizationUri: $organizationUri - filter: $filter - ) { - count - page - pages - hasNext - hasPrevious - nodes { - groupUri - } - } - } - ` -}); - -export default listOrganizationNotInvitedGroups; diff --git a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js index 0191c3c65..843351082 100644 --- a/frontend/src/views/Environments/EnvironmentTeamInviteForm.js +++ b/frontend/src/views/Environments/EnvironmentTeamInviteForm.js @@ -41,10 +41,15 @@ const EnvironmentTeamInviteForm = (props) => { const [groupOptions, setGroupOptions] = useState([]); const [permissionsError, setPermissionsError] = useState(null); + const filter = { + type: "environment", + uri: environment.environmentUri + } + const fetchGroups = useCallback(async () => { try { setLoadingGroups(true); - const response = await client.query(listCognitoGroups({type: "environment", uri: environment.environmentUri})); + const response = await client.query(listCognitoGroups({ filter })); if (!response.errors) { setGroupOptions( response.data.listCognitoGroups.map((g) => ({ diff --git a/frontend/src/views/Organizations/OrganizationTeamInviteForm.js b/frontend/src/views/Organizations/OrganizationTeamInviteForm.js index 23ca855dd..d0ea4b11b 100644 --- a/frontend/src/views/Organizations/OrganizationTeamInviteForm.js +++ b/frontend/src/views/Organizations/OrganizationTeamInviteForm.js @@ -37,16 +37,21 @@ const OrganizationTeamInviteForm = (props) => { const [loadingGroups, setLoadingGroups] = useState(true); const [groupOptions, setGroupOptions] = useState([]); + const filter = { + type: "organization", + uri: organization.organizationUri + } + const fetchGroups = useCallback(async () => { try { setLoadingGroups(true); - const response = await client.query(listCognitoGroups({type: "organization", uri: organization.organizationUri})); + const response = await client.query(listCognitoGroups({ filter })); if (!response.errors) { setGroupOptions( response.data.listCognitoGroups.map((g) => ({ ...g, - value: g.groupUri, - label: g.groupUri + value: g.groupName, + label: g.groupName })) ); } else { diff --git a/tests/api/test_environment.py b/tests/api/test_environment.py index c4dc64433..e961a445c 100644 --- a/tests/api/test_environment.py +++ b/tests/api/test_environment.py @@ -472,26 +472,6 @@ def test_group_invitation(db, client, env1, org1, group2, user, group3, group, d assert response.data.listEnvironmentInvitedGroups.count == 1 - response = client.query( - """ - query listEnvironmentNotInvitedGroups($environmentUri: String!, $filter:GroupFilter){ - listEnvironmentNotInvitedGroups(environmentUri:$environmentUri, filter:$filter){ - count - nodes{ - groupUri - name - } - } - } - """, - username=user.userName, - groups=[group.name, group2.name, group3.name], - environmentUri=env1.environmentUri, - filter={}, - ) - - assert response.data.listEnvironmentNotInvitedGroups.count == 1 - response = client.query( """ query listEnvironmentGroups($environmentUri: String!, $filter:GroupFilter){ @@ -618,26 +598,6 @@ def test_group_invitation(db, client, env1, org1, group2, user, group3, group, d assert response.data.listEnvironmentInvitedGroups.count == 0 - response = client.query( - """ - query listEnvironmentNotInvitedGroups($environmentUri: String!, $filter:GroupFilter){ - listEnvironmentNotInvitedGroups(environmentUri:$environmentUri, filter:$filter){ - count - nodes{ - groupUri - name - } - } - } - """, - username=user.userName, - groups=[group.name, group2.name, group3.name], - environmentUri=env1.environmentUri, - filter={}, - ) - - assert response.data.listEnvironmentNotInvitedGroups.count == 2 - response = client.query( """ query listEnvironmentGroups($environmentUri: String!, $filter:GroupFilter){ diff --git a/tests/api/test_organization.py b/tests/api/test_organization.py index 87ee65127..74f656278 100644 --- a/tests/api/test_organization.py +++ b/tests/api/test_organization.py @@ -222,26 +222,6 @@ def test_group_invitation( assert response.data.listOrganizationInvitedGroups.count == 1 - response = client.query( - """ - query listOrganizationNotInvitedGroups($organizationUri: String!, $filter:GroupFilter){ - listOrganizationNotInvitedGroups(organizationUri:$organizationUri, filter:$filter){ - count - nodes{ - groupUri - name - } - } - } - """, - username=user.userName, - groups=[group.name, group2.name, group3.name], - organizationUri=org1.organizationUri, - filter={}, - ) - - assert response.data.listOrganizationNotInvitedGroups.count == 1 - response = client.query( """ query listOrganizationGroups($organizationUri: String!, $filter:GroupFilter){ @@ -326,26 +306,6 @@ def test_group_invitation( assert response.data.listOrganizationInvitedGroups.count == 0 - response = client.query( - """ - query listOrganizationNotInvitedGroups($organizationUri: String!, $filter:GroupFilter){ - listOrganizationNotInvitedGroups(organizationUri:$organizationUri, filter:$filter){ - count - nodes{ - groupUri - name - } - } - } - """, - username=user.userName, - groups=[group.name, group2.name, group3.name], - organizationUri=org1.organizationUri, - filter={}, - ) - - assert response.data.listOrganizationNotInvitedGroups.count == 2 - response = client.query( """ query listOrganizationGroups($organizationUri: String!, $filter:GroupFilter){ From 359e15d4ed72156de84a476b91fe3467c29e82e6 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 17 Oct 2022 13:05:21 +0200 Subject: [PATCH 08/11] Cognito list groups in handlers + added integration test for list groups --- backend/dataall/api/Objects/Group/resolvers.py | 10 +++------- backend/dataall/aws/handlers/cognito.py | 12 ++++++++---- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index 2eaea725d..54219a5d1 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -1,14 +1,14 @@ import os import logging -import boto3 from .... import db from ....db import exceptions from ....db.models import Group -from ...constants import * +from ....aws.handlers.cognito import Cognito log = logging.getLogger() + def resolve_group_environment_permissions(context, source, environmentUri): if not source: return None @@ -82,11 +82,7 @@ def list_cognito_groups(context, source, filter: dict = None): if envname in ['local', 'dkrcompose']: return [{"groupName": 'DAAdministrators'}, {"groupName": 'Engineers'}, {"groupName": 'Scientists'}] current_region = os.getenv('AWS_REGION', 'eu-west-1') - parameter_path = f'/dataall/{envname}/cognito/userpool' - ssm = boto3.client('ssm', region_name=current_region) - cognito = boto3.client('cognito-idp', region_name=current_region) - user_pool_id = ssm.get_parameter(Name=parameter_path)['Parameter']['Value'] - groups = cognito.list_groups(UserPoolId=user_pool_id)['Groups'] + groups = Cognito.list_cognito_groups(envname=envname, region=current_region) category, category_uri = filter.get("type"), filter.get("uri") if category and category_uri: if category == 'environment': diff --git a/backend/dataall/aws/handlers/cognito.py b/backend/dataall/aws/handlers/cognito.py index eb0c23f62..e3c9ea7c2 100644 --- a/backend/dataall/aws/handlers/cognito.py +++ b/backend/dataall/aws/handlers/cognito.py @@ -1,4 +1,5 @@ import logging +import boto3 from .sts import SessionHelper @@ -13,13 +14,16 @@ def client(account_id: str, region_name: str, client_type: str): return session.client(client_type, region_name=region_name) @staticmethod - def list_cognito_groups(account_id: str, region: str, user_pool_id: str): + def list_cognito_groups(envname: str, region: str): try: - cognitoCli = Cognito.client(account_id, region, "cognito-idp") - response = cognitoCli.list_groups(UserPoolId=user_pool_id) + parameter_path = f'/dataall/{envname}/cognito/userpool' + ssm = boto3.client('ssm', region_name=region) + user_pool_id = ssm.get_parameter(Name=parameter_path)['Parameter']['Value'] + cognito = boto3.client('cognito-idp', region_name=region) + groups = cognito.list_groups(UserPoolId=user_pool_id)['Groups'] except Exception as e: log.error( f'Failed to list groups of user pool {user_pool_id} due to {e}' ) else: - return response['Groups'] + return groups From 2984336c79eb14e160531d4f939e9ed0b4258f7d Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 17 Oct 2022 13:21:00 +0200 Subject: [PATCH 09/11] added integration test for list groups --- tests/api/test_group.py | 45 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/api/test_group.py diff --git a/tests/api/test_group.py b/tests/api/test_group.py new file mode 100644 index 000000000..27b7f1a70 --- /dev/null +++ b/tests/api/test_group.py @@ -0,0 +1,45 @@ +import pytest + +import dataall +from dataall.db import permissions + + +@pytest.fixture(scope='module', autouse=True) +def org1(org, user, group, tenant): + org1 = org('testorg', user.userName, group.name) + yield org1 + + +@pytest.fixture(scope='module', autouse=True) +def env1(env, org1, user, group, tenant, module_mocker): + module_mocker.patch('requests.post', return_value=True) + module_mocker.patch( + 'dataall.api.Objects.Environment.resolvers.check_environment', return_value=True + ) + env1 = env(org1, 'dev', user.userName, group.name, '111111111111', 'eu-west-1') + yield env1 + + +def test_list_cognito_groups_env(client, env1, group, module_mocker): + module_mocker.patch( + 'dataall.aws.handlers.cognito.Cognito.list_cognito_groups', + return_value=[{"groupName": 'cognitos'}, {"groupName": 'testadmins'}], + ) + response = client.query( + """ + query listCognitoGroups ( + $filter: CognitoGroupFilter + ) { + listCognitoGroups ( + filter: $filter + ){ + groupName + } + } + """, + username='alice', + filter={'type': 'environment', 'uri': env1.environmentUri}, + ) + assert response.data.listCognitoGroups[0].groupName == 'cognitos' + + From e2c7f2847f127b0ea779c9c1f6d06f9ee876ddf7 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 17 Oct 2022 13:36:43 +0200 Subject: [PATCH 10/11] testing of env non-local --- backend/dataall/api/Objects/Group/resolvers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/backend/dataall/api/Objects/Group/resolvers.py b/backend/dataall/api/Objects/Group/resolvers.py index 54219a5d1..c17b052f7 100644 --- a/backend/dataall/api/Objects/Group/resolvers.py +++ b/backend/dataall/api/Objects/Group/resolvers.py @@ -79,8 +79,8 @@ def list_data_items_shared_with_env_group( def list_cognito_groups(context, source, filter: dict = None): envname = os.getenv('envname', 'local') - if envname in ['local', 'dkrcompose']: - return [{"groupName": 'DAAdministrators'}, {"groupName": 'Engineers'}, {"groupName": 'Scientists'}] + if envname in ['dkrcompose']: + return [{"groupName": 'Docker'}] current_region = os.getenv('AWS_REGION', 'eu-west-1') groups = Cognito.list_cognito_groups(envname=envname, region=current_region) category, category_uri = filter.get("type"), filter.get("uri") From b9b839db790d2d976287f5e628e274fce8689077 Mon Sep 17 00:00:00 2001 From: dlpzx Date: Mon, 17 Oct 2022 13:48:18 +0200 Subject: [PATCH 11/11] testing of env non-local --- tests/api/test_group.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/api/test_group.py b/tests/api/test_group.py index 27b7f1a70..7cab78314 100644 --- a/tests/api/test_group.py +++ b/tests/api/test_group.py @@ -23,7 +23,7 @@ def env1(env, org1, user, group, tenant, module_mocker): def test_list_cognito_groups_env(client, env1, group, module_mocker): module_mocker.patch( 'dataall.aws.handlers.cognito.Cognito.list_cognito_groups', - return_value=[{"groupName": 'cognitos'}, {"groupName": 'testadmins'}], + return_value=[{"GroupName": 'cognitos'}, {"GroupName": 'testadmins'}], ) response = client.query( """