From 7bf493c562dbc57bf28c62e3991a85f00a65bab9 Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Wed, 31 May 2023 17:08:50 +0300 Subject: [PATCH 1/6] lambda manager: send signal before calling function --- cvat/apps/lambda_manager/signals.py | 7 +++++++ cvat/apps/lambda_manager/views.py | 11 ++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) create mode 100644 cvat/apps/lambda_manager/signals.py diff --git a/cvat/apps/lambda_manager/signals.py b/cvat/apps/lambda_manager/signals.py new file mode 100644 index 00000000000..65687630dc4 --- /dev/null +++ b/cvat/apps/lambda_manager/signals.py @@ -0,0 +1,7 @@ +# Copyright (C) 2023 CVAT.ai Corporation +# +# SPDX-License-Identifier: MIT + +from django.dispatch import Signal + +lambda_function_call_signal = Signal() \ No newline at end of file diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index b5a0ca91824..eb044a50b45 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -19,6 +19,7 @@ import numpy as np import requests import rq +from cvat.apps.lambda_manager.signals import lambda_function_call_signal from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError from drf_spectacular.types import OpenApiTypes @@ -27,6 +28,7 @@ inline_serializer) from rest_framework import serializers, status, viewsets from rest_framework.response import Response +from rest_framework.request import Request import cvat.apps.dataset_manager as dm from cvat.apps.engine.frame_provider import FrameProvider @@ -194,7 +196,7 @@ def to_dict(self): return response - def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] = None): + def invoke(self, db_task: Task, request: Request, *, db_job: Optional[Job] = None): try: if db_job is not None and db_job.get_task_id() != db_task.id: raise ValidationError("Job task id does not match task id", @@ -202,7 +204,7 @@ def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] = ) payload = {} - data = {k: v for k,v in data.items() if v is not None} + data = {k: v for k,v in request.data.items() if v is not None} threshold = data.get("threshold") if threshold: payload.update({ "threshold": threshold }) @@ -299,7 +301,10 @@ def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] = .format(self.id, str(err)), code=status.HTTP_400_BAD_REQUEST) + lambda_function_call_signal.send(sender=self, request=request) + response = self.gateway.invoke(self, payload) + response_filtered = [] def check_attr_value(value, func_attr, db_attr): if db_attr is None: @@ -788,7 +793,7 @@ def call(self, request, func_id): gateway = LambdaGateway() lambda_func = gateway.get(func_id) - return lambda_func.invoke(db_task, request.data, db_job=job) + return lambda_func.invoke(db_task, request, db_job=job) @extend_schema(tags=['lambda']) @extend_schema_view( From f154e204a3bf024cb4e5592f1a04df903875b87f Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Wed, 31 May 2023 17:55:16 +0300 Subject: [PATCH 2/6] fix pylint warnings --- cvat/apps/lambda_manager/signals.py | 2 +- cvat/apps/lambda_manager/views.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/cvat/apps/lambda_manager/signals.py b/cvat/apps/lambda_manager/signals.py index 65687630dc4..783e850d427 100644 --- a/cvat/apps/lambda_manager/signals.py +++ b/cvat/apps/lambda_manager/signals.py @@ -4,4 +4,4 @@ from django.dispatch import Signal -lambda_function_call_signal = Signal() \ No newline at end of file +lambda_function_call_signal = Signal() diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index eb044a50b45..3e4ed3a58e8 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -12,7 +12,7 @@ from copy import deepcopy from enum import Enum from functools import wraps -from typing import Any, Dict, Optional +from typing import Optional import datumaro.util.mask_tools as mask_tools import django_rq From 1ab7b7da588a6c2261b1fd5a082d385bb1887aeb Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Thu, 1 Jun 2023 14:58:55 +0300 Subject: [PATCH 3/6] fixes --- cvat/apps/lambda_manager/views.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index 6748c64c610..da6eb8cac2b 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -12,7 +12,7 @@ from copy import deepcopy from enum import Enum from functools import wraps -from typing import Optional +from typing import Any, Dict, Optional import datumaro.util.mask_tools as mask_tools import django_rq @@ -196,7 +196,7 @@ def to_dict(self): return response - def invoke(self, db_task: Task, request: Request, *, db_job: Optional[Job] = None): + def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] = None, request: Optional[Request] = None): try: if db_job is not None and db_job.get_task_id() != db_task.id: raise ValidationError("Job task id does not match task id", @@ -204,7 +204,7 @@ def invoke(self, db_task: Task, request: Request, *, db_job: Optional[Job] = Non ) payload = {} - data = {k: v for k,v in request.data.items() if v is not None} + data = {k: v for k,v in data.items() if v is not None} threshold = data.get("threshold") if threshold: payload.update({ "threshold": threshold }) @@ -301,7 +301,8 @@ def invoke(self, db_task: Task, request: Request, *, db_job: Optional[Job] = Non .format(self.id, str(err)), code=status.HTTP_400_BAD_REQUEST) - lambda_function_call_signal.send(sender=self, request=request) + if request: + lambda_function_call_signal.send(sender=self, request=request) response = self.gateway.invoke(self, payload) @@ -793,7 +794,7 @@ def call(self, request, func_id): gateway = LambdaGateway() lambda_func = gateway.get(func_id) - return lambda_func.invoke(db_task, request, db_job=job) + return lambda_func.invoke(db_task, request.data, db_job=job, request=request) @extend_schema(tags=['lambda']) @extend_schema_view( From 1670f5e87ce2380e254afd89bd92a672481bd068 Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Wed, 7 Jun 2023 12:00:36 +0300 Subject: [PATCH 4/6] fix getting organization for functions permissions --- cvat/apps/iam/permissions.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 11d5c5d624f..183f55a2ddf 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -56,13 +56,13 @@ def get_organization(request, obj): if obj is not None and isinstance(obj, Organization): return obj - if obj: + view = request.parser_context.get('view') + if obj and view.basename not in ("function", "request"): try: organization_id = getattr(obj, 'organization_id') except AttributeError as exc: # Skip initialization of organization for those objects that don't related with organization - view = request.parser_context.get('view') - if view and view.basename in ('user', 'function', 'request'): + if view and view.basename == 'user': return None raise exc @@ -72,8 +72,6 @@ def get_organization(request, obj): except Organization.DoesNotExist: return None - - return request.iam_context['organization'] def get_membership(request, organization): From 306e1a27ac10146a38028fe0a06e1dc8716c8a32 Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Wed, 7 Jun 2023 17:34:03 +0300 Subject: [PATCH 5/6] fix getting org id for objects that doesn't related with organizations --- cvat/apps/iam/permissions.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cvat/apps/iam/permissions.py b/cvat/apps/iam/permissions.py index 183f55a2ddf..f29f6d53ad9 100644 --- a/cvat/apps/iam/permissions.py +++ b/cvat/apps/iam/permissions.py @@ -56,14 +56,14 @@ def get_organization(request, obj): if obj is not None and isinstance(obj, Organization): return obj - view = request.parser_context.get('view') - if obj and view.basename not in ("function", "request"): + if obj: try: organization_id = getattr(obj, 'organization_id') except AttributeError as exc: # Skip initialization of organization for those objects that don't related with organization - if view and view.basename == 'user': - return None + view = request.parser_context.get('view') + if view and view.basename in ('user', 'function', 'request',): + return request.iam_context['organization'] raise exc From 734406eaa9ab2302d8b6f64447da81dfa9948981 Mon Sep 17 00:00:00 2001 From: kirill-sizov Date: Thu, 8 Jun 2023 12:37:43 +0300 Subject: [PATCH 6/6] clarify signal --- cvat/apps/lambda_manager/signals.py | 2 +- cvat/apps/lambda_manager/views.py | 18 +++++++++++++----- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/cvat/apps/lambda_manager/signals.py b/cvat/apps/lambda_manager/signals.py index 783e850d427..8eeafa6f5ae 100644 --- a/cvat/apps/lambda_manager/signals.py +++ b/cvat/apps/lambda_manager/signals.py @@ -4,4 +4,4 @@ from django.dispatch import Signal -lambda_function_call_signal = Signal() +interactive_function_call_signal = Signal() diff --git a/cvat/apps/lambda_manager/views.py b/cvat/apps/lambda_manager/views.py index da6eb8cac2b..f3c5f564b0f 100644 --- a/cvat/apps/lambda_manager/views.py +++ b/cvat/apps/lambda_manager/views.py @@ -19,7 +19,7 @@ import numpy as np import requests import rq -from cvat.apps.lambda_manager.signals import lambda_function_call_signal +from cvat.apps.lambda_manager.signals import interactive_function_call_signal from django.conf import settings from django.core.exceptions import ObjectDoesNotExist, ValidationError from drf_spectacular.types import OpenApiTypes @@ -196,7 +196,15 @@ def to_dict(self): return response - def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] = None, request: Optional[Request] = None): + def invoke( + self, + db_task: Task, + data: Dict[str, Any], + *, + db_job: Optional[Job] = None, + is_interactive: Optional[bool] = False, + request: Optional[Request] = None + ): try: if db_job is not None and db_job.get_task_id() != db_task.id: raise ValidationError("Job task id does not match task id", @@ -301,8 +309,8 @@ def invoke(self, db_task: Task, data: Dict[str, Any], *, db_job: Optional[Job] .format(self.id, str(err)), code=status.HTTP_400_BAD_REQUEST) - if request: - lambda_function_call_signal.send(sender=self, request=request) + if is_interactive and request: + interactive_function_call_signal.send(sender=self, request=request) response = self.gateway.invoke(self, payload) @@ -794,7 +802,7 @@ def call(self, request, func_id): gateway = LambdaGateway() lambda_func = gateway.get(func_id) - return lambda_func.invoke(db_task, request.data, db_job=job, request=request) + return lambda_func.invoke(db_task, request.data, db_job=job, is_interactive=True, request=request) @extend_schema(tags=['lambda']) @extend_schema_view(