Skip to content

feat: Import and Export function lib #2294

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 1 commit into from
Feb 17, 2025
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 52 additions & 1 deletion apps/function_lib/serializers/function_lib_serializer.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,15 +7,21 @@
@desc:
"""
import json
import pickle
import re
import uuid
from typing import List

from django.core import validators
from django.db import transaction
from django.db.models import QuerySet, Q
from rest_framework import serializers
from django.http import HttpResponse
from rest_framework import serializers, status

from common.db.search import page_search
from common.exception.app_exception import AppApiException
from common.field.common import UploadedFileField
from common.response import result
from common.util.field_message import ErrMessage
from common.util.function_code import FunctionExecutor
from function_lib.models.function import FunctionLib
Expand All @@ -24,6 +30,11 @@

function_executor = FunctionExecutor(CONFIG.get('SANDBOX'))

class FlibInstance:
def __init__(self, function_lib: dict, version: str):
self.function_lib = function_lib
self.version = version


class FunctionLibModelSerializer(serializers.ModelSerializer):
class Meta:
Expand Down Expand Up @@ -227,3 +238,43 @@ def one(self, with_valid=True):
raise AppApiException(500, _('Function does not exist'))
function_lib = QuerySet(FunctionLib).filter(id=self.data.get('id')).first()
return FunctionLibModelSerializer(function_lib).data

def export(self, with_valid=True):
try:
if with_valid:
self.is_valid()
id = self.data.get('id')
function_lib = QuerySet(FunctionLib).filter(id=id).first()
application_dict = FunctionLibModelSerializer(function_lib).data
mk_instance = FlibInstance(application_dict, 'v1')
application_pickle = pickle.dumps(mk_instance)
response = HttpResponse(content_type='text/plain', content=application_pickle)
response['Content-Disposition'] = f'attachment; filename="{function_lib.name}.flib"'
return response
except Exception as e:
return result.error(str(e), response_status=status.HTTP_500_INTERNAL_SERVER_ERROR)

class Import(serializers.Serializer):
file = UploadedFileField(required=True, error_messages=ErrMessage.image(_("file")))
user_id = serializers.UUIDField(required=True, error_messages=ErrMessage.uuid(_("User ID")))

@transaction.atomic
def import_(self, with_valid=True):
if with_valid:
self.is_valid()
user_id = self.data.get('user_id')
flib_instance_bytes = self.data.get('file').read()
try:
flib_instance = pickle.loads(flib_instance_bytes)
except Exception as e:
raise AppApiException(1001, _("Unsupported file format"))
function_lib = flib_instance.function_lib
function_lib_model = FunctionLib(id=uuid.uuid1(), name=function_lib.get('name'),
desc=function_lib.get('desc'),
code=function_lib.get('code'),
user_id=user_id,
input_field_list=function_lib.get('input_field_list'),
permission_type=function_lib.get('permission_type'),
is_active=function_lib.get('is_active'))
function_lib_model.save()
return True
21 changes: 21 additions & 0 deletions apps/function_lib/swagger_api/function_lib_api.py
Original file line number Diff line number Diff line change
Expand Up @@ -194,3 +194,24 @@ def get_request_body_api():
}))
}
)

class Export(ApiMixin):
@staticmethod
def get_request_params_api():
return [openapi.Parameter(name='id',
in_=openapi.IN_PATH,
type=openapi.TYPE_STRING,
required=True,
description=_('ID')),

]

class Import(ApiMixin):
@staticmethod
def get_request_params_api():
return [openapi.Parameter(name='file',
in_=openapi.IN_FORM,
type=openapi.TYPE_FILE,
required=True,
description=_('Upload image files'))
]
2 changes: 2 additions & 0 deletions apps/function_lib/urls.py
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,8 @@
urlpatterns = [
path('function_lib', views.FunctionLibView.as_view()),
path('function_lib/debug', views.FunctionLibView.Debug.as_view()),
path('function_lib/<str:id>/export', views.FunctionLibView.Export.as_view()),
path('function_lib/import', views.FunctionLibView.Import.as_view()),
path('function_lib/pylint', views.PyLintView.as_view()),
path('function_lib/<str:function_lib_id>', views.FunctionLibView.Operate.as_view()),
path("function_lib/<int:current_page>/<int:page_size>", views.FunctionLibView.Page.as_view(),
Expand Down
30 changes: 29 additions & 1 deletion apps/function_lib/views/function_lib_views.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,11 +8,12 @@
"""
from drf_yasg.utils import swagger_auto_schema
from rest_framework.decorators import action
from rest_framework.parsers import MultiPartParser
from rest_framework.request import Request
from rest_framework.views import APIView

from common.auth import TokenAuth, has_permissions
from common.constants.permission_constants import RoleConstants
from common.constants.permission_constants import RoleConstants, Permission, Group, Operate
from common.response import result
from function_lib.serializers.function_lib_serializer import FunctionLibSerializer
from function_lib.swagger_api.function_lib_api import FunctionLibApi
Expand Down Expand Up @@ -109,3 +110,30 @@ def get(self, request: Request, current_page: int, page_size: int):
'user_id': request.user.id,
'select_user_id': request.query_params.get('select_user_id')}).page(
current_page, page_size))

class Import(APIView):
authentication_classes = [TokenAuth]
parser_classes = [MultiPartParser]

@action(methods="POST", detail=False)
@swagger_auto_schema(operation_summary=_("Import function"), operation_id=_("Import function"),
manual_parameters=FunctionLibApi.Import.get_request_params_api(),
tags=[_("function")]
)
@has_permissions(RoleConstants.ADMIN, RoleConstants.USER)
def post(self, request: Request):
return result.success(FunctionLibSerializer.Import(
data={'user_id': request.user.id, 'file': request.FILES.get('file')}).import_())

class Export(APIView):
authentication_classes = [TokenAuth]

@action(methods="GET", detail=False)
@swagger_auto_schema(operation_summary=_("Export function"), operation_id=_("Export function"),
manual_parameters=FunctionLibApi.Export.get_request_params_api(),
tags=[_("function")]
)
@has_permissions(RoleConstants.ADMIN, RoleConstants.USER)
def get(self, request: Request, id: str):
return FunctionLibSerializer.Operate(
data={'id': id, 'user_id': request.user.id}).export()
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The provided Python code is mostly clear and well-documented, but there are a few areas that can be optimized or improved:

Optimizations/Recommendations

  1. Avoid Unnecessary Imports:

    • The group import (from common.constants.permission_constants import Group) and operate import (from common.constants.permission_constants import Operate) at the beginning seem unnecessary since they are only used within specific functions, such as Import, which might make it more efficient to import them locally when needed.
  2. Simplify Function Descriptions:

    • In FunctionLibApi.Import.get_request_params_api() and FunctionLibApi.Export.get_request_params_api(), consider using simpler descriptions without including long strings like _("Import function") multiple times.
  3. Use Context Managers Properly:

    • Ensure that you use context managers properly, especially if handling files (e.g., request.FILES.get('file')). This helps manage resources efficiently and avoids potential resource leaks.
  4. Consistent Error Handling:

    • Consider adding consistent error handling across different parts of the viewset. If an exception occurs during file processing in Import.post(), ensure that appropriate responses are returned.
  5. Docstring Clarity:

    • Improve clarity and completeness of docstrings for methods that handle requests, ensuring all parameters and return values are documented accurately.
  6. Performance Considerations:

    • If this viewset handles large datasets or frequent operations, consider implementing caching strategies to improve performance.

Here's a refined version of the Import method with some optimizations:

class Import(APIView):
    authentication_classes = [TokenAuth]
    parser_classes = [MultiPartParser]

    @action(methods="POST", detail=False)
    @swagger_auto_schema(operation_summary=_("Import function"), operation_id=_("Import function"),
                         manual_parameters=[
                             FunctionLibApi.Import.get_request_params_api().get("file")
                         ],
                         tags=[_("function")]
                         )
    def post(self, request: Request):
        try:
            # Read the uploaded file content
            file_content = request.FILES['file'].read()

            return result.success(
                FunctionLibSerializer.Import(data={
                    'user_id': request.user.id,
                    'data': file_content  # Assuming imported data comes from a field named "data"
                }).import_()
            
        except FileNotFoundError:
            return result.failure(_("File not found"), status_code=404)
        
        except Exception as e:
            return result.failure(str(e), log_exception=True)  # Log exceptions for debugging purposes

This refactored version reduces redundancy by importing necessary elements locally and improves error handling by explicitly catching and responding to file-related errors.

23 changes: 22 additions & 1 deletion ui/src/api/function-lib.ts
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
import { Result } from '@/request/Result'
import { get, post, del, put } from '@/request/index'
import { get, post, del, put, exportFile } from '@/request/index'
import type { pageRequest } from '@/api/type/common'
import type { functionLibData } from '@/api/type/function-lib'
import { type Ref } from 'vue'
Expand Down Expand Up @@ -99,6 +99,25 @@ const pylint: (code: string, loading?: Ref<boolean>) => Promise<Result<any>> = (
return post(`${prefix}/pylint`, { code }, {}, loading)
}

const exportFunctionLib = (
id: string,
name: string,
loading?: Ref<boolean>
) => {
return exportFile(
name + '.flib',
`${prefix}/${id}/export`,
undefined,
loading
)
}

const importFunctionLib: (data: any, loading?: Ref<boolean>) => Promise<Result<any>> = (
data,
loading
) => {
return post(`${prefix}/import`, data, undefined, loading)
}
export default {
getFunctionLib,
postFunctionLib,
Expand All @@ -107,5 +126,7 @@ export default {
getAllFunctionLib,
delFunctionLib,
getFunctionLibById,
exportFunctionLib,
importFunctionLib,
pylint
}
1 change: 1 addition & 0 deletions ui/src/locales/lang/en-US/views/function-lib.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export default {
createFunction: 'Create Function',
editFunction: 'Edit Function',
copyFunction: 'Copy Function',
importFunction: 'Import Function',
searchBar: {
placeholder: 'Search by function name'
},
Expand Down
1 change: 1 addition & 0 deletions ui/src/locales/lang/zh-CN/views/function-lib.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export default {
createFunction: '创建函数',
editFunction: '编辑函数',
copyFunction: '复制函数',
importFunction: '导入函数',
searchBar: {
placeholder: '按函数名称搜索'
},
Expand Down
1 change: 1 addition & 0 deletions ui/src/locales/lang/zh-Hant/views/function-lib.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ export default {
createFunction: '建立函數',
editFunction: '編輯函數',
copyFunction: '複製函數',
importFunction: '匯入函數',
searchBar: {
placeholder: '按函數名稱搜尋'
},
Expand Down
103 changes: 101 additions & 2 deletions ui/src/views/function-lib/index.vue
Original file line number Diff line number Diff line change
Expand Up @@ -42,7 +42,29 @@
>
<el-row :gutter="15">
<el-col :xs="24" :sm="12" :md="8" :lg="6" :xl="6" class="mb-16">
<CardAdd :title="$t('views.functionLib.createFunction')" @click="openCreateDialog()" />
<el-card shadow="hover" class="application-card-add" style="--el-card-padding: 8px">
<div class="card-add-button flex align-center cursor p-8" @click="openCreateDialog">
<AppIcon iconName="app-add-application" class="mr-8"></AppIcon>
{{ $t('views.functionLib.createFunction') }}
</div>
<el-divider style="margin: 8px 0" />
<el-upload
ref="elUploadRef"
:file-list="[]"
action="#"
multiple
:auto-upload="false"
:show-file-list="false"
:limit="1"
:on-change="(file: any, fileList: any) => importFunctionLib(file)"
class="card-add-button"
>
<div class="flex align-center cursor p-8">
<AppIcon iconName="app-import" class="mr-8"></AppIcon>
{{ $t('views.functionLib.importFunction') }}
</div>
</el-upload>
</el-card>
</el-col>
<el-col
:xs="24"
Expand Down Expand Up @@ -98,6 +120,12 @@
</el-button>
</el-tooltip>
<el-divider direction="vertical" />
<el-tooltip effect="dark" :content="$t('common.export')" placement="top">
<el-button text @click.stop="exportFunctionLib(item)">
<AppIcon iconName="app-export"></AppIcon>
</el-button>
</el-tooltip>
<el-divider direction="vertical" />
<el-tooltip effect="dark" :content="$t('common.delete')" placement="top">
<el-button
:disabled="item.permission_type === 'PUBLIC' && !canEdit(item)"
Expand Down Expand Up @@ -131,7 +159,7 @@ import { ref, onMounted, reactive } from 'vue'
import { cloneDeep } from 'lodash'
import functionLibApi from '@/api/function-lib'
import FunctionFormDrawer from './component/FunctionFormDrawer.vue'
import { MsgSuccess, MsgConfirm } from '@/utils/message'
import { MsgSuccess, MsgConfirm, MsgError } from '@/utils/message'
import useStore from '@/stores'
import applicationApi from '@/api/application'
import { t } from '@/locales'
Expand Down Expand Up @@ -161,6 +189,7 @@ interface UserOption {
const userOptions = ref<UserOption[]>([])

const selectUserId = ref('all')
const elUploadRef = ref<any>()

const canEdit = (row: any) => {
return user.userInfo?.id === row?.user_id
Expand Down Expand Up @@ -242,6 +271,40 @@ function copyFunctionLib(row: any) {
FunctionFormDrawerRef.value.open(obj)
}

function exportFunctionLib(row: any) {
functionLibApi.exportFunctionLib(row.id, row.name, loading)
.catch((e: any) => {
if (e.response.status !== 403) {
e.response.data.text().then((res: string) => {
MsgError(`${t('views.application.tip.ExportError')}:${JSON.parse(res).message}`)
})
}
})
}

function importFunctionLib(file: any) {
const formData = new FormData()
formData.append('file', file.raw, file.name)
elUploadRef.value.clearFiles()
functionLibApi
.importFunctionLib(formData, loading)
.then(async (res: any) => {
if (res?.data) {
searchHandle()
}
})
.catch((e: any) => {
if (e.code === 400) {
MsgConfirm(t('common.tip'), t('views.application.tip.professionalMessage'), {
cancelButtonText: t('common.confirm'),
confirmButtonText: t('common.professional')
}).then(() => {
window.open('https://maxkb.cn/pricing.html', '_blank')
})
}
})
}

function getList() {
const params = {
...(searchValue.value && { name: searchValue.value }),
Expand Down Expand Up @@ -303,6 +366,42 @@ onMounted(() => {
})
</script>
<style lang="scss" scoped>
.application-card-add {
width: 100%;
font-size: 14px;
min-height: var(--card-min-height);
border: 1px dashed var(--el-border-color);
background: var(--el-disabled-bg-color);
border-radius: 8px;
box-sizing: border-box;

&:hover {
border: 1px solid var(--el-card-bg-color);
background-color: var(--el-card-bg-color);
}

.card-add-button {
&:hover {
border-radius: 4px;
background: var(--app-text-color-light-1);
}

:deep(.el-upload) {
display: block;
width: 100%;
color: var(--el-text-color-regular);
}
}
}

.application-card {
.status-tag {
position: absolute;
right: 16px;
top: 15px;
}
}

.function-lib-list-container {
.status-button {
position: absolute;
Expand Down