From 61fb4f0a60f108e53be8b77c78dc5eb924a24940 Mon Sep 17 00:00:00 2001 From: dlimeng Date: Tue, 2 Jan 2024 03:46:02 +0800 Subject: [PATCH] solidui job page --- solidui/app.py | 4 +- solidui/config.py | 1 + solidui/daos/project.py | 9 ++- solidui/initialization/__init__.py | 14 ++-- solidui/utils/login_utils.py | 11 ++- solidui/views/base.py | 113 ++++++++++++++++++++++------- solidui/views/base_schemas.py | 6 +- solidui/views/job/api.py | 24 +++--- solidui/views/model/api.py | 5 +- solidui/views/model/schemas.py | 6 ++ solidui/views/project/api.py | 2 +- 11 files changed, 140 insertions(+), 55 deletions(-) diff --git a/solidui/app.py b/solidui/app.py index 11f301e..4009949 100644 --- a/solidui/app.py +++ b/solidui/app.py @@ -22,6 +22,7 @@ logger = logging.getLogger(__name__) + def create_app(solidui_config_module: Optional[str] = None) -> Flask: app = SolidUIApp(__name__) try: @@ -41,7 +42,6 @@ def create_app(solidui_config_module: Optional[str] = None) -> Flask: logger.exception("Failed to create app") raise ex + class SolidUIApp(Flask): pass - - diff --git a/solidui/config.py b/solidui/config.py index 2231970..8090e11 100644 --- a/solidui/config.py +++ b/solidui/config.py @@ -49,6 +49,7 @@ #SQLALCHEMY_DATABASE_URI = 'mysql+pymysql://username:password@host:port/dbname' + # SQLALCHEMY_CUSTOM_PASSWORD_STORE = lookup_password SQLALCHEMY_CUSTOM_PASSWORD_STORE = None diff --git a/solidui/daos/project.py b/solidui/daos/project.py index e95d0d3..4963b79 100644 --- a/solidui/daos/project.py +++ b/solidui/daos/project.py @@ -15,6 +15,9 @@ from datetime import datetime from typing import Optional + +from solidui.extensions import db + from solidui.daos.base import BaseDAO from solidui.daos.exceptions import DAONotFound from solidui.entity.core import Project @@ -76,7 +79,11 @@ def get_project(cls, project_id: int) -> Project: return project @classmethod - def query_project_list_paging(cls, search_name: str, page_no: int, page_size: int) -> PageInfo[Project]: + def get_project_list(cls, status: int) -> list[Project]: + return db.session.query(Project).filter_by(status=status).all() + + @classmethod + def get_project_list_paging(cls, search_name: str, page_no: int, page_size: int) -> PageInfo[Project]: # Build custom filters custom_filters = None if search_name: diff --git a/solidui/initialization/__init__.py b/solidui/initialization/__init__.py index e3a2e65..e3122e6 100644 --- a/solidui/initialization/__init__.py +++ b/solidui/initialization/__init__.py @@ -16,7 +16,7 @@ import logging import os import sys -from typing import Any, Callable, TYPE_CHECKING +from typing import Any,TYPE_CHECKING from deprecation import deprecated import wtforms_json @@ -27,14 +27,15 @@ appbuilder, db ) -from solidui.solidui_typing import FlaskResponse from solidui.utils.base import pessimistic_connection_handling, is_test + if TYPE_CHECKING: from solidui.app import SolidUIApp logger = logging.getLogger(__name__) + class SolidUIAppInitializer: def __init__(self, app: SolidUIApp) -> None: super().__init__() @@ -48,7 +49,6 @@ def __init__(self, app: SolidUIApp) -> None: def flask_app(self) -> SolidUIApp: return self.solidui_app - def pre_init(self) -> None: """ Called before all other init tasks are complete @@ -62,6 +62,8 @@ def post_init(self) -> None: """ Called after any other init tasks """ + from solidui.views.base import schedule_clean_job_element_page + schedule_clean_job_element_page() def setup_db(self) -> None: db.init_app(self.solidui_app) @@ -110,16 +112,12 @@ def init_views(self) -> None: for rule in self.solidui_app.url_map.iter_rules(): print(rule) - def configure_fab(self) -> None: if self.config["SILENCE_FAB"]: logging.getLogger("flask_appbuilder").setLevel(logging.ERROR) - appbuilder.init_app(self.solidui_app, db.session) - - def configure_session(self) -> None: if self.config["SESSION_SERVER_SIDE"]: Session(self.solidui_app) @@ -175,6 +173,6 @@ def init_app(self) -> None: with self.solidui_app.app_context(): self.init_app_in_ctx() + self.post_init() - self.post_init() diff --git a/solidui/utils/login_utils.py b/solidui/utils/login_utils.py index 8fe0004..1b48f2c 100644 --- a/solidui/utils/login_utils.py +++ b/solidui/utils/login_utils.py @@ -18,6 +18,7 @@ from flask import request, make_response from solidui.common.constants import SESSION_TIMEOUT, TICKETHEADER, CRYPTKEY, SESSION_TICKETID_KEY, ADMIN_NAME +from solidui.daos.exceptions import DAOException from solidui.utils.des_utils import DESUtil logger = logging.getLogger(__name__) @@ -33,14 +34,18 @@ def remove_timeout_user_tickets(): if current_time - value > SESSION_TIMEOUT: logger.info(f"remove timeout userTicket {key}, since the last access time is {value}.") user_ticket_id_to_last_access_time.pop(key, None) - except Exception as e: + except DAOException as e: logger.error("Failed to remove timeout user ticket id.", exc_info=True) + threads = threading.Timer(SESSION_TIMEOUT / 1000 / 10, remove_timeout_user_tickets) + threads.daemon = True + threads.start() thread = threading.Timer(SESSION_TIMEOUT / 1000 / 10, remove_timeout_user_tickets) thread.daemon = True thread.start() + schedule_timeout_removal() @@ -48,7 +53,7 @@ def get_user_ticket_id(username: str): timeout_user = f"{username},{int(time.time() * 1000)}" try: return DESUtil.encrypt(f"{TICKETHEADER}{timeout_user}", CRYPTKEY) - except Exception as e: + except DAOException as e: logger.info(f"Failed to encrypt user ticket id, username: {username}") return None @@ -85,6 +90,6 @@ def get_login_user(cookies): timeout_user = DESUtil.decrypt(user_ticket_id, CRYPTKEY) if timeout_user and timeout_user.startswith(TICKETHEADER): return timeout_user.split(',')[0][len(TICKETHEADER):] - except Exception as e: + except DAOException as e: logger.info(f"Failed to decrypt user ticket id, userTicketId: {user_ticket_id}") return ADMIN_NAME diff --git a/solidui/views/base.py b/solidui/views/base.py index 4aa832a..105365e 100644 --- a/solidui/views/base.py +++ b/solidui/views/base.py @@ -11,51 +11,69 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from __future__ import annotations import json import logging +import threading +import time from datetime import datetime - +from dataclasses import asdict +from solidui.common.constants import CLEAN_PERIOD +from solidui.daos.exceptions import DAODeleteFailedError +from solidui.daos.job_element import JobElementDAO from solidui.daos.job_element_page import JobElementPageDAO +from solidui.daos.job_page import JobPageDAO +from solidui.daos.project import ProjectDAO from solidui.entity.core import JobElementPage, JobElement, JobPage -from solidui.views.base_schemas import View, DataView, Data, Size, Position, JobPageDTO +from solidui.views.base_schemas import View, DataView, Data, Size, Position, JobPageDTO, JobElementPageVO, Page logger = logging.getLogger(__name__) -def deep_copy_view_to_data_view(view: View) -> DataView: +def serialize_dataclass(dataclass_instance): + """ + Serialize a dataclass instance to a dictionary, handling nested dataclasses. + """ + if isinstance(dataclass_instance, list): + return [serialize_dataclass(item) for item in dataclass_instance] + elif hasattr(dataclass_instance, "__dict__"): + return {k: serialize_dataclass(v) for k, v in asdict(dataclass_instance).items()} + else: + return dataclass_instance + + +def deep_copy_view_to_data_view(view: View): # Copy Position - new_position = None - if view.position is not None: - new_position = Position(top=view.position.top, left=view.position.left) + new_position = Position(**view.position) if isinstance(view.position, dict) else view.position # Copy Size - new_size = None - if view.size is not None: - new_size = Size(width=view.size.width, height=view.size.height) + new_size = Size(**view.size) if isinstance(view.size, dict) else view.size # Copy Data - new_data = None - if view.data is not None: - new_data = Data( - dataSourceId=view.data.dataSourceId, - dataSourceName=view.data.dataSourceName, - dataSourceTypeId=view.data.dataSourceTypeId, - dataSourceTypeName=view.data.dataSourceTypeName, - sql=view.data.sql, - table=view.data.table - ) + new_data = Data(**view.data) if isinstance(view.data, dict) else view.data # Copy DataView data_view = DataView(position=new_position, size=new_size, options=view.options, data=new_data) - return data_view + serialized_data_view = serialize_dataclass(data_view) + + return json.dumps(serialized_data_view, ensure_ascii=False) + +def save_job_element_page(job_element_page_vo: JobElementPageVO, job_element_id: int) -> None: + # page_id: int + new_page = Page(**job_element_page_vo.page) if isinstance(job_element_page_vo.page, + dict) else job_element_page_vo.page + # size: Size + new_size = Size(**job_element_page_vo.size) if isinstance(job_element_page_vo.size, + dict) else job_element_page_vo.size + + serialized_size = serialize_dataclass(new_size) -def save_job_element_page(page_id: int, job_element_id: int, size: Size) -> None: job_element_page = JobElementPage( - job_page_id=page_id, + job_page_id=new_page.id, job_element_id=job_element_id, - position=json.dumps(size), + position=json.dumps(serialized_size, ensure_ascii=False), create_time=datetime.now(), update_time=datetime.now() ) @@ -69,12 +87,19 @@ def create_job_element_page_vo(job_element: JobElement, views: list[View]) -> No view = View() view.id = job_element.id view.title = job_element.name - view.type = job_element.dataType + view.type = job_element.data_type # Assuming JSONUtils is replaced with a Python JSON library - data_view: DataView = json.loads(job_element.data) - if not data_view: + data_view_dict = json.loads(job_element.data) + if not data_view_dict: return + data_view = DataView( + position=Position(**data_view_dict['position']) if 'position' in data_view_dict else None, + size=Size(**data_view_dict['size']) if 'size' in data_view_dict else None, + options=data_view_dict.get('options'), + data=Data(**data_view_dict['data']) if 'data' in data_view_dict else None + ) + view.position = data_view.position view.size = data_view.size view.options = data_view.options @@ -97,3 +122,39 @@ def convert_to_dto(job_page: JobPage): children=[] # Initially empty, to be filled later if needed ) return job_page_dto + + +def schedule_clean_job_element_page(): + thread = threading.Timer(CLEAN_PERIOD / 1000 / 10, clean_job_element_page_task) + thread.daemon = True + thread.start() + + +def clean_job_element_page_task(): + try: + clean_job_element_page() + except DAODeleteFailedError as e: + logger.error(f"Error cleaning job element page : {e}") + thread = threading.Timer(CLEAN_PERIOD / 1000 / 10, clean_job_element_page_task) + thread.daemon = True + thread.start() + + +def clean_job_element_page(): + """ + Clean job element pages for projects. + """ + projects = ProjectDAO.get_project_list(1) + if projects: + for project in projects: + JobElementPageDAO.delete_project_id(project.id) + logger.info(f"cleanJobElementPage delete_project_id projectId: {project.id}") + + JobElementDAO.delete_job_element_project_id(project.id) + logger.info(f"cleanJobElementPage delete_job_element_project_id projectId: {project.id}") + # Delete associated records + JobPageDAO.delete_project_id(project.id) + logger.info(f"cleanJobElementPage delete_project_id projectId: {project.id}") + + ProjectDAO.delete(project) + logger.info(f"cleanJobElementPage projectId: {project.id}") diff --git a/solidui/views/base_schemas.py b/solidui/views/base_schemas.py index ded69f1..90bf18e 100644 --- a/solidui/views/base_schemas.py +++ b/solidui/views/base_schemas.py @@ -14,12 +14,12 @@ from collections import deque from dataclasses import dataclass from datetime import datetime -from typing import Optional, Dict, List - +from typing import Dict, List from marshmallow_sqlalchemy.schema import Schema from marshmallow_sqlalchemy.fields import fields + @dataclass class Position: top: str = None @@ -130,3 +130,5 @@ class JobPageDTOSchema(Schema): orders = fields.Int() # Use a Nested field for children, with many=True to indicate it's a list children = fields.Nested('self', many=True, exclude=('children',)) + + diff --git a/solidui/views/job/api.py b/solidui/views/job/api.py index a82b659..715237a 100644 --- a/solidui/views/job/api.py +++ b/solidui/views/job/api.py @@ -55,13 +55,14 @@ def save_page(self) -> FlaskResponse: if not views: # If the list is empty, save the job element page with jobElementId set to 0 try: - save_job_element_page(job_element_page_vo.page.id, 0, job_element_page_vo.size) + save_job_element_page(job_element_page_vo, 0) return self.response_format() except DAOCreateFailedError as ex: logger.exception(ex) return self.handle_error(SolidUIErrorType.CREATE_JOB_PAGE_ERROR) - for view in views: + for view_dict in views: + view = View(**view_dict) job_element = JobElement( project_id=job_element_page_vo.projectId, data_type=view.type, @@ -72,7 +73,7 @@ def save_page(self) -> FlaskResponse: job_element.data = deep_copy_view_to_data_view(view) try: JobElementDAO.create(item=job_element) - save_job_element_page(job_element_page_vo.page.id, job_element.id, job_element_page_vo.size) + save_job_element_page(job_element_page_vo, job_element.id) except DAOCreateFailedError as ex: logger.exception(ex) return self.handle_error(SolidUIErrorType.CREATE_JOB_ERROR) @@ -91,7 +92,9 @@ def update_job_page(self) -> FlaskResponse: if not job_element_page_vo.page or not job_element_page_vo.size: return self.handle_error(SolidUIErrorType.QUERY_JOB_PAGE_ERROR) - job_element_pages: list[JobElementPage] = JobElementPageDAO.get_job_element_page_id(job_element_page_vo.page.id) + new_page = Page(**job_element_page_vo.page) if isinstance(job_element_page_vo.page, + dict) else job_element_page_vo.page + job_element_pages: list[JobElementPage] = JobElementPageDAO.get_job_element_page_id(new_page.id) if job_element_pages: for ep in job_element_pages: JobElementPageDAO.delete(ep) @@ -99,19 +102,20 @@ def update_job_page(self) -> FlaskResponse: if ep.job_element_id and ep.job_element_id > 0: job_element = JobElementDAO.find_by_id(ep.job_element_id) if job_element: - JobElementDAO.delete(ep.job_element) + JobElementDAO.delete(job_element) views = job_element_page_vo.views if not views: # If the list is empty, save the job element page with jobElementId set to 0 try: - save_job_element_page(job_element_page_vo.page.id, 0, job_element_page_vo.size) + save_job_element_page(job_element_page_vo, 0) return self.response_format() except DAOCreateFailedError as ex: logger.exception(ex) return self.handle_error(SolidUIErrorType.UPDATE_JOB_ERROR) - for view in views: + for view_dict in views: + view = View(**view_dict) job_element = JobElement( project_id=job_element_page_vo.projectId, data_type=view.type, @@ -122,7 +126,7 @@ def update_job_page(self) -> FlaskResponse: job_element.data = deep_copy_view_to_data_view(view) try: JobElementDAO.create(item=job_element) - save_job_element_page(job_element_page_vo.page.id, job_element.id, job_element_page_vo.size) + save_job_element_page(job_element_page_vo, job_element.id) except DAOCreateFailedError as ex: logger.exception(ex) return self.handle_error(SolidUIErrorType.UPDATE_JOB_ERROR) @@ -150,13 +154,13 @@ def query_job_page(self) -> FlaskResponse: for job_element_page in job_element_pages: # Retrieve the associated JobElement - job_element_id = job_element_page.jobElementId + job_element_id = job_element_page.job_element_id if job_element_id and job_element_id > 0: # Assuming job_element_mapper is an instance with a select_by_id method job_element = JobElementDAO.find_by_id(job_element_id) if first: - job_element_page_vos.page = Page(id=job_element_page.jobPageId) + job_element_page_vos.page = Page(id=job_element_page.job_page_id) # Assuming JSONUtils is replaced with a Python JSON library job_element_page_vos.size = json.loads(job_element_page.position) first = False diff --git a/solidui/views/model/api.py b/solidui/views/model/api.py index 4954306..cc1791a 100644 --- a/solidui/views/model/api.py +++ b/solidui/views/model/api.py @@ -27,7 +27,7 @@ from solidui.views.base_api import BaseSolidUIApi from flask_appbuilder.api import expose, safe from solidui.kernel_program.main import APP_PORT as KERNEL_APP_PORT -from solidui.views.model.schemas import ModelKeyVO, ModelTypePageInfoSchema +from solidui.views.model.schemas import ModelKeyVO, ModelTypePageInfoSchema, ModelKeyVOSchema from solidui.views.utils import get_code, add_prompt_type_buffer logger = logging.getLogger(__name__) @@ -48,7 +48,8 @@ def get_model_list(self) -> FlaskResponse: for m in model_types: model_key_vos.append(ModelKeyVO(m.id, f"{m.name}_{m.code}", m.type_name)) - return self.response_format(data=model_key_vos) + schema = ModelKeyVOSchema() + return self.response_format(data=schema.dump(model_key_vos, many=True)) @expose('/api/', methods=('GET', 'POST')) @safe diff --git a/solidui/views/model/schemas.py b/solidui/views/model/schemas.py index 12352e1..a8c7c9c 100644 --- a/solidui/views/model/schemas.py +++ b/solidui/views/model/schemas.py @@ -38,3 +38,9 @@ class ModelTypePageInfoSchema(Schema): total = fields.Int() total_page = fields.Int() total_list = fields.Nested(ModelTypeSchema, many=True) + + +class ModelKeyVOSchema(Schema): + id = fields.Int() + name = fields.Str() + type_name = fields.Str() diff --git a/solidui/views/project/api.py b/solidui/views/project/api.py index ccdfaa3..da6ba29 100644 --- a/solidui/views/project/api.py +++ b/solidui/views/project/api.py @@ -74,7 +74,7 @@ def query_Project_List_Paging(self) -> FlaskResponse: page_size = request.args.get('pageSize', default=10, type=int) page_no = request.args.get('pageNo', default=1, type=int) - page_info = ProjectDAO.query_project_list_paging(search_name, page_no, page_size) + page_info = ProjectDAO.get_project_list_paging(search_name, page_no, page_size) page_info_schema = PageInfoSchema()