From dc24f39962697ed302632e2e68e97ceb65018cbd Mon Sep 17 00:00:00 2001 From: dlimeng Date: Sun, 24 Dec 2023 04:00:14 +0800 Subject: [PATCH] solidui datasouerce --- README.md | 8 +- README_CN.md | 7 +- solidui/config.py | 7 +- solidui/daos/datasource.py | 91 ++++++++- solidui/daos/datasource_type.py | 21 +- .../datasource_plugin/{plugin.py => base.py} | 10 +- solidui/datasource_plugin/base_jdbc_client.py | 55 ------ .../base_jdbc_client_factory.py | 37 ---- solidui/datasource_plugin/connect_dto.py | 23 --- .../datasource_plugin/connection_factory.py | 21 -- solidui/datasource_plugin/constants.py | 24 --- .../datasource_plugin/data_source_utils.py | 47 ----- solidui/datasource_plugin/jdbc_client.py | 40 ---- .../datasource_plugin/jdbc_client_factory.py | 24 --- .../datasource_plugin/jdbc_client_manager.py | 23 --- .../datasource_plugin/mysql/mysql_client.py | 51 ----- .../mysql/mysql_client_factory.py | 30 --- .../mysql/mysql_connection_factory.py | 32 --- solidui/views/datasource/api.py | 187 +++++++++++++++++- solidui/views/datasource/schemas.py | 22 +++ solidui/views/project/api.py | 18 +- 21 files changed, 341 insertions(+), 437 deletions(-) rename solidui/datasource_plugin/{plugin.py => base.py} (83%) delete mode 100644 solidui/datasource_plugin/base_jdbc_client.py delete mode 100644 solidui/datasource_plugin/base_jdbc_client_factory.py delete mode 100644 solidui/datasource_plugin/connect_dto.py delete mode 100644 solidui/datasource_plugin/connection_factory.py delete mode 100644 solidui/datasource_plugin/constants.py delete mode 100644 solidui/datasource_plugin/data_source_utils.py delete mode 100644 solidui/datasource_plugin/jdbc_client.py delete mode 100644 solidui/datasource_plugin/jdbc_client_factory.py delete mode 100644 solidui/datasource_plugin/jdbc_client_manager.py delete mode 100644 solidui/datasource_plugin/mysql/mysql_client.py delete mode 100644 solidui/datasource_plugin/mysql/mysql_client_factory.py delete mode 100644 solidui/datasource_plugin/mysql/mysql_connection_factory.py diff --git a/README.md b/README.md index 143ace6d..de46ab0c 100644 --- a/README.md +++ b/README.md @@ -10,17 +10,11 @@ - - - - github forks - -github stars - + github contributors diff --git a/README_CN.md b/README_CN.md index d837e691..190639ea 100644 --- a/README_CN.md +++ b/README_CN.md @@ -10,9 +10,6 @@ - - - github forks @@ -61,8 +58,8 @@ AI生成图形模型 * 支持多种数据源 * 支持Huggingface space * 支持插件机器人 -* 支持SolidUI-Model -* 支持Large Language Model +* 支持SolidUI模型 +* 支持大模型接入 * 容器化部署 # 快速开始 diff --git a/solidui/config.py b/solidui/config.py index 99b7d6b9..f88e182c 100644 --- a/solidui/config.py +++ b/solidui/config.py @@ -60,4 +60,9 @@ # Console Log Settings LOG_FORMAT = "%(asctime)s:%(levelname)s:%(name)s:%(message)s" -LOG_LEVEL = "DEBUG" \ No newline at end of file +LOG_LEVEL = "DEBUG" + +PREFERRED_DATABASES: list[str] = [ + "MySQL", + "SQLite", +] \ No newline at end of file diff --git a/solidui/daos/datasource.py b/solidui/daos/datasource.py index 236ba8c0..dc3a9448 100644 --- a/solidui/daos/datasource.py +++ b/solidui/daos/datasource.py @@ -11,9 +11,98 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from typing import Any +import logging + +from solidui.utils.page_info import PageInfo + from solidui.daos.base import BaseDAO +from solidui.daos.exceptions import DAOCreateFailedError, DAOUpdateFailedError, DAONotFound, DAODeleteFailedError from solidui.entity.core import DataSource +logger = logging.getLogger(__name__) + -class DatasourceDAO(BaseDAO[DataSource]): +class DataSourceDAO(BaseDAO[DataSource]): model_cls = DataSource + + @classmethod + def create_data_source(cls, data_source: DataSource) -> DataSource: + new_data_source = super().find_one_or_none(datasource_name=data_source.datasource_name) + if new_data_source is not None: + raise DAOCreateFailedError(message="DataSource is already exists") + + return super().create(item=data_source) + + @classmethod + def get_data_source_name(cls, data_source_name: int) -> DataSource: + return super().find_one_or_none(datasource_name=data_source_name) + + @classmethod + def get_data_source_id(cls, data_source_id: int) -> DataSource: + return super().find_by_id(data_source_id) + + @classmethod + def exist_data_source(cls, data_source_id: int) -> DataSource: + data_source = super().find_by_id(data_source_id) + if data_source is None: + raise DAONotFound(message="DataSource is null") + + try: + return super().update(item=data_source, attributes={'expire': True}) + except DAOUpdateFailedError as ex: + logger.exception(ex.exception) + raise ex + + @classmethod + def get_data_source_page(cls, page_no: int, page_size: int, name_filter: str = None, type_filter: int = None, + expire_filter: bool = None) -> PageInfo: + query = cls.model_cls.query + + # Apply filters + if name_filter is not None: + query = query.filter(cls.model_cls.datasource_name.like(f"%{name_filter}%")) + if type_filter is not None: + query = query.filter(cls.model_cls.datasource_type_id == type_filter) + if expire_filter is not None: + query = query.filter(cls.model_cls.expire == expire_filter) + + # Apply pagination + pagination = query.paginate(page_no, page_size, error_out=False) + + page_info = PageInfo(page_no, page_size) + page_info.set_total(pagination.total) + page_info.set_total_list(pagination.items) + + return page_info + + @classmethod + def delete_data_source(cls, data_source_id: int) -> None: + if data_source_id is None: + raise DAONotFound(message="DataSource id is required") + + data_source = cls.find_by_id(data_source_id) + if not data_source: + raise DAONotFound(message="DataSource is required") + + try: + super().delete(data_source) + except DAODeleteFailedError as ex: + logger.exception(ex.exception) + raise ex + + @classmethod + def update_data_source(cls, data_source: DataSource) -> DataSource: + if data_source.id is None: + raise DAONotFound(message="DataSource id is required") + + existing_data_source = cls.find_by_id(data_source.id) + if not existing_data_source: + raise DAONotFound(message="DataSource is required") + + try: + return super().update(existing_data_source, data_source.__dict__) + except DAOUpdateFailedError as ex: + logger.exception(ex) + raise ex + diff --git a/solidui/daos/datasource_type.py b/solidui/daos/datasource_type.py index 2aef8ab2..a8a1d9f2 100644 --- a/solidui/daos/datasource_type.py +++ b/solidui/daos/datasource_type.py @@ -10,4 +10,23 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. \ No newline at end of file +# limitations under the License. + +from solidui.daos.base import BaseDAO +from solidui.daos.exceptions import DAONotFound +from solidui.entity.core import DataSourceType + + +class DataSourceTypeDAO(BaseDAO[DataSourceType]): + model_cls = DataSourceType + + @classmethod + def all_list(cls) -> DataSourceType: + return super().find_all() + + @classmethod + def get_id(cls, id: int) -> DataSourceType: + data_source_type = super().find_by_id(id) + if not data_source_type: + raise DAONotFound(message="DataSourceType not found") + return data_source_type diff --git a/solidui/datasource_plugin/plugin.py b/solidui/datasource_plugin/base.py similarity index 83% rename from solidui/datasource_plugin/plugin.py rename to solidui/datasource_plugin/base.py index b587b6c7..2aef8ab2 100644 --- a/solidui/datasource_plugin/plugin.py +++ b/solidui/datasource_plugin/base.py @@ -10,12 +10,4 @@ # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod - -class Plugin(ABC): - - @abstractmethod - def get_name(self): - pass - +# limitations under the License. \ No newline at end of file diff --git a/solidui/datasource_plugin/base_jdbc_client.py b/solidui/datasource_plugin/base_jdbc_client.py deleted file mode 100644 index 8bf7ce97..00000000 --- a/solidui/datasource_plugin/base_jdbc_client.py +++ /dev/null @@ -1,55 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import logging -from abc import ABC, abstractmethod - - -class BaseJdbcClient(ABC): - - def __init__(self, conn): - self.conn = conn - self.logger = logging.getLogger(__name__) - - def close_resource(self, connection, statement, result_set): - try: - if result_set is not None and not result_set.closed: - result_set.close() - if statement is not None and not statement.closed: - statement.close() - if connection is not None and not connection.closed: - connection.close() - except Exception as e: - self.logger.warn(f"Failed to release resources: {e}") - - def close(self): - self.close_resource(self.conn, None, None) - - @abstractmethod - def generate_select_all_data_sql(self, database, table_name): - pass - - @abstractmethod - def get_all_databases(self): - pass - - @abstractmethod - def get_all_tables(self, database): - pass - - @abstractmethod - def get_select_result(self, sql): - pass - - def get_conn(self): - return self.conn diff --git a/solidui/datasource_plugin/base_jdbc_client_factory.py b/solidui/datasource_plugin/base_jdbc_client_factory.py deleted file mode 100644 index a213f3bd..00000000 --- a/solidui/datasource_plugin/base_jdbc_client_factory.py +++ /dev/null @@ -1,37 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod - -from .constants import * -from .connect_dto import ConnectDTO - -class BaseJdbcClientFactory(ABC): - - def get_connect_dto(self, conn_params): - connect_dto = ConnectDTO() - connect_dto.host = conn_params.get(PARAM_SQL_HOST) - connect_dto.port = conn_params.get(PARAM_SQL_PORT) - connect_dto.username = conn_params.get(PARAM_SQL_USERNAME) - connect_dto.password = conn_params.get(PARAM_SQL_PASSWORD) - connect_dto.database = conn_params.get(PARAM_SQL_DATABASE) - extra_params = conn_params.get(PARAM_SQL_EXTRA_PARAMS) - if extra_params is None: - connect_dto.extra_params = {} - else: - connect_dto.extra_params = extra_params - return connect_dto - - @abstractmethod - def create_jdbc_client(self, connect_dto): - pass diff --git a/solidui/datasource_plugin/connect_dto.py b/solidui/datasource_plugin/connect_dto.py deleted file mode 100644 index 24bdd8d4..00000000 --- a/solidui/datasource_plugin/connect_dto.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -class ConnectDTO: - - def __init__(self): - self.host = None - self.port = None - self.username = None - self.password = None - self.database = None - self.driver_class_name = None - self.extra_params = {} diff --git a/solidui/datasource_plugin/connection_factory.py b/solidui/datasource_plugin/connection_factory.py deleted file mode 100644 index 99e5a6e7..00000000 --- a/solidui/datasource_plugin/connection_factory.py +++ /dev/null @@ -1,21 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from .plugin import Plugin -from abc import ABC, abstractmethod - -class ConnectionFactory(Plugin, ABC): - - @abstractmethod - def open_connection(self, host, port, username, password, database, extra_params): - pass diff --git a/solidui/datasource_plugin/constants.py b/solidui/datasource_plugin/constants.py deleted file mode 100644 index a940cc9e..00000000 --- a/solidui/datasource_plugin/constants.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -PARAM_SQL_HOST = "host" -PARAM_SQL_PORT = "port" -PARAM_SQL_USERNAME = "username" -PARAM_SQL_PASSWORD = "password" -PARAM_SQL_EXTRA_PARAMS = "params" -PARAM_SQL_DATABASE = "database" -PARAM_SQL_DRIVERCLASSNAME = "driverClassName" - -DATASOURCE_CLASSNAME = "com.cloudorc.solidui.plugin.jdbc.%sClientFactory" - -DATASOURCE_TYPE_LIST = "mysql,doris" diff --git a/solidui/datasource_plugin/data_source_utils.py b/solidui/datasource_plugin/data_source_utils.py deleted file mode 100644 index 90917b51..00000000 --- a/solidui/datasource_plugin/data_source_utils.py +++ /dev/null @@ -1,47 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import importlib -from types import ModuleType - -from constants import DATASOURCE_TYPE_LIST, DATASOURCE_CLASSNAME_FORMAT - - -class DataSourceUtils: - jdbc_client_factory_instances = {} - - @staticmethod - def query_jdbc_client_factory(type_name): - - if type_name not in DATASOURCE_TYPE_LIST: - raise ValueError(f"{type_name} is not supported") - - factory = DataSourceUtils.jdbc_client_factory_instances.get(type_name) - if factory is None: - class_name = DATASOURCE_CLASSNAME_FORMAT % type_name - module_name = class_name.rsplit(".", 1)[0] - factory_cls = getattr(importlib.import_module(module_name), class_name.split(".")[-1]) - factory = factory_cls() - DataSourceUtils.jdbc_client_factory_instances[type_name] = factory - - return factory - - @staticmethod - def query_jdbc_client(type_name, data_source): - - factory = DataSourceUtils.query_jdbc_client_factory(type_name) - - connect_dto = factory.get_connect_dto(data_source.params) - - if connect_dto: - return factory.create_jdbc_client(connect_dto) diff --git a/solidui/datasource_plugin/jdbc_client.py b/solidui/datasource_plugin/jdbc_client.py deleted file mode 100644 index 4d763f6c..00000000 --- a/solidui/datasource_plugin/jdbc_client.py +++ /dev/null @@ -1,40 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod - -class JdbcClient(ABC): - - @abstractmethod - def generate_select_all_data_sql(self, database, table_name): - pass - - @abstractmethod - def get_all_databases(self): - pass - - @abstractmethod - def get_all_tables(self, database): - pass - - @abstractmethod - def get_select_result(self, sql): - pass - - @abstractmethod - def close(self): - pass - - @abstractmethod - def get_conn(self): - pass diff --git a/solidui/datasource_plugin/jdbc_client_factory.py b/solidui/datasource_plugin/jdbc_client_factory.py deleted file mode 100644 index ef5b7bea..00000000 --- a/solidui/datasource_plugin/jdbc_client_factory.py +++ /dev/null @@ -1,24 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from abc import ABC, abstractmethod - -class JdbcClientFactory(ABC): - - @abstractmethod - def create_jdbc_client(self, connect_dto): - pass - - @abstractmethod - def get_connect_dto(self, conn_params): - pass diff --git a/solidui/datasource_plugin/jdbc_client_manager.py b/solidui/datasource_plugin/jdbc_client_manager.py deleted file mode 100644 index 6debe68d..00000000 --- a/solidui/datasource_plugin/jdbc_client_manager.py +++ /dev/null @@ -1,23 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import importlib - -class JdbcClientManager: - - @staticmethod - def load_connection_factory(class_name): - parts = class_name.rsplit('.', 1) - module = importlib.import_module(parts[0]) - connection_factory_cls = getattr(module, parts[1]) - return connection_factory_cls() diff --git a/solidui/datasource_plugin/mysql/mysql_client.py b/solidui/datasource_plugin/mysql/mysql_client.py deleted file mode 100644 index 1b33c337..00000000 --- a/solidui/datasource_plugin/mysql/mysql_client.py +++ /dev/null @@ -1,51 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -from solidui.datasource_plugin.base_jdbc_client import BaseJdbcClient - -import mysql.connector - - -class MysqlClient(BaseJdbcClient): - - def __init__(self, conn): - super().__init__(conn) - - def generate_select_all_data_sql(self, database, table_name): - return f"SELECT * FROM {database}.{table_name}" - - def get_all_databases(self): - cursor = self.conn.cursor() - cursor.execute("SHOW DATABASES") - return [db[0] for db in cursor] - - def get_all_tables(self, database): - cursor = self.conn.cursor() - cursor.execute(f"SHOW TABLES FROM {database}") - return [table[0] for table in cursor] - - def get_select_result(self, sql): - cursor = self.conn.cursor() - cursor.execute(sql) - - # 获取列名 - column_names = [col[0] for col in cursor.description] - - results = [] - results.append(column_names) - - # 获取结果 - for row in cursor: - results.append(list(row)) - - return results diff --git a/solidui/datasource_plugin/mysql/mysql_client_factory.py b/solidui/datasource_plugin/mysql/mysql_client_factory.py deleted file mode 100644 index 3d801626..00000000 --- a/solidui/datasource_plugin/mysql/mysql_client_factory.py +++ /dev/null @@ -1,30 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import mysql.connector - -from .base_jdbc_client_factory import BaseJdbcClientFactory -from .mysql_client import MysqlClient - -class MysqlClientFactory(BaseJdbcClientFactory): - - def create_jdbc_client(self, connect_dto): - conn = mysql.connector.connect( - host=connect_dto.host, - port=connect_dto.port, - user=connect_dto.username, - password=connect_dto.password, - database=connect_dto.database, - **connect_dto.extra_params - ) - return MysqlClient(conn) diff --git a/solidui/datasource_plugin/mysql/mysql_connection_factory.py b/solidui/datasource_plugin/mysql/mysql_connection_factory.py deleted file mode 100644 index 3707e28e..00000000 --- a/solidui/datasource_plugin/mysql/mysql_connection_factory.py +++ /dev/null @@ -1,32 +0,0 @@ -# -# Licensed to the Apache Software Foundation (ASF) under one or more -# contributor license agreements. See the NOTICE file distributed with -# this work for additional information regarding copyright ownership. -# The ASF licenses this file to You under the Apache License, Version 2.0 -# (the "License"); you may not use this file except in compliance with -# the License. You may obtain a copy of the License at -# http://www.apache.org/licenses/LICENSE-2.0 -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. -import mysql.connector - -from .connection_factory import ConnectionFactory - -class MysqlConnectionFactory(ConnectionFactory): - - def open_connection(self, host, port, username, password, database, extra_params): - conn = mysql.connector.connect( - host=host, - port=port, - user=username, - password=password, - database=database, - **extra_params - ) - return conn - - def get_name(self): - return "mysql" diff --git a/solidui/views/datasource/api.py b/solidui/views/datasource/api.py index 74965971..de279878 100644 --- a/solidui/views/datasource/api.py +++ b/solidui/views/datasource/api.py @@ -12,14 +12,195 @@ # See the License for the specific language governing permissions and # limitations under the License. +import logging from flask import request +from solidui.daos.datasource import DataSourceDAO +from solidui.daos.datasource_type import DataSourceTypeDAO +from solidui.daos.exceptions import DAONotFound, DAOUpdateFailedError, DAOException, DAODeleteFailedError +from solidui.entity.core import DataSource from solidui.errors import SolidUIErrorType from solidui.solidui_typing import FlaskResponse from solidui.views.base_api import BaseSolidUIApi from flask_appbuilder.api import expose, safe -from solidui.utils.login_utils import get_login_user +from solidui.views.datasource.schemas import DataSourceTypeSchema, DataSourceSchema, DataSourcePageInfoSchema +logger = logging.getLogger(__name__) -class DatasourceRestApi(BaseSolidUIApi): - route_base = "/solidui/datasource" \ No newline at end of file + +class DataSourceRestApi(BaseSolidUIApi): + route_base = "/solidui/datasource" + + @expose('/type/all', methods=("GET",)) + @safe + def get_all_data_source_types(self) -> FlaskResponse: + """ + get all data source types + """ + data_list = DataSourceTypeDAO.find_all() + schema = DataSourceTypeSchema(many=True) + return self.response_format(data=schema.dump(data_list)) + + @expose('/key/type/', methods=("GET",)) + @safe + def get_key_by_type(self, type_id) -> FlaskResponse: + """ + get key definitions by type + """ + data = DataSourceTypeDAO.get_id(type_id) + schema = DataSourceTypeSchema() + return self.response_format(data=schema.dump(data)) + + @expose('/info/json', methods=("POST",)) + @safe + def insert_json_info(self) -> FlaskResponse: + """ + insert json info + """ + data = request.get_json() + data_source = DataSource(**data) + + # Verify data source name and type ID + if not data_source.datasource_name or not data_source.datasource_type_id: + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + + try: + DataSourceDAO.create_data_source(data_source) + return self.response_format() + + except DAONotFound as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.CREATE_DATASOURCE_ERROR) + + @expose('/info//json', methods=("PUT",)) + @safe + def update_json_info(self, data_source_id) -> FlaskResponse: + """ + update data source in json + """ + data = request.get_json() + + if not data: + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + + try: + + data_source = DataSource(**data) + data_source.id = data_source_id # Make sure the ID passed in is set to the data source object + DataSourceDAO.update_data_source(data_source) + return self.response_format() + except DAONotFound as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + except DAOUpdateFailedError as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.UPDATE_DATASOURCE_ERROR) + + @expose('/info/', methods=("GET",)) + @safe + def get_info_by_data_source_id(self, data_source_id) -> FlaskResponse: + """ + get info by data source id + """ + try: + data_source = DataSourceDAO.get_data_source_id(data_source_id) + if data_source is None: + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + + schema = DataSourceSchema() + return self.response_format(data=schema.dump(data_source)) + except DAOException as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.INTERNAL_SERVER_ERROR) + + @expose('/info/name/', methods=("GET",)) + @safe + def get_info_by_data_source_name(self, data_source_name) -> FlaskResponse: + """ + get info by data source name + """ + try: + data_source = DataSourceDAO.get_data_source_name(data_source_name) + if data_source is not None: + schema = DataSourceSchema() + return self.response_format(data=schema.dump(data_source)) + else: + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + except DAOException as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.INTERNAL_SERVER_ERROR) + + @expose('/info/delete/', methods=("DELETE",)) + @safe + def remove_data_source(self, data_source_id) -> FlaskResponse: + """ + get info by data source name + """ + try: + + DataSourceDAO.delete_data_source(data_source_id) + return self.response_format() + + except DAODeleteFailedError as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.DELETE_DATASOURCE_ERROR) + + @expose('/info//expire', methods=("PUT",)) + @safe + def expire_data_source(self, data_source_id) -> FlaskResponse: + """ + expire data source + """ + try: + + DataSourceDAO.exist_data_source(data_source_id) + return self.response_format() + + except DAOUpdateFailedError as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.UPDATE_DATASOURCE_ERROR) + + @expose('/info', methods=("GET",)) + @safe + def query_data_source(self) -> FlaskResponse: + """ + query data source + """ + try: + data_source_name = request.args.get('name', default=None, type=str) + data_source_type_id = request.args.get('typeId', default=None, type=int) + expire = request.args.get('expire', default=None, type=bool) + page_size = request.args.get('pageSize', default=10, type=int) + page_no = request.args.get('pageNo', default=1, type=int) + + # Ensure page size and page number are positive integers + page_size = max(1, page_size) + page_no = max(1, page_no) + + # Query and paginate data sources + page_info = DataSourceDAO.get_data_source_page( + page_no=page_no, + page_size=page_size, + name_filter=data_source_name, + type_filter=data_source_type_id, + expire_filter=expire + ) + + # Format the response + page_info_schema = DataSourcePageInfoSchema() + return self.response_format(data=page_info_schema.dump(page_info)) + + except DAONotFound as ex: + logger.exception(ex) + return self.handle_error(SolidUIErrorType.QUERY_DATASOURCE_ERROR) + + @expose('/connect/json', methods=("POST",)) + @safe + def connect(self) -> FlaskResponse: + """ + connect + """ + data_source_name = request.form.get('dataSourceName') + type_name = request.form.get('typeName', default=None) + + return self.response_format() diff --git a/solidui/views/datasource/schemas.py b/solidui/views/datasource/schemas.py index ae59bead..87495e9b 100644 --- a/solidui/views/datasource/schemas.py +++ b/solidui/views/datasource/schemas.py @@ -11,4 +11,26 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +from marshmallow_sqlalchemy import SQLAlchemyAutoSchema +from solidui.entity.core import DataSourceType, DataSource +from marshmallow_sqlalchemy.schema import Schema +from marshmallow_sqlalchemy.fields import fields + +class DataSourceTypeSchema(SQLAlchemyAutoSchema): + class Meta: + model = DataSourceType + load_instance = True # Optional: if you also want to use it for deserialization + + +class DataSourceSchema(SQLAlchemyAutoSchema): + class Meta: + model = DataSource + load_instance = True # Optional: if you also want to use it for deserialization + +class DataSourcePageInfoSchema(Schema): + current_page = fields.Int() + page_size = fields.Int() + total = fields.Int() + total_page = fields.Int() + total_list = fields.Nested(DataSourceSchema, many=True) diff --git a/solidui/views/project/api.py b/solidui/views/project/api.py index 992109b4..bae68f01 100644 --- a/solidui/views/project/api.py +++ b/solidui/views/project/api.py @@ -28,6 +28,9 @@ class ProjectRestApi(BaseSolidUIApi): @expose('', methods=("POST",)) @safe def create(self) -> FlaskResponse: + """ + create project + """ project_name = request.form.get('projectName') description = request.form.get('description', default='') @@ -50,6 +53,9 @@ def create(self) -> FlaskResponse: @expose('/', methods=("PUT",)) @safe def update(self, pk: int) -> FlaskResponse: + """ + update project + """ project_name = request.form.get('projectName') background_image = request.form.get('backgroundImage') description = request.form.get('description') @@ -65,7 +71,9 @@ def update(self, pk: int) -> FlaskResponse: @expose('/queryProjectListPaging', methods=("GET",)) @safe def query_Project_List_Paging(self) -> FlaskResponse: - + """ + query project list paging + """ search_name = request.args.get('searchName', default='') page_size = request.args.get('pageSize', default=10, type=int) page_no = request.args.get('pageNo', default=1, type=int) @@ -79,14 +87,18 @@ def query_Project_List_Paging(self) -> FlaskResponse: @expose('/', methods=("DELETE",)) @safe def delete(self, pk: int) -> FlaskResponse: - + """ + delete project by id + """ ProjectDAO.delete_project(pk) return self.response_format() @expose('/', methods=("GET",)) @safe def get(self, pk: int) -> FlaskResponse: - + """ + get project by id + """ project = ProjectDAO.get_project(pk) if project: project_schema = ProjectSchema()