From a3bec06d0f41b4016d22deb5c979a85bf2bbe6d4 Mon Sep 17 00:00:00 2001 From: Shashwat Tiwari Date: Wed, 6 Nov 2024 15:54:42 +0530 Subject: [PATCH 1/6] [Bug][Seatunnel-web][Oracle] Fix fetch table fields for Oracle Jdbc (#236) Signed-off-by: Mohammad Arshad --- .../oracle/jdbc/OracleDataSourceChannel.java | 24 ++++++++++--------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java index 775decb1c..baa408ab6 100644 --- a/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java +++ b/seatunnel-datasource/seatunnel-datasource-plugins/datasource-jdbc-oracle/src/main/java/org/apache/seatunnel/datasource/plugin/oracle/jdbc/OracleDataSourceChannel.java @@ -18,10 +18,10 @@ package org.apache.seatunnel.datasource.plugin.oracle.jdbc; import org.apache.seatunnel.api.configuration.util.OptionRule; +import org.apache.seatunnel.common.utils.SeaTunnelException; import org.apache.seatunnel.datasource.plugin.api.DataSourceChannel; import org.apache.seatunnel.datasource.plugin.api.DataSourcePluginException; import org.apache.seatunnel.datasource.plugin.api.model.TableField; -import org.apache.seatunnel.datasource.plugin.api.utils.JdbcUtils; import org.apache.commons.lang3.StringUtils; @@ -170,10 +170,19 @@ public List getTableFields( @NonNull String database, @NonNull String table) { List tableFields = new ArrayList<>(); - try (Connection connection = getConnection(requestParams, database)) { + try (Connection connection = getConnection(requestParams)) { DatabaseMetaData metaData = connection.getMetaData(); String primaryKey = getPrimaryKey(metaData, database, table); - try (ResultSet resultSet = metaData.getColumns(database, null, table, null)) { + String[] split = table.split("\\."); + if (split.length != 2) { + throw new SeaTunnelException( + "The tableName for oracle must be schemaName.tableName, but tableName is " + + table); + } + + String schemaName = split[0]; + String tableName = split[1]; + try (ResultSet resultSet = metaData.getColumns(database, schemaName, tableName, null)) { while (resultSet.next()) { TableField tableField = new TableField(); String columnName = resultSet.getString("COLUMN_NAME"); @@ -215,16 +224,9 @@ private String getPrimaryKey(DatabaseMetaData metaData, String dbName, String ta private Connection getConnection(Map requestParams) throws SQLException, ClassNotFoundException { - return getConnection(requestParams, null); - } - - private Connection getConnection(Map requestParams, String databaseName) - throws SQLException, ClassNotFoundException { checkNotNull(requestParams.get(OracleOptionRule.DRIVER.key())); checkNotNull(requestParams.get(OracleOptionRule.URL.key()), "Jdbc url cannot be null"); - String url = - JdbcUtils.replaceDatabase( - requestParams.get(OracleOptionRule.URL.key()), databaseName); + String url = requestParams.get(OracleOptionRule.URL.key()); if (requestParams.containsKey(OracleOptionRule.USER.key())) { String username = requestParams.get(OracleOptionRule.USER.key()); String password = requestParams.get(OracleOptionRule.PASSWORD.key()); From d862b6110411fc7a1f9cca82c0d73724a9638f45 Mon Sep 17 00:00:00 2001 From: BilwaST Date: Wed, 6 Nov 2024 16:23:55 +0530 Subject: [PATCH 2/6] [Bug] [Seatunnel-web] Error when conditional column is not used in SELECT clause (#238) Signed-off-by: Mohammad Arshad --- .../service/impl/JobInstanceServiceImpl.java | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java index 8d0b79579..684f529dc 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java @@ -103,6 +103,10 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl private static final String DAG_PARSING_MODE = "dag-parsing.mode"; + private static final String WHERE_CONDITION = "where_condition"; + + private static final String QUERY = "query"; + @Resource private ConnectorDataSourceMapperConfig dataSourceMapperConfig; @Resource private IDatasourceService datasourceService; @@ -233,6 +237,7 @@ public String generateJobConfig( businessMode, config, optionRule); + mergeConfig = appendWhereClauseToQuery(mergeConfig); sourceMap .get(task.getConnectorType()) .add(filterEmptyValue(mergeConfig)); @@ -330,6 +335,19 @@ public String generateJobConfig( return JobUtils.replaceJobConfigPlaceholders(jobConfig, executeParam); } + private Config appendWhereClauseToQuery(Config mergeConfig) { + String where_condition = mergeConfig.getString(WHERE_CONDITION); + if (where_condition != null && !where_condition.isEmpty()) { + String query = mergeConfig.getString(QUERY); + String queryWithWhereClause = query + " " + where_condition; + mergeConfig = + mergeConfig.withValue( + QUERY, ConfigValueFactory.fromAnyRef(queryWithWhereClause)); + mergeConfig = mergeConfig.withoutPath(WHERE_CONDITION); + } + return mergeConfig; + } + @Override public JobExecutorRes getExecuteResource(@NonNull Long jobEngineId) { funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_EXECUTOR_INSTANCE, 0); From 53b5de85c4b2c936d9d764889a75929dc0118e27 Mon Sep 17 00:00:00 2001 From: BilwaST Date: Thu, 7 Nov 2024 07:04:40 +0530 Subject: [PATCH 3/6] [Addendum][Bug] [Seatunnel-web] Error when conditional column is not used in SE (#239) --- .../service/impl/JobInstanceServiceImpl.java | 18 ------------------ .../impl/BaseJdbcDataSourceConfigSwitcher.java | 9 +++++++++ 2 files changed, 9 insertions(+), 18 deletions(-) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java index 684f529dc..8d0b79579 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/service/impl/JobInstanceServiceImpl.java @@ -103,10 +103,6 @@ public class JobInstanceServiceImpl extends SeatunnelBaseServiceImpl private static final String DAG_PARSING_MODE = "dag-parsing.mode"; - private static final String WHERE_CONDITION = "where_condition"; - - private static final String QUERY = "query"; - @Resource private ConnectorDataSourceMapperConfig dataSourceMapperConfig; @Resource private IDatasourceService datasourceService; @@ -237,7 +233,6 @@ public String generateJobConfig( businessMode, config, optionRule); - mergeConfig = appendWhereClauseToQuery(mergeConfig); sourceMap .get(task.getConnectorType()) .add(filterEmptyValue(mergeConfig)); @@ -335,19 +330,6 @@ public String generateJobConfig( return JobUtils.replaceJobConfigPlaceholders(jobConfig, executeParam); } - private Config appendWhereClauseToQuery(Config mergeConfig) { - String where_condition = mergeConfig.getString(WHERE_CONDITION); - if (where_condition != null && !where_condition.isEmpty()) { - String query = mergeConfig.getString(QUERY); - String queryWithWhereClause = query + " " + where_condition; - mergeConfig = - mergeConfig.withValue( - QUERY, ConfigValueFactory.fromAnyRef(queryWithWhereClause)); - mergeConfig = mergeConfig.withoutPath(WHERE_CONDITION); - } - return mergeConfig; - } - @Override public JobExecutorRes getExecuteResource(@NonNull Long jobEngineId) { funcPermissionCheck(SeatunnelFuncPermissionKeyConstant.JOB_EXECUTOR_INSTANCE, 0); diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java index d3cdb8e8b..9c2a21053 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java @@ -61,6 +61,8 @@ public abstract class BaseJdbcDataSourceConfigSwitcher extends AbstractDataSourc private static final String BASE_URL = "base-url"; private static final String CATALOG_SCHEMA = "schema"; + private static final String WHERE_CONDITION = "where_condition"; + private static final Option DATABASE_SCHEMA = Options.key("database_schema") .stringType() @@ -138,6 +140,13 @@ public Config mergeDatasourceConfig( String sql = tableFieldsToSql(tableFields, databaseName, tableName); + String where_condition = connectorConfig.getString(WHERE_CONDITION); + + if (where_condition != null && !where_condition.isEmpty()) { + sql = sql + " " + where_condition; + connectorConfig = connectorConfig.withoutPath(WHERE_CONDITION); + } + connectorConfig = connectorConfig.withValue(QUERY_KEY, ConfigValueFactory.fromAnyRef(sql)); } else if (pluginType.equals(PluginType.SINK)) { From d11f6c5b8c39bbbbcd74c96f5b2d3609cde93af7 Mon Sep 17 00:00:00 2001 From: Mohammad Arshad Date: Mon, 11 Nov 2024 08:13:30 +0530 Subject: [PATCH 4/6] =?UTF-8?q?[Bug]=20[Seatunnel-web]=20No=20configuratio?= =?UTF-8?q?n=20setting=20found=20for=20key=20'where=5Fc=E2=80=A6ondition'?= =?UTF-8?q?=20(#240)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../impl/BaseJdbcDataSourceConfigSwitcher.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java index 9c2a21053..f5136b87b 100644 --- a/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java +++ b/seatunnel-server/seatunnel-app/src/main/java/org/apache/seatunnel/app/thirdparty/datasource/impl/BaseJdbcDataSourceConfigSwitcher.java @@ -139,12 +139,12 @@ public Config mergeDatasourceConfig( List tableFields = selectTableFields.getTableFields(); String sql = tableFieldsToSql(tableFields, databaseName, tableName); - - String where_condition = connectorConfig.getString(WHERE_CONDITION); - - if (where_condition != null && !where_condition.isEmpty()) { - sql = sql + " " + where_condition; - connectorConfig = connectorConfig.withoutPath(WHERE_CONDITION); + if (connectorConfig.hasPath(WHERE_CONDITION)) { + String where_condition = connectorConfig.getString(WHERE_CONDITION); + if (where_condition != null && !where_condition.isEmpty()) { + sql = sql + " " + where_condition; + connectorConfig = connectorConfig.withoutPath(WHERE_CONDITION); + } } connectorConfig = From 72be9252c86b99a07d282d32a91e172d74627610 Mon Sep 17 00:00:00 2001 From: susanbushisan <39783183+susanbushisan@users.noreply.github.com> Date: Mon, 9 Dec 2024 01:39:53 +0800 Subject: [PATCH 5/6] [Bug] [Seatunnel-web] fix build prod (#241) Co-authored-by: susanbushisan --- .../task/synchronization-definition/dag/sidebar/index.tsx | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/seatunnel-ui/src/views/task/synchronization-definition/dag/sidebar/index.tsx b/seatunnel-ui/src/views/task/synchronization-definition/dag/sidebar/index.tsx index cf05a971b..79aff2c3f 100644 --- a/seatunnel-ui/src/views/task/synchronization-definition/dag/sidebar/index.tsx +++ b/seatunnel-ui/src/views/task/synchronization-definition/dag/sidebar/index.tsx @@ -72,7 +72,7 @@ const DagSidebar = defineComponent({ class="task-item-info ml-auto inline-block" title={'Drag Source into Canvas and Double Click to Setup Configurations'} > - +
- +
{this.transforms.length > 0 && ( @@ -133,7 +133,7 @@ const DagSidebar = defineComponent({ class="task-item-info ml-auto inline-block" title={'Drag '+ item.name +' into Canvas and Double Click to Setup Configurations'} > - + ) From 0fb6a65f1697028c7f7982c16fcc6e2c548b7e6b Mon Sep 17 00:00:00 2001 From: xiaochen <598457447@qq.com> Date: Mon, 9 Dec 2024 15:48:10 +0800 Subject: [PATCH 6/6] [Bug] [Seatunnel-web] Fix status always not runnning (#246) --- .../src/main/bin/seatunnel-backend-daemon.sh | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh b/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh index d73b06374..98506c8d2 100755 --- a/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh +++ b/seatunnel-server/seatunnel-app/src/main/bin/seatunnel-backend-daemon.sh @@ -35,7 +35,7 @@ check() { # start start() { - echo "starting seatunnel..." + echo "starting seatunnel-web..." check @@ -58,27 +58,27 @@ start() { -cp "$WORKDIR/../conf":"$WORKDIR/../libs/*":"$WORKDIR/../datasource/*" \ $SPRING_OPTS \ org.apache.seatunnel.app.SeatunnelApplication >> "${LOGDIR}/seatunnel.out" 2>&1 & - echo "seatunnel started" + echo "seatunnel-web started" } # stop stop() { - echo "stopping seatunnel..." + echo "stopping seatunnel-web..." pid=$(jcmd | grep -i 'org.apache.seatunnel.app.SeatunnelApplication' | grep -v grep | awk '{print $1}') if [ -n "$pid" ]; then kill -15 $pid - echo "seatunnel stopped" + echo "seatunnel-web stopped" else - echo "seatunnel is not running" + echo "seatunnel-web is not running" fi } #status status() { - pid=$(jcmd | grep -i 'seatunnel-app-.*jar' | grep -v grep | awk '{print $1}') + pid=$(jcmd | grep -i 'org.apache.seatunnel.app.SeatunnelApplication' | grep -v grep | awk '{print $1}') if [ -n "$pid" ]; then - echo "seatunnel is running" + echo "seatunnel-web is running" else - echo "seatunnel is not running" + echo "seatunnel-web is not running" fi }