From e89e447f117ce686d29ce266b015f5f163cb01be Mon Sep 17 00:00:00 2001 From: yaobin <51393259+krihy@users.noreply.github.com> Date: Tue, 5 Sep 2023 21:43:17 +0800 Subject: [PATCH] fix(osc): fix create osc task ddl contains unique key but oms precheck table not found (#165) * valid nullable unique key * 1.modify uk check failed 18n error message * 1.method rename --- ...lineSchemaChangeValidatorOBOracleTest.java | 125 ++++++++++++++++++ .../OnlineSchemaChangeValidatorTest.java | 62 ++++++++- .../resources/i18n/ErrorMessages.properties | 2 +- .../i18n/ErrorMessages_zh_CN.properties | 2 +- .../i18n/ErrorMessages_zh_TW.properties | 2 +- .../oms/request/CommonTransferConfig.java | 2 +- .../OnlineSchemaChangeValidator.java | 42 +++++- 7 files changed, 225 insertions(+), 12 deletions(-) create mode 100644 server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorOBOracleTest.java diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorOBOracleTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorOBOracleTest.java new file mode 100644 index 0000000000..5bdcdf7d51 --- /dev/null +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorOBOracleTest.java @@ -0,0 +1,125 @@ +/* + * Copyright (c) 2023 OceanBase. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.oceanbase.odc.service.onlineschemachange; + +import org.junit.Before; +import org.junit.Test; +import org.mockito.Mockito; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.boot.test.mock.mockito.MockBean; + +import com.oceanbase.odc.ServiceTestEnv; +import com.oceanbase.odc.TestConnectionUtil; +import com.oceanbase.odc.core.session.ConnectionSession; +import com.oceanbase.odc.core.session.ConnectionSessionConstants; +import com.oceanbase.odc.core.shared.constant.ConnectType; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; +import com.oceanbase.odc.service.connection.ConnectionService; +import com.oceanbase.odc.service.connection.model.ConnectionConfig; +import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; +import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType; +import com.oceanbase.odc.service.onlineschemachange.validator.OnlineSchemaChangeValidator; + +public class OnlineSchemaChangeValidatorOBOracleTest extends ServiceTestEnv { + + @Autowired + private OnlineSchemaChangeValidator validService; + @MockBean + private ConnectionService connectionService; + private ConnectionConfig config; + private SyncJdbcExecutor obMySqlSyncJdbcExecutor; + + + @Before + public void setUp() { + + config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_ORACLE); + ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_ORACLE); + Mockito.when(connectionService.getForConnectionSkipPermissionCheck(Mockito.anyLong())).thenReturn(config); + obMySqlSyncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + } + + + @Test + public void TestUniqueNotNullOBOracle_Successfully() { + String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY (\n" + + "col number NOT NULL,\n" + + "col1 number DEFAULT NULL,\n" + + "CONSTRAINT u1 UNIQUE (col)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY"); + } + + } + + @Test(expected = UnsupportedException.class) + public void TestUniqueContainNotNullOBOracle_Failed() { + String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY2 (\n" + + "col number NOT NULL,\n" + + "col1 number DEFAULT NULL,\n" + + "CONSTRAINT u1 UNIQUE (col,col1)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY2"); + } + + } + + @Test(expected = UnsupportedException.class) + public void TestUniqueColumnNotNullOBOracle_Failed() { + String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY3 (\n" + + "col number NOT NULL,\n" + + "col1 number DEFAULT NULL,\n" + + "CONSTRAINT u1 UNIQUE (col1)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY3"); + } + + } + + + private CreateFlowInstanceReq getCreateRequest(String sql, OnlineSchemaChangeSqlType sqlType) { + OnlineSchemaChangeParameters parameter = new OnlineSchemaChangeParameters(); + parameter.setSqlType(sqlType); + parameter.setSqlContent(sql); + parameter.setDelimiter(";"); + CreateFlowInstanceReq req = new CreateFlowInstanceReq(); + req.setParameters(parameter); + req.setConnectionId(1L); + req.setDatabaseName(config.defaultSchema()); + return req; + } + +} diff --git a/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorTest.java b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorTest.java index 8dd9d0f181..9ce5a78078 100644 --- a/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorTest.java +++ b/server/integration-test/src/test/java/com/oceanbase/odc/service/onlineschemachange/OnlineSchemaChangeValidatorTest.java @@ -31,6 +31,8 @@ import com.oceanbase.odc.core.shared.constant.ConnectType; import com.oceanbase.odc.core.shared.constant.ErrorCodes; import com.oceanbase.odc.core.shared.exception.BadArgumentException; +import com.oceanbase.odc.core.shared.exception.UnsupportedException; +import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor; import com.oceanbase.odc.service.connection.ConnectionService; import com.oceanbase.odc.service.connection.model.ConnectionConfig; import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq; @@ -49,6 +51,7 @@ public class OnlineSchemaChangeValidatorTest extends ServiceTestEnv { private ConnectionService connectionService; private ConnectionSession session; private ConnectionConfig config; + private SyncJdbcExecutor obMySqlSyncJdbcExecutor; @Before @@ -63,8 +66,8 @@ public void setUp() { config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_MYSQL); session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL); Mockito.when(connectionService.getForConnectionSkipPermissionCheck(Mockito.anyLong())).thenReturn(config); - - session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(CREATE_STMT); + obMySqlSyncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY); + obMySqlSyncJdbcExecutor.execute(CREATE_STMT); } @After @@ -115,6 +118,61 @@ public void test_Validate_Invalid_Sql() { } } + @Test + public void TestUniqueNotNullOBMySql_Successfully() { + String createSql = "CREATE TABLE `not_null_unique_key` (\n" + + "`col` int NOT NULL,\n" + + "`col1` int DEFAULT NULL,\n" + + "CONSTRAINT `u1` UNIQUE (`col`)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key`"); + } + + } + + @Test(expected = UnsupportedException.class) + public void TestUniqueContainNotNullOBMySql_Failed() { + String createSql = "CREATE TABLE `not_null_unique_key2` (\n" + + "`col` int NOT NULL,\n" + + "`col1` int DEFAULT NULL,\n" + + "CONSTRAINT `u1` UNIQUE (`col`,`col1`)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key2`"); + } + + } + + @Test(expected = UnsupportedException.class) + public void TestUniqueColumnNotNullOBMySql_Failed() { + String createSql = "CREATE TABLE `not_null_unique_key3` (\n" + + "`col` int NOT NULL,\n" + + "`col1` int DEFAULT NULL,\n" + + "CONSTRAINT `u1` UNIQUE (`col1`)\n" + + ")"; + obMySqlSyncJdbcExecutor.execute(createSql); + try { + validService.validate(getCreateRequest( + createSql, + OnlineSchemaChangeSqlType.CREATE)); + } finally { + obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key3`"); + } + + } + + private CreateFlowInstanceReq getCreateRequest(String sql, OnlineSchemaChangeSqlType sqlType) { OnlineSchemaChangeParameters parameter = new OnlineSchemaChangeParameters(); parameter.setSqlType(sqlType); diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties index 9c057bcec1..bce74408bc 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages.properties @@ -132,7 +132,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=Insufficient auth com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=Too many permissions, details: {0} com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=Export Excel failed, we recommend you to export CSV file, error details：{0} com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=The input sql type {0} is inconsistent with expected type {1}, sql: {2} -com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=There is no primary key or unique key exists in table {0} +com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=There is no primary key or not nullable unique key exists in table {0} com.oceanbase.odc.ErrorCodes.OscNotEnabled=Online schema change is not enabled on current instance {0}, please contact administrator com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=Online schema change data check is inconsistent com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) data check is inconsistent diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties index 75a117a314..c942019e1d 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_CN.properties @@ -131,7 +131,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=权限不足， com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=过多的权限，权限详情：{0} com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=导出 Excel 文件失败，推荐导出 CSV 格式的文件，错误详情：{0} com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=输入的 Sql 类型 {0} 与预期的 Sql 类型 {1} 不一致，sql：{2} -com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中没有主键索引或唯一索引 +com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中没有主键索引或非空唯一索引 com.oceanbase.odc.ErrorCodes.OscNotEnabled=当前集群 {0} 未开启使用无锁结构变更功能，请联系管理员 com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=无锁结构变更数据校验不一致 com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) 数据校验不一致 diff --git a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties index 31de2b9ca4..9c07880bbd 100644 --- a/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties +++ b/server/odc-core/src/main/resources/i18n/ErrorMessages_zh_TW.properties @@ -130,7 +130,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=權限不足， com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=過多的權限，權限詳情：{0} com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=導出 Excel 文件失敗，推薦導出 CSV 格式的文件，錯誤詳情：{0} com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=輸入的 Sql 類型 {0} 與預期的 Sql 類型 {1} 不一致，sql：{2} -com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中沒有主鍵索引或唯一索引 +com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中沒有主鍵索引或非空唯壹索引 com.oceanbase.odc.ErrorCodes.OscNotEnabled=當前集群 {0} 未開啟使用無鎖結構變更功能，請聯系管理員 com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=無鎖結構變更數據校驗不壹致 com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) 數據校驗不壹致 diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/CommonTransferConfig.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/CommonTransferConfig.java index 16abeb6138..cc559e00a9 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/CommonTransferConfig.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/oms/request/CommonTransferConfig.java @@ -32,7 +32,7 @@ public class CommonTransferConfig { * * WITHOUT_UNIQUE_ROW_ID 没有唯一行标识的表。WITH_UNIQUE_ROW_ID 的反集。; */ - private String tableCategory = "WITH_UNIQUE_ROW_ID"; + private String tableCategory = "ALL"; /** * 是否双活场景 */ diff --git a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java index fcf1a0a047..dc65cddffd 100644 --- a/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java +++ b/server/odc-service/src/main/java/com/oceanbase/odc/service/onlineschemachange/validator/OnlineSchemaChangeValidator.java @@ -15,10 +15,9 @@ */ package com.oceanbase.odc.service.onlineschemachange.validator; -import static com.oceanbase.tools.dbbrowser.model.DBIndexType.UNIQUE; - import java.io.StringReader; import java.util.List; +import java.util.Map; import java.util.stream.Collectors; import org.apache.commons.collections.CollectionUtils; @@ -47,7 +46,9 @@ import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters; import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType; import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory; -import com.oceanbase.tools.dbbrowser.model.DBTableIndex; +import com.oceanbase.tools.dbbrowser.model.DBConstraintType; +import com.oceanbase.tools.dbbrowser.model.DBTableColumn; +import com.oceanbase.tools.dbbrowser.model.DBTableConstraint; import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor; import com.oceanbase.tools.sqlparser.OBMySQLParser; import com.oceanbase.tools.sqlparser.OBOracleSQLParser; @@ -173,10 +174,39 @@ private void validateOldTableNotExists(String database, String tableName, Connec private void validateTableConstraints(String database, String tableName, ConnectionSession session) { DBSchemaAccessor accessor = DBSchemaAccessors.create(session); - List indexes = accessor.listTableIndexes(database, DdlUtils.getUnwrappedName(tableName)); - if (indexes.stream().noneMatch(index -> index.getType() == UNIQUE)) { + List constraints = + accessor.listTableConstraints(database, DdlUtils.getUnwrappedName(tableName)); + if (CollectionUtils.isEmpty(constraints)) { + throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName}, + "There is no primary key or not nullable unique key in table " + tableName); + } + if (constraints.stream().anyMatch(index -> index.getType() == DBConstraintType.PRIMARY_KEY)) { + return; + } + // Check unique key reference columns is not null + List uniques = constraints.stream() + .filter(c -> c.getType() == DBConstraintType.UNIQUE_KEY).collect(Collectors.toList()); + + if (CollectionUtils.isEmpty(uniques)) { + throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName}, + "There is no primary key or not nullable unique key in table " + tableName); + } + + validateUniqueKeyIsConstraintNullable(database, tableName, session, uniques); + } + + private void validateUniqueKeyIsConstraintNullable(String database, String tableName, ConnectionSession session, + List uniques) { + Map dbTableColumns = + DBSchemaAccessors.create(session).listTableColumns(database, + DdlUtils.getUnwrappedName(tableName)).stream().collect( + Collectors.toMap(DBTableColumn::getName, v -> v)); + boolean existsNullableColumnUk = uniques.stream().anyMatch( + uk -> uk.getColumnNames().stream().noneMatch(column -> dbTableColumns.get(column).getNullable())); + + if (!existsNullableColumnUk) { throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName}, - "There is no primary key or unique key exists in table " + tableName); + "There is no primary key or not nullable unique key in table " + tableName); } }