Skip to content

Commit

Permalink
fix(osc): fix create osc task ddl contains unique key but oms prechec…
Browse files Browse the repository at this point in the history
…k table not found (#165)

* valid nullable unique key

* 1.modify uk check failed 18n error message

* 1.method rename
  • Loading branch information
krihy committed Sep 5, 2023
1 parent aff12d9 commit e89e447
Show file tree
Hide file tree
Showing 7 changed files with 225 additions and 12 deletions.
Original file line number Diff line number Diff line change
@@ -0,0 +1,125 @@
/*
* Copyright (c) 2023 OceanBase.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package com.oceanbase.odc.service.onlineschemachange;

import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.mock.mockito.MockBean;

import com.oceanbase.odc.ServiceTestEnv;
import com.oceanbase.odc.TestConnectionUtil;
import com.oceanbase.odc.core.session.ConnectionSession;
import com.oceanbase.odc.core.session.ConnectionSessionConstants;
import com.oceanbase.odc.core.shared.constant.ConnectType;
import com.oceanbase.odc.core.shared.exception.UnsupportedException;
import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor;
import com.oceanbase.odc.service.connection.ConnectionService;
import com.oceanbase.odc.service.connection.model.ConnectionConfig;
import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq;
import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters;
import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType;
import com.oceanbase.odc.service.onlineschemachange.validator.OnlineSchemaChangeValidator;

public class OnlineSchemaChangeValidatorOBOracleTest extends ServiceTestEnv {

@Autowired
private OnlineSchemaChangeValidator validService;
@MockBean
private ConnectionService connectionService;
private ConnectionConfig config;
private SyncJdbcExecutor obMySqlSyncJdbcExecutor;


@Before
public void setUp() {

config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_ORACLE);
ConnectionSession session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_ORACLE);
Mockito.when(connectionService.getForConnectionSkipPermissionCheck(Mockito.anyLong())).thenReturn(config);
obMySqlSyncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY);
}


@Test
public void TestUniqueNotNullOBOracle_Successfully() {
String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY (\n"
+ "col number NOT NULL,\n"
+ "col1 number DEFAULT NULL,\n"
+ "CONSTRAINT u1 UNIQUE (col)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY");
}

}

@Test(expected = UnsupportedException.class)
public void TestUniqueContainNotNullOBOracle_Failed() {
String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY2 (\n"
+ "col number NOT NULL,\n"
+ "col1 number DEFAULT NULL,\n"
+ "CONSTRAINT u1 UNIQUE (col,col1)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY2");
}

}

@Test(expected = UnsupportedException.class)
public void TestUniqueColumnNotNullOBOracle_Failed() {
String createSql = "CREATE TABLE NOT_NULL_UNIQUE_KEY3 (\n"
+ "col number NOT NULL,\n"
+ "col1 number DEFAULT NULL,\n"
+ "CONSTRAINT u1 UNIQUE (col1)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE NOT_NULL_UNIQUE_KEY3");
}

}


private CreateFlowInstanceReq getCreateRequest(String sql, OnlineSchemaChangeSqlType sqlType) {
OnlineSchemaChangeParameters parameter = new OnlineSchemaChangeParameters();
parameter.setSqlType(sqlType);
parameter.setSqlContent(sql);
parameter.setDelimiter(";");
CreateFlowInstanceReq req = new CreateFlowInstanceReq();
req.setParameters(parameter);
req.setConnectionId(1L);
req.setDatabaseName(config.defaultSchema());
return req;
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,8 @@
import com.oceanbase.odc.core.shared.constant.ConnectType;
import com.oceanbase.odc.core.shared.constant.ErrorCodes;
import com.oceanbase.odc.core.shared.exception.BadArgumentException;
import com.oceanbase.odc.core.shared.exception.UnsupportedException;
import com.oceanbase.odc.core.sql.execute.SyncJdbcExecutor;
import com.oceanbase.odc.service.connection.ConnectionService;
import com.oceanbase.odc.service.connection.model.ConnectionConfig;
import com.oceanbase.odc.service.flow.model.CreateFlowInstanceReq;
Expand All @@ -49,6 +51,7 @@ public class OnlineSchemaChangeValidatorTest extends ServiceTestEnv {
private ConnectionService connectionService;
private ConnectionSession session;
private ConnectionConfig config;
private SyncJdbcExecutor obMySqlSyncJdbcExecutor;


@Before
Expand All @@ -63,8 +66,8 @@ public void setUp() {
config = TestConnectionUtil.getTestConnectionConfig(ConnectType.OB_MYSQL);
session = TestConnectionUtil.getTestConnectionSession(ConnectType.OB_MYSQL);
Mockito.when(connectionService.getForConnectionSkipPermissionCheck(Mockito.anyLong())).thenReturn(config);

session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY).execute(CREATE_STMT);
obMySqlSyncJdbcExecutor = session.getSyncJdbcExecutor(ConnectionSessionConstants.CONSOLE_DS_KEY);
obMySqlSyncJdbcExecutor.execute(CREATE_STMT);
}

@After
Expand Down Expand Up @@ -115,6 +118,61 @@ public void test_Validate_Invalid_Sql() {
}
}

@Test
public void TestUniqueNotNullOBMySql_Successfully() {
String createSql = "CREATE TABLE `not_null_unique_key` (\n"
+ "`col` int NOT NULL,\n"
+ "`col1` int DEFAULT NULL,\n"
+ "CONSTRAINT `u1` UNIQUE (`col`)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key`");
}

}

@Test(expected = UnsupportedException.class)
public void TestUniqueContainNotNullOBMySql_Failed() {
String createSql = "CREATE TABLE `not_null_unique_key2` (\n"
+ "`col` int NOT NULL,\n"
+ "`col1` int DEFAULT NULL,\n"
+ "CONSTRAINT `u1` UNIQUE (`col`,`col1`)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key2`");
}

}

@Test(expected = UnsupportedException.class)
public void TestUniqueColumnNotNullOBMySql_Failed() {
String createSql = "CREATE TABLE `not_null_unique_key3` (\n"
+ "`col` int NOT NULL,\n"
+ "`col1` int DEFAULT NULL,\n"
+ "CONSTRAINT `u1` UNIQUE (`col1`)\n"
+ ")";
obMySqlSyncJdbcExecutor.execute(createSql);
try {
validService.validate(getCreateRequest(
createSql,
OnlineSchemaChangeSqlType.CREATE));
} finally {
obMySqlSyncJdbcExecutor.execute("DROP TABLE IF EXISTS `not_null_unique_key3`");
}

}


private CreateFlowInstanceReq getCreateRequest(String sql, OnlineSchemaChangeSqlType sqlType) {
OnlineSchemaChangeParameters parameter = new OnlineSchemaChangeParameters();
parameter.setSqlType(sqlType);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -132,7 +132,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=Insufficient auth
com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=Too many permissions, details: {0}
com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=Export Excel failed, we recommend you to export CSV file, error details:{0}
com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=The input sql type {0} is inconsistent with expected type {1}, sql: {2}
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=There is no primary key or unique key exists in table {0}
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=There is no primary key or not nullable unique key exists in table {0}
com.oceanbase.odc.ErrorCodes.OscNotEnabled=Online schema change is not enabled on current instance {0}, please contact administrator
com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=Online schema change data check is inconsistent
com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) data check is inconsistent
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -131,7 +131,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=权限不足,
com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=过多的权限,权限详情:{0}
com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=导出 Excel 文件失败,推荐导出 CSV 格式的文件,错误详情:{0}
com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=输入的 Sql 类型 {0} 与预期的 Sql 类型 {1} 不一致,sql:{2}
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中没有主键索引或唯一索引
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中没有主键索引或非空唯一索引
com.oceanbase.odc.ErrorCodes.OscNotEnabled=当前集群 {0} 未开启使用无锁结构变更功能,请联系管理员
com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=无锁结构变更数据校验不一致
com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) 数据校验不一致
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -130,7 +130,7 @@ com.oceanbase.odc.ErrorCodes.ConnectionInsufficientPermissions=權限不足,
com.oceanbase.odc.ErrorCodes.ConnectionTooManyPermissions=過多的權限,權限詳情:{0}
com.oceanbase.odc.ErrorCodes.ExportExcelFileFailed=導出 Excel 文件失敗,推薦導出 CSV 格式的文件,錯誤詳情:{0}
com.oceanbase.odc.ErrorCodes.OscSqlTypeInconsistent=輸入的 Sql 類型 {0} 與預期的 Sql 類型 {1} 不一致,sql:{2}
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中沒有主鍵索引或唯一索引
com.oceanbase.odc.ErrorCodes.NoUniqueKeyExists=表 {0} 中沒有主鍵索引或非空唯壹索引
com.oceanbase.odc.ErrorCodes.OscNotEnabled=當前集群 {0} 未開啟使用無鎖結構變更功能,請聯系管理員
com.oceanbase.odc.ErrorCodes.OscDataCheckInconsistent=無鎖結構變更數據校驗不壹致
com.oceanbase.odc.ErrorCodes.OmsDataCheckInconsistent=OceanBase Migration Service (OMS) 數據校驗不壹致
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -32,7 +32,7 @@ public class CommonTransferConfig {
*
* WITHOUT_UNIQUE_ROW_ID 没有唯一行标识的表。WITH_UNIQUE_ROW_ID 的反集。;
*/
private String tableCategory = "WITH_UNIQUE_ROW_ID";
private String tableCategory = "ALL";
/**
* 是否双活场景
*/
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,10 +15,9 @@
*/
package com.oceanbase.odc.service.onlineschemachange.validator;

import static com.oceanbase.tools.dbbrowser.model.DBIndexType.UNIQUE;

import java.io.StringReader;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;

import org.apache.commons.collections.CollectionUtils;
Expand Down Expand Up @@ -47,7 +46,9 @@
import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeParameters;
import com.oceanbase.odc.service.onlineschemachange.model.OnlineSchemaChangeSqlType;
import com.oceanbase.odc.service.session.factory.DefaultConnectSessionFactory;
import com.oceanbase.tools.dbbrowser.model.DBTableIndex;
import com.oceanbase.tools.dbbrowser.model.DBConstraintType;
import com.oceanbase.tools.dbbrowser.model.DBTableColumn;
import com.oceanbase.tools.dbbrowser.model.DBTableConstraint;
import com.oceanbase.tools.dbbrowser.schema.DBSchemaAccessor;
import com.oceanbase.tools.sqlparser.OBMySQLParser;
import com.oceanbase.tools.sqlparser.OBOracleSQLParser;
Expand Down Expand Up @@ -173,10 +174,39 @@ private void validateOldTableNotExists(String database, String tableName, Connec

private void validateTableConstraints(String database, String tableName, ConnectionSession session) {
DBSchemaAccessor accessor = DBSchemaAccessors.create(session);
List<DBTableIndex> indexes = accessor.listTableIndexes(database, DdlUtils.getUnwrappedName(tableName));
if (indexes.stream().noneMatch(index -> index.getType() == UNIQUE)) {
List<DBTableConstraint> constraints =
accessor.listTableConstraints(database, DdlUtils.getUnwrappedName(tableName));
if (CollectionUtils.isEmpty(constraints)) {
throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName},
"There is no primary key or not nullable unique key in table " + tableName);
}
if (constraints.stream().anyMatch(index -> index.getType() == DBConstraintType.PRIMARY_KEY)) {
return;
}
// Check unique key reference columns is not null
List<DBTableConstraint> uniques = constraints.stream()
.filter(c -> c.getType() == DBConstraintType.UNIQUE_KEY).collect(Collectors.toList());

if (CollectionUtils.isEmpty(uniques)) {
throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName},
"There is no primary key or not nullable unique key in table " + tableName);
}

validateUniqueKeyIsConstraintNullable(database, tableName, session, uniques);
}

private void validateUniqueKeyIsConstraintNullable(String database, String tableName, ConnectionSession session,
List<DBTableConstraint> uniques) {
Map<String, DBTableColumn> dbTableColumns =
DBSchemaAccessors.create(session).listTableColumns(database,
DdlUtils.getUnwrappedName(tableName)).stream().collect(
Collectors.toMap(DBTableColumn::getName, v -> v));
boolean existsNullableColumnUk = uniques.stream().anyMatch(
uk -> uk.getColumnNames().stream().noneMatch(column -> dbTableColumns.get(column).getNullable()));

if (!existsNullableColumnUk) {
throw new UnsupportedException(ErrorCodes.NoUniqueKeyExists, new Object[] {tableName},
"There is no primary key or unique key exists in table " + tableName);
"There is no primary key or not nullable unique key in table " + tableName);
}
}

Expand Down

0 comments on commit e89e447

Please sign in to comment.