Skip to content

Commit

Permalink
[opt](binlog) support rename binlog (apache#37031)
Browse files Browse the repository at this point in the history
Rename operator doesn't have binlog now. This PR will create binlog when
execute rename. The rename operator means :
1. rename table : ALTER TABLE table1 RENAME table2;
2. rename rollup : ALTER TABLE example_table RENAME ROLLUP rollup1
rollup2;
3. rename partition : ALTER TABLE example_table RENAME PARTITION p1 p2;

After SQL analyzing, we can get the old and new table name (rollup name
or partition name), then record the info to binlog, so we can use the
info from binlog.
  • Loading branch information
lsy3993 authored Jul 3, 2024
1 parent 77233b5 commit ad48d00
Show file tree
Hide file tree
Showing 5 changed files with 61 additions and 10 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.doris.persist.ModifyTablePropertyOperationLog;
import org.apache.doris.persist.ReplacePartitionOperationLog;
import org.apache.doris.persist.TableAddOrDropColumnsInfo;
import org.apache.doris.persist.TableInfo;
import org.apache.doris.persist.TruncateTableInfo;
import org.apache.doris.thrift.TBinlog;
import org.apache.doris.thrift.TBinlogType;
Expand Down Expand Up @@ -319,6 +320,16 @@ public void addTruncateTable(TruncateTableInfo info, long commitSeq) {
addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false);
}

public void addTableRename(TableInfo info, long commitSeq) {
long dbId = info.getDbId();
List<Long> tableIds = Lists.newArrayList();
tableIds.add(info.getTableId());
long timestamp = -1;
TBinlogType type = TBinlogType.RENAME_TABLE;
String data = info.toJson();
addBinlog(dbId, tableIds, commitSeq, timestamp, type, data, false);
}

// get binlog by dbId, return first binlog.version > version
public Pair<TStatus, TBinlog> getBinlog(long dbId, long tableId, long prevCommitSeq) {
TStatus status = new TStatus(TStatusCode.OK);
Expand Down
8 changes: 5 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Env.java
Original file line number Diff line number Diff line change
Expand Up @@ -4637,7 +4637,8 @@ public void renameTable(Database db, Table table, String newTableName) throws Dd
db.unregisterTable(oldTableName);
db.registerTable(table);

TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), newTableName);
TableInfo tableInfo = TableInfo.createForTableRename(db.getId(), table.getId(), oldTableName,
newTableName);
editLog.logTableRename(tableInfo);
LOG.info("rename table[{}] to {}", oldTableName, newTableName);
} finally {
Expand Down Expand Up @@ -4824,7 +4825,8 @@ public void renameRollup(Database db, OlapTable table, RollupRenameClause rename
indexNameToIdMap.put(newRollupName, indexId);

// log
TableInfo tableInfo = TableInfo.createForRollupRename(db.getId(), table.getId(), indexId, newRollupName);
TableInfo tableInfo = TableInfo.createForRollupRename(db.getId(), table.getId(), indexId,
rollupName, newRollupName);
editLog.logRollupRename(tableInfo);
LOG.info("rename rollup[{}] to {}", rollupName, newRollupName);
} finally {
Expand Down Expand Up @@ -4883,7 +4885,7 @@ public void renamePartition(Database db, OlapTable table, PartitionRenameClause

// log
TableInfo tableInfo = TableInfo.createForPartitionRename(db.getId(), table.getId(), partition.getId(),
newPartitionName);
partitionName, newPartitionName);
editLog.logPartitionRename(tableInfo);
LOG.info("rename partition[{}] to {}", partitionName, newPartitionName);
} finally {
Expand Down
12 changes: 9 additions & 3 deletions fe/fe-core/src/main/java/org/apache/doris/persist/EditLog.java
Original file line number Diff line number Diff line change
Expand Up @@ -1535,19 +1535,25 @@ public void logDatabaseRename(DatabaseInfo databaseInfo) {
}

public void logTableRename(TableInfo tableInfo) {
logEdit(OperationType.OP_RENAME_TABLE, tableInfo);
long logId = logEdit(OperationType.OP_RENAME_TABLE, tableInfo);
LOG.info("log table rename, logId : {}, infos: {}", logId, tableInfo);
Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId);
}

public void logModifyViewDef(AlterViewInfo alterViewInfo) {
logEdit(OperationType.OP_MODIFY_VIEW_DEF, alterViewInfo);
}

public void logRollupRename(TableInfo tableInfo) {
logEdit(OperationType.OP_RENAME_ROLLUP, tableInfo);
long logId = logEdit(OperationType.OP_RENAME_ROLLUP, tableInfo);
LOG.info("log rollup rename, logId : {}, infos: {}", logId, tableInfo);
Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId);
}

public void logPartitionRename(TableInfo tableInfo) {
logEdit(OperationType.OP_RENAME_PARTITION, tableInfo);
long logId = logEdit(OperationType.OP_RENAME_PARTITION, tableInfo);
LOG.info("log partition rename, logId : {}, infos: {}", logId, tableInfo);
Env.getCurrentEnv().getBinlogManager().addTableRename(tableInfo, logId);
}

public void logColumnRename(TableRenameColumnInfo info) {
Expand Down
39 changes: 35 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/persist/TableInfo.java
Original file line number Diff line number Diff line change
Expand Up @@ -42,10 +42,16 @@ public class TableInfo implements Writable {

@SerializedName("nT")
private String newTableName;
@SerializedName("oT")
private String oldTableName;
@SerializedName("nR")
private String newRollupName;
@SerializedName("oR")
private String oldRollupName;
@SerializedName("nP")
private String newPartitionName;
@SerializedName("oP")
private String oldPartitionName;

public TableInfo() {
// for persist
Expand All @@ -63,17 +69,38 @@ private TableInfo(long dbId, long tableId, long indexId, long partitionId,
this.newPartitionName = newPartitionName;
}

private TableInfo(long dbId, long tableId, long indexId, long partitionId,
String newTableName, String oldTableName, String newRollupName, String oldRollupName,
String newPartitionName, String oldPartitionName) {
this.dbId = dbId;
this.tableId = tableId;
this.indexId = indexId;
this.partitionId = partitionId;

this.newTableName = newTableName;
this.oldTableName = oldTableName;
this.newRollupName = newRollupName;
this.oldRollupName = oldRollupName;
this.newPartitionName = newPartitionName;
this.oldPartitionName = oldPartitionName;
}

public static TableInfo createForTableRename(long dbId, long tableId, String newTableName) {
return new TableInfo(dbId, tableId, -1L, -1L, newTableName, "", "");
}

public static TableInfo createForRollupRename(long dbId, long tableId, long indexId, String newRollupName) {
return new TableInfo(dbId, tableId, indexId, -1L, "", newRollupName, "");
public static TableInfo createForTableRename(long dbId, long tableId, String oldTableName, String newTableName) {
return new TableInfo(dbId, tableId, -1L, -1L, newTableName, oldTableName, "", "", "", "");
}

public static TableInfo createForRollupRename(long dbId, long tableId, long indexId, String oldRollupName,
String newRollupName) {
return new TableInfo(dbId, tableId, indexId, -1L, "", "", newRollupName, oldRollupName, "", "");
}

public static TableInfo createForPartitionRename(long dbId, long tableId, long partitionId,
String newPartitionName) {
return new TableInfo(dbId, tableId, -1L, partitionId, "", "", newPartitionName);
String oldPartitionName, String newPartitionName) {
return new TableInfo(dbId, tableId, -1L, partitionId, "", "", "", "", newPartitionName, oldPartitionName);
}

public static TableInfo createForModifyDistribution(long dbId, long tableId) {
Expand Down Expand Up @@ -134,4 +161,8 @@ public void readFields(DataInput in) throws IOException {
newRollupName = Text.readString(in);
newPartitionName = Text.readString(in);
}

public String toJson() {
return GsonUtils.GSON.toJson(this);
}
}
1 change: 1 addition & 0 deletions gensrc/thrift/FrontendService.thrift
Original file line number Diff line number Diff line change
Expand Up @@ -1143,6 +1143,7 @@ enum TBinlogType {
MODIFY_PARTITIONS = 11,
REPLACE_PARTITIONS = 12,
TRUNCATE_TABLE = 13,
RENAME_TABLE = 14,
}

struct TBinlog {
Expand Down

0 comments on commit ad48d00

Please sign in to comment.