Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[Enhancement] Support move truncated old data to recycle bin #43107

Merged
merged 2 commits into from
Nov 28, 2024
Merged

[Enhancement] Support move truncated old data to recycle bin #43107

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
246674f
[Enhancement] Support move truncated old data to recycle bin
Vallishp Nov 1, 2024
65b69a8
Fix issue
Vallishp Nov 25, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -865,7 +865,7 @@ unsupportedUseStatement
;

unsupportedDmlStatement
: TRUNCATE TABLE multipartIdentifier specifiedPartition? #truncateTable
: TRUNCATE TABLE multipartIdentifier specifiedPartition? FORCE? #truncateTable
| COPY INTO name=multipartIdentifier columns=identifierList? FROM
(stageAndPattern | (LEFT_PAREN SELECT selectColumnClause
FROM stageAndPattern whereClause? RIGHT_PAREN))
Expand Down
4 changes: 2 additions & 2 deletions fe/fe-core/src/main/cup/sql_parser.cup
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7976,9 +7976,9 @@ admin_stmt ::=
;

truncate_stmt ::=
KW_TRUNCATE KW_TABLE base_table_ref:tblRef
KW_TRUNCATE KW_TABLE base_table_ref:tblRef opt_force:force
{:
RESULT = new TruncateTableStmt(tblRef);
RESULT = new TruncateTableStmt(tblRef, force);
:}
;

Expand Down
14 changes: 13 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/analysis/TruncateTableStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,15 +30,21 @@
public class TruncateTableStmt extends DdlStmt implements NotFallbackInParser {

private TableRef tblRef;
private boolean forceDrop;

public TruncateTableStmt(TableRef tblRef) {
public TruncateTableStmt(TableRef tblRef, boolean forceDrop) {
this.tblRef = tblRef;
this.forceDrop = forceDrop;
}

public TableRef getTblRef() {
return tblRef;
}

public boolean isForceDrop() {
return forceDrop;
}
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

handle force in toSQL.


@Override
public void analyze(Analyzer analyzer) throws AnalysisException, UserException {
super.analyze(analyzer);
Expand Down Expand Up @@ -75,6 +81,9 @@ public String toSql() {
if (tblRef.getPartitionNames() != null) {
sb.append(tblRef.getPartitionNames().toSql());
}
if (isForceDrop()) {
sb.append(" FORCE");
}
return sb.toString();
}

Expand All @@ -83,6 +92,9 @@ public String toSqlWithoutTable() {
if (tblRef.getPartitionNames() != null) {
sb.append(tblRef.getPartitionNames().toSql());
}
if (isForceDrop()) {
sb.append(" FORCE");
}
return sb.toString();
}

Expand Down
154 changes: 96 additions & 58 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1127,63 +1127,9 @@ private Partition dropPartition(long dbId, String partitionName, boolean isForce
if (partition != null) {
idToPartition.remove(partition.getId());
nameToPartition.remove(partitionName);

if (!isForceDrop) {
// recycle partition
if (partitionInfo.getType() == PartitionType.RANGE) {
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
partitionInfo.getItem(partition.getId()).getItems(),
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));

} else if (partitionInfo.getType() == PartitionType.LIST) {
// construct a dummy range
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);

Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
partitionInfo.getItem(partition.getId()),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));
} else {
// unpartition
// construct a dummy range and dummy list.
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
partitionInfo.getDataProperty(partition.getId()),
partitionInfo.getReplicaAllocation(partition.getId()),
partitionInfo.getIsInMemory(partition.getId()),
partitionInfo.getIsMutable(partition.getId()));
}
} else if (!reserveTablets) {
Env.getCurrentEnv().onErasePartition(partition);
}

// drop partition info
partitionInfo.dropPartition(partition.getId());
RecyclePartitionParam recyclePartitionParam = new RecyclePartitionParam();
fillInfo(partition, recyclePartitionParam);
dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, reserveTablets);
}
return partition;
}
Expand All @@ -1196,6 +1142,81 @@ public Partition dropPartition(long dbId, String partitionName, boolean isForceD
return dropPartition(dbId, partitionName, isForceDrop, !isForceDrop);
}

private void dropPartitionCommon(long dbId, boolean isForceDrop,
RecyclePartitionParam recyclePartitionParam,
Partition partition,
boolean reserveTablets) {
if (!isForceDrop) {
// recycle partition
if (partitionInfo.getType() == PartitionType.RANGE) {
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
recyclePartitionParam.partitionItem.getItems(),
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);

} else if (partitionInfo.getType() == PartitionType.LIST) {
// construct a dummy range
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);

Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
recyclePartitionParam.partitionItem,
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);
} else {
// unpartition
// construct a dummy range and dummy list.
List<Column> dummyColumns = new ArrayList<>();
dummyColumns.add(new Column("dummy", PrimitiveType.INT));
PartitionKey dummyKey = null;
try {
dummyKey = PartitionKey.createInfinityPartitionKey(dummyColumns, false);
} catch (AnalysisException e) {
LOG.warn("should not happen", e);
}
Range<PartitionKey> dummyRange = Range.open(new PartitionKey(), dummyKey);
Env.getCurrentRecycleBin().recyclePartition(dbId, id, name, partition,
dummyRange,
new ListPartitionItem(Lists.newArrayList(new PartitionKey())),
recyclePartitionParam.dataProperty,
recyclePartitionParam.replicaAlloc,
recyclePartitionParam.isInMemory,
recyclePartitionParam.isMutable);
}
} else if (!reserveTablets) {
Env.getCurrentEnv().onErasePartition(partition);
}

// drop partition info
partitionInfo.dropPartition(partition.getId());
}

public Partition dropPartitionForTruncate(long dbId, boolean isForceDrop,
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

OlapTable has a function dropPartition, its code seem much like this, can refactor code to reuse them ?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

thank you. i will refactor it

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, this had resolved

RecyclePartitionParam recyclePartitionParam) {
// 1. If "isForceDrop" is false, the partition will be added to the Catalog Recyle bin, and all tablets of this
// partition will not be deleted.
// 2. If "ifForceDrop" is true, the partition will be dropped immediately
Partition partition = recyclePartitionParam.partition;
if (partition != null) {
idToPartition.remove(partition.getId());
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

need add nameToPartition.remove(partitionName), just like OlapTable.dropPartition function.

so refactor dropPartition and dropPartitionForTruncate to reuse code;

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

replacePartition had already removed old partition from nameToPartition and added new partition created by truncate process. so it should not be called in truncate process.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, this had resolved

dropPartitionCommon(dbId, isForceDrop, recyclePartitionParam, partition, false);
}
return partition;
}

/*
* A table may contain both formal and temporary partitions.
* There are several methods to get the partition of a table.
Expand Down Expand Up @@ -2027,13 +2048,24 @@ public static OlapTable read(DataInput in) throws IOException {
return GsonUtils.GSON.fromJson(Text.readString(in), OlapTable.class);
}


public void fillInfo(Partition partition, RecyclePartitionParam recyclePartitionParam) {
recyclePartitionParam.dataProperty = partitionInfo.getDataProperty(partition.getId());
recyclePartitionParam.replicaAlloc = partitionInfo.getReplicaAllocation(partition.getId());
recyclePartitionParam.isInMemory = partitionInfo.getIsInMemory(partition.getId());
recyclePartitionParam.isMutable = partitionInfo.getIsMutable(partition.getId());
recyclePartitionParam.partitionItem = partitionInfo.getItem(partition.getId());
recyclePartitionParam.partition = partition;
}

/*
* this method is currently used for truncating table(partitions).
* the new partition has new id, so we need to change all 'id-related' members
*
* return the old partition.
*/
public Partition replacePartition(Partition newPartition) {
public Partition replacePartition(Partition newPartition,
RecyclePartitionParam recyclePartitionParam) {
Partition oldPartition = nameToPartition.remove(newPartition.getName());
idToPartition.remove(oldPartition.getId());

Expand All @@ -2044,6 +2076,12 @@ public Partition replacePartition(Partition newPartition) {
ReplicaAllocation replicaAlloc = partitionInfo.getReplicaAllocation(oldPartition.getId());
boolean isInMemory = partitionInfo.getIsInMemory(oldPartition.getId());
boolean isMutable = partitionInfo.getIsMutable(oldPartition.getId());
recyclePartitionParam.dataProperty = dataProperty;
recyclePartitionParam.replicaAlloc = replicaAlloc;
recyclePartitionParam.isInMemory = isInMemory;
recyclePartitionParam.isMutable = isMutable;
recyclePartitionParam.partitionItem = partitionInfo.getItem(oldPartition.getId());
recyclePartitionParam.partition = oldPartition;

if (partitionInfo.getType() == PartitionType.RANGE
|| partitionInfo.getType() == PartitionType.LIST) {
Expand Down
31 changes: 31 additions & 0 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/RecyclePartitionParam.java
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
// Licensed to the Apache Software Foundation (ASF) under one
// or more contributor license agreements. See the NOTICE file
// distributed with this work for additional information
// regarding copyright ownership. The ASF licenses this file
// to you under the Apache License, Version 2.0 (the
// "License"); you may not use this file except in compliance
// with the License. You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing,
// software distributed under the License is distributed on an
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.

package org.apache.doris.catalog;

public class RecyclePartitionParam {
public Partition partition;
public PartitionItem partitionItem;
public DataProperty dataProperty;
public ReplicaAllocation replicaAlloc;
public boolean isInMemory;
public boolean isMutable = true;

public RecyclePartitionParam() {
// do nothing.
}
}
Loading