[Fix](load) fix commit txn timeout when loading to table with many ta…

…blet (apache#40031) fix commit txn timeout when loading to table with many tablet
zhannngchen · Aug 29, 2024 · 83e6eac · 83e6eac
1 parent 5253642
commit 83e6eac
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 18 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java b/fe/fe-core/src/main/java/org/apache/doris/load/BrokerFileGroup.java
@@ -152,12 +152,16 @@ public void parse(Database db, DataDescription dataDescription) throws DdlExcept
  }
  }
 
- boolean isPartitionRestoring = olapTable.getPartitions().stream().anyMatch(
- partition -> partition.getState() == PartitionState.RESTORE
- );
- // restore table
- if (!isPartitionRestoring && olapTable.getState() == OlapTableState.RESTORE) {
- throw new DdlException("Table [" + olapTable.getName() + "] is under restore");
+ // only do check when here's restore on this table now
+ if (olapTable.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring = olapTable.getPartitions().stream()
+ .anyMatch(partition -> partition.getState() == PartitionState.RESTORE);
+ // tbl RESTORE && all partition NOT RESTORE -> whole table restore
+ // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new DdlException("Table [" + olapTable.getName() + "] is under restore");
+ }
  }
 
  if (olapTable.getKeysType() != KeysType.AGG_KEYS && dataDescription.isNegative()) {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java b/fe/fe-core/src/main/java/org/apache/doris/transaction/DatabaseTransactionMgr.java
@@ -509,7 +509,9 @@ private void checkCommitStatus(List<Table> tableList, TransactionState transacti
  List<Long> tabletIds = tabletCommitInfos.stream()
  .map(TabletCommitInfo::getTabletId).collect(Collectors.toList());
  List<TabletMeta> tabletMetaList = tabletInvertedIndex.getTabletMetaList(tabletIds);
+ HashMap<Long, Boolean> tableIdtoRestoring = new HashMap<>();
  for (int i = 0; i < tabletMetaList.size(); i++) {
+ // get partition and table of this tablet
  TabletMeta tabletMeta = tabletMetaList.get(i);
  if (tabletMeta == TabletInvertedIndex.NOT_EXIST_TABLET_META) {
  continue;
@@ -518,29 +520,43 @@ private void checkCommitStatus(List<Table> tableList, TransactionState transacti
  long tableId = tabletMeta.getTableId();
  OlapTable tbl = (OlapTable) idToTable.get(tableId);
  if (tbl == null) {
- // this can happen when tableId == -1 (tablet being dropping)
- // or table really not exist.
+ // this can happen when tableId == -1 (tablet being dropping) or table really not exist.
  continue;
  }
 
+ // check relative partition restore here
  long partitionId = tabletMeta.getPartitionId();
  if (tbl.getPartition(partitionId) == null) {
- // this can happen when partitionId == -1 (tablet being dropping)
- // or partition really not exist.
+ // this can happen when partitionId == -1 (tablet being dropping) or partition really not exist.
  continue;
- } else if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) {
+ }
+ if (tbl.getPartition(partitionId).getState() == PartitionState.RESTORE) {
  // partition in restore process which can not load data
  throw new LoadException("Table [" + tbl.getName() + "], Partition ["
  + tbl.getPartition(partitionId).getName() + "] is in restore process. Can not load into it");
  }
 
- boolean isPartitionRestoring = tbl.getPartitions().stream().anyMatch(
- partition -> partition.getState() == PartitionState.RESTORE
- );
- // restore table
- if (!isPartitionRestoring && tbl.getState() == OlapTableState.RESTORE) {
- throw new LoadException("Table " + tbl.getName() + " is in restore process. "
- + "Can not load into it");
+ // only do check when here's restore on this table now
+ if (tbl.getState() == OlapTableState.RESTORE) {
+ boolean hasPartitionRestoring = false;
+ if (tableIdtoRestoring.containsKey(tableId)) {
+ hasPartitionRestoring = tableIdtoRestoring.get(tableId);
+ } else {
+ for (Partition partition : tbl.getPartitions()) {
+ if (partition.getState() == PartitionState.RESTORE) {
+ hasPartitionRestoring = true;
+ break;
+ }
+ }
+ tableIdtoRestoring.put(tableId, hasPartitionRestoring);
+ }
+ // tbl RESTORE && all partition NOT RESTORE -> whole table restore
+ // tbl RESTORE && some partition RESTORE -> just partitions restore, NOT WHOLE TABLE
+ // so check wether the whole table restore here
+ if (!hasPartitionRestoring) {
+ throw new LoadException(
+ "Table " + tbl.getName() + " is in restore process. " + "Can not load into it");
+ }
  }
 
  if (!tableToPartition.containsKey(tableId)) {