[Fix](Nereids) fix sql-cache for nereids.

dutyu · Aug 12, 2023 · 8298440 · 8298440
1 parent b24e82e
commit 8298440
Show file tree

Hide file tree

Showing 14 changed files with 447 additions and 176 deletions.
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AddPartitionEvent.java
@@ -46,7 +46,7 @@ public class AddPartitionEvent extends MetastorePartitionEvent {
  // for test
  public AddPartitionEvent(long eventId, String catalogName, String dbName,
  String tblName, List<String> partitionNames) {
- super(eventId, catalogName, dbName, tblName);
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.ADD_PARTITION);
  this.partitionNames = partitionNames;
  this.hmsTbl = null;
  }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterDatabaseEvent.java
@@ -41,13 +41,15 @@ public class AlterDatabaseEvent extends MetastoreEvent {
 
  // true if this alter event was due to a rename operation
  private final boolean isRename;
+ private final String dbNameAfter;
 
  // for test
  public AlterDatabaseEvent(long eventId, String catalogName, String dbName, boolean isRename) {
- super(eventId, catalogName, dbName, null);
+ super(eventId, catalogName, dbName, null, MetastoreEventType.ALTER_DATABASE);
  this.isRename = isRename;
  this.dbBefore = null;
  this.dbAfter = null;
+ this.dbNameAfter = isRename ? (dbName + "_new") : dbName;
  }
 
  private AlterDatabaseEvent(NotificationEvent event,
@@ -61,6 +63,7 @@ private AlterDatabaseEvent(NotificationEvent event,
  .getAlterDatabaseMessage(event.getMessage());
  dbBefore = Preconditions.checkNotNull(alterDatabaseMessage.getDbObjBefore());
  dbAfter = Preconditions.checkNotNull(alterDatabaseMessage.getDbObjAfter());
+ dbNameAfter = dbAfter.getName();
  } catch (Exception e) {
  throw new MetastoreNotificationException(
  debugString("Unable to parse the alter database message"), e);
@@ -97,6 +100,10 @@ public boolean isRename() {
  return isRename;
  }
 
+ public String getDbNameAfter() {
+ return dbNameAfter;
+ }
+
  @Override
  protected void process() throws MetastoreNotificationException {
  try {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterPartitionEvent.java
@@ -18,12 +18,11 @@
 
 package org.apache.doris.datasource.hive.event;
 
-import com.google.common.collect.ImmutableSet;
-import org.apache.commons.collections.CollectionUtils;
 import org.apache.doris.catalog.Env;
 import org.apache.doris.common.DdlException;
 
 import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Lists;
 import org.apache.hadoop.hive.common.FileUtils;
 import org.apache.hadoop.hive.metastore.api.FieldSchema;
@@ -49,14 +48,14 @@ public class AlterPartitionEvent extends MetastorePartitionEvent {
 
  // for test
  public AlterPartitionEvent(long eventId, String catalogName, String dbName, String tblName,
- String partitionNameBefore, String partitionNameAfter) {
- super(eventId, catalogName, dbName, tblName);
+ String partitionNameBefore, boolean isRename) {
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.ALTER_PARTITION);
  this.partitionNameBefore = partitionNameBefore;
- this.partitionNameAfter = partitionNameAfter;
+ this.partitionNameAfter = isRename ? (partitionNameBefore + "_new") : partitionNameBefore;
  this.hmsTbl = null;
  this.partitionAfter = null;
  this.partitionBefore = null;
- isRename = !partitionNameBefore.equalsIgnoreCase(partitionNameAfter);
+ this.isRename = isRename;
  }
 
  private AlterPartitionEvent(NotificationEvent event,
@@ -88,10 +87,18 @@ protected boolean willChangePartitionName() {
  }
 
  @Override
- protected Set<String> getAllPartitionNames() {
+ public Set<String> getAllPartitionNames() {
  return ImmutableSet.of(partitionNameBefore);
  }
 
+ public String getPartitionNameAfter() {
+ return partitionNameAfter;
+ }
+
+ public boolean isRename() {
+ return isRename;
+ }
+
  protected static List<MetastoreEvent> getEvents(NotificationEvent event,
  String catalogName) {
  return Lists.newArrayList(new AlterPartitionEvent(event, catalogName));

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/AlterTableEvent.java
@@ -41,17 +41,17 @@ public class AlterTableEvent extends MetastoreTableEvent {
  // true if this alter event was due to a rename operation
  private final boolean isRename;
  private final boolean isView;
- private final boolean willCreateOrDropTable;
+ private final String tblNameAfter;
 
  // for test
  public AlterTableEvent(long eventId, String catalogName, String dbName,
  String tblName, boolean isRename, boolean isView) {
- super(eventId, catalogName, dbName, tblName);
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.ALTER_TABLE);
  this.isRename = isRename;
  this.isView = isView;
  this.tableBefore = null;
  this.tableAfter = null;
- this.willCreateOrDropTable = isRename || isView;
+ this.tblNameAfter = isRename ? (tblName + "_new") : tblName;
  }
 
  private AlterTableEvent(NotificationEvent event, String catalogName) {
@@ -65,6 +65,7 @@ private AlterTableEvent(NotificationEvent event, String catalogName) {
  .getAlterTableMessage(event.getMessage());
  tableAfter = Preconditions.checkNotNull(alterTableMessage.getTableObjAfter());
  tableBefore = Preconditions.checkNotNull(alterTableMessage.getTableObjBefore());
+ tblNameAfter = tableAfter.getTableName();
  } catch (Exception e) {
  throw new MetastoreNotificationException(
  debugString("Unable to parse the alter table message"), e);
@@ -73,7 +74,6 @@ private AlterTableEvent(NotificationEvent event, String catalogName) {
  isRename = !tableBefore.getDbName().equalsIgnoreCase(tableAfter.getDbName())
  || !tableBefore.getTableName().equalsIgnoreCase(tableAfter.getTableName());
  isView = tableBefore.isSetViewExpandedText() || tableBefore.isSetViewOriginalText();
- this.willCreateOrDropTable = isRename || isView;
  }
 
  public static List<MetastoreEvent> getEvents(NotificationEvent event,
@@ -83,7 +83,7 @@ public static List<MetastoreEvent> getEvents(NotificationEvent event,
 
  @Override
  protected boolean willCreateOrDropTable() {
- return willCreateOrDropTable;
+ return isRename || isView;
  }
 
  @Override
@@ -128,6 +128,10 @@ public boolean isView() {
  return isView;
  }
 
+ public String getTblNameAfter() {
+ return tblNameAfter;
+ }
+
  /**
  * If the ALTER_TABLE event is due a table rename, this method removes the old table
  * and creates a new table with the new name. Else, we just refresh table

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateDatabaseEvent.java
@@ -34,7 +34,7 @@ public class CreateDatabaseEvent extends MetastoreEvent {
 
  // for test
  public CreateDatabaseEvent(long eventId, String catalogName, String dbName) {
- super(eventId, catalogName, dbName, null);
+ super(eventId, catalogName, dbName, null, MetastoreEventType.CREATE_DATABASE);
  }
 
  private CreateDatabaseEvent(NotificationEvent event,

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/CreateTableEvent.java
@@ -37,7 +37,7 @@ public class CreateTableEvent extends MetastoreTableEvent {
 
  // for test
  public CreateTableEvent(long eventId, String catalogName, String dbName, String tblName) {
- super(eventId, catalogName, dbName, tblName);
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.CREATE_TABLE);
  this.hmsTbl = null;
  }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropDatabaseEvent.java
@@ -32,6 +32,11 @@
  */
 public class DropDatabaseEvent extends MetastoreEvent {
 
+ // for test
+ public DropDatabaseEvent(long eventId, String catalogName, String dbName) {
+ super(eventId, catalogName, dbName, null, MetastoreEventType.DROP_DATABASE);
+ }
+
  private DropDatabaseEvent(NotificationEvent event,
  String catalogName) {
  super(event, catalogName);

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropPartitionEvent.java
@@ -42,6 +42,14 @@ public class DropPartitionEvent extends MetastorePartitionEvent {
  private final Table hmsTbl;
  private final List<String> partitionNames;
 
+ // for test
+ public DropPartitionEvent(long eventId, String catalogName, String dbName,
+ String tblName, List<String> partitionNames) {
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.DROP_PARTITION);
+ this.partitionNames = partitionNames;
+ this.hmsTbl = null;
+ }
+
  private DropPartitionEvent(NotificationEvent event,
  String catalogName) {
  super(event, catalogName);
@@ -70,7 +78,7 @@ protected boolean willChangePartitionName() {
  }
 
  @Override
- protected Set<String> getAllPartitionNames() {
+ public Set<String> getAllPartitionNames() {
  return ImmutableSet.copyOf(partitionNames);
  }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/DropTableEvent.java
@@ -37,7 +37,7 @@ public class DropTableEvent extends MetastoreTableEvent {
  // for test
  public DropTableEvent(long eventId, String catalogName, String dbName,
  String tblName) {
- super(eventId, catalogName, dbName, tblName);
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.DROP_TABLE);
  this.tableName = tblName;
  }
 
@@ -85,7 +85,7 @@ protected void process() throws MetastoreNotificationException {
 
  @Override
  protected boolean canBeBatched(MetastoreEvent that) {
- if(!isSameTable(that)) {
+ if (!isSameTable(that)) {
  return false;
  }
 

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/InsertEvent.java
@@ -38,7 +38,7 @@ public class InsertEvent extends MetastoreTableEvent {
  // for test
  public InsertEvent(long eventId, String catalogName, String dbName,
  String tblName) {
- super(eventId, catalogName, dbName, tblName);
+ super(eventId, catalogName, dbName, tblName, MetastoreEventType.INSERT);
  this.hmsTbl = null;
  }
 
@@ -66,6 +66,11 @@ protected boolean willCreateOrDropTable() {
  return false;
  }
 
+ @Override
+ protected boolean willChangeTableName() {
+ return false;
+ }
+
  @Override
  protected void process() throws MetastoreNotificationException {
  try {

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreEvent.java
@@ -58,12 +58,13 @@ public abstract class MetastoreEvent {
  protected final String catalogName;
 
  // for test
- protected MetastoreEvent(long eventId, String catalogName, String dbName, String tblName) {
+ protected MetastoreEvent(long eventId, String catalogName, String dbName,
+ String tblName, MetastoreEventType eventType) {
  this.eventId = eventId;
  this.catalogName = catalogName;
  this.dbName = dbName;
  this.tblName = tblName;
- this.eventType = null;
+ this.eventType = eventType;
  this.metastoreNotificationEvent = null;
  this.event = null;
  }

diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastorePartitionEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastorePartitionEvent.java
@@ -28,8 +28,9 @@
 public abstract class MetastorePartitionEvent extends MetastoreTableEvent {
 
  // for test
- protected MetastorePartitionEvent(long eventId, String catalogName, String dbName, String tblName) {
- super(eventId, catalogName, dbName, tblName);
+ protected MetastorePartitionEvent(long eventId, String catalogName, String dbName,
+ String tblName, MetastoreEventType eventType) {
+ super(eventId, catalogName, dbName, tblName, eventType);
  }
 
  protected MetastorePartitionEvent(NotificationEvent event, String catalogName) {
@@ -49,5 +50,5 @@ protected boolean willChangeTableName() {
  */
  protected abstract boolean willChangePartitionName();
 
- protected abstract Set<String> getAllPartitionNames();
+ public abstract Set<String> getAllPartitionNames();
 }
diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreTableEvent.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/event/MetastoreTableEvent.java
@@ -31,8 +31,9 @@
 public abstract class MetastoreTableEvent extends MetastoreEvent {
 
  // for test
- protected MetastoreTableEvent(long eventId, String catalogName, String dbName, String tblName) {
- super(eventId, catalogName, dbName, tblName);
+ protected MetastoreTableEvent(long eventId, String catalogName, String dbName,
+ String tblName, MetastoreEventType eventType) {
+ super(eventId, catalogName, dbName, tblName, eventType);
  }
 
  protected MetastoreTableEvent(NotificationEvent event, String catalogName) {