diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java index 204a0ca440fe49..6a0933b0c6c0a7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java @@ -144,7 +144,6 @@ protected void doInitialize() throws UserException { table.getName())); } } - computeColumnsFilter(); initBackendPolicy(); initSchemaParams(); } @@ -213,6 +212,7 @@ protected void doFinalize() throws UserException { if (ConnectContext.get().getExecutor() != null) { ConnectContext.get().getExecutor().getSummaryProfile().setFinalizeScanNodeStartTime(); } + convertPredicate(); createScanRangeLocations(); updateRequiredSlots(); if (ConnectContext.get().getExecutor() != null) { @@ -220,6 +220,15 @@ protected void doFinalize() throws UserException { } } + /** + * Used as a predicate to convert conjuncts into corresponding data sources. + * All predicate conversions from different data sources should override this method. + * and this method must be called in finalize, + * because in nereids planner, conjuncts are only generated in the finalize stage. + */ + protected void convertPredicate() { + } + private void setColumnPositionMapping() throws UserException { TableIf tbl = getTargetTable(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java index 6f4314bd08b913..a56c44733216c6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/es/source/EsScanNode.java @@ -115,11 +115,13 @@ public void init() throws UserException { @Override public void finalize(Analyzer analyzer) throws UserException { + buildQuery(); doFinalize(); } @Override public void finalizeForNereids() throws UserException { + buildQuery(); doFinalize(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java index a215cba3f9cee6..516448bdfbb9ec 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java @@ -37,6 +37,7 @@ import org.apache.doris.mtmv.MTMVSnapshotIf; import org.apache.doris.mtmv.MTMVTimestampSnapshot; import org.apache.doris.nereids.exceptions.NotSupportedException; +import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; import org.apache.doris.qe.GlobalVariable; import org.apache.doris.statistics.AnalysisInfo; import org.apache.doris.statistics.BaseAnalysisTask; @@ -288,6 +289,21 @@ public List getPartitionColumns() { .orElse(Collections.emptyList()); } + public SelectedPartitions getAllPartitions() { + if (CollectionUtils.isEmpty(this.getPartitionColumns())) { + return SelectedPartitions.NOT_PRUNED; + } + + HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() + .getMetaStoreCache((HMSExternalCatalog) this.getCatalog()); + List partitionColumnTypes = this.getPartitionColumnTypes(); + HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( + this.getDbName(), this.getName(), partitionColumnTypes); + Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + + return new SelectedPartitions(idToPartitionItem.size(), idToPartitionItem, false); + } + public boolean isHiveTransactionalTable() { return dlaType == DLAType.HIVE && AcidUtils.isTransactionalTable(remoteTable) && isSupportedTransactionalFileFormat(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java index 0dcf4724a7b1e8..cd7e2b58425be3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/hive/source/HiveScanNode.java @@ -41,7 +41,6 @@ import org.apache.doris.datasource.hive.HiveTransaction; import org.apache.doris.datasource.hive.source.HiveSplit.HiveSplitCreator; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan.SelectedPartitions; -import org.apache.doris.planner.ListPartitionPrunerV2; import org.apache.doris.planner.PlanNodeId; import org.apache.doris.qe.ConnectContext; import org.apache.doris.spi.Split; @@ -142,46 +141,16 @@ protected void doInitialize() throws UserException { protected List getPartitions() throws AnalysisException { List resPartitions = Lists.newArrayList(); - long start = System.currentTimeMillis(); HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() .getMetaStoreCache((HMSExternalCatalog) hmsTable.getCatalog()); List partitionColumnTypes = hmsTable.getPartitionColumnTypes(); if (!partitionColumnTypes.isEmpty()) { // partitioned table - boolean isPartitionPruned = selectedPartitions != null && selectedPartitions.isPruned; Collection partitionItems; - if (!isPartitionPruned) { - // partitionItems is null means that the partition is not pruned by Nereids, - // so need to prune partitions here by legacy ListPartitionPrunerV2. - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - hmsTable.getDbName(), hmsTable.getName(), partitionColumnTypes); - Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); - this.totalPartitionNum = idToPartitionItem.size(); - if (!conjuncts.isEmpty()) { - ListPartitionPrunerV2 pruner = new ListPartitionPrunerV2(idToPartitionItem, - hmsTable.getPartitionColumns(), columnNameToRange, - hivePartitionValues.getUidToPartitionRange(), - hivePartitionValues.getRangeToId(), - hivePartitionValues.getSingleColumnRangeMap(), - true); - Collection filteredPartitionIds = pruner.prune(); - if (LOG.isDebugEnabled()) { - LOG.debug("hive partition fetch and prune for table {}.{} cost: {} ms", - hmsTable.getDbName(), hmsTable.getName(), (System.currentTimeMillis() - start)); - } - partitionItems = Lists.newArrayListWithCapacity(filteredPartitionIds.size()); - for (Long id : filteredPartitionIds) { - partitionItems.add(idToPartitionItem.get(id)); - } - } else { - partitionItems = idToPartitionItem.values(); - } - } else { - // partitions has benn pruned by Nereids, in PruneFileScanPartition, - // so just use the selected partitions. - this.totalPartitionNum = selectedPartitions.totalPartitionNum; - partitionItems = selectedPartitions.selectedPartitions.values(); - } + // partitions has benn pruned by Nereids, in PruneFileScanPartition, + // so just use the selected partitions. + this.totalPartitionNum = selectedPartitions.totalPartitionNum; + partitionItems = selectedPartitions.selectedPartitions.values(); Preconditions.checkNotNull(partitionItems); this.selectedPartitionNum = partitionItems.size(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java index 856bcfff1614fc..4ec4319c228c05 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/maxcompute/source/MaxComputeScanNode.java @@ -82,6 +82,7 @@ public class MaxComputeScanNode extends FileQueryScanNode { private final MaxComputeExternalTable table; TableBatchReadSession tableBatchReadSession; + private Predicate filterPredicate; public MaxComputeScanNode(PlanNodeId id, TupleDescriptor desc, boolean needCheckColumnPriv) { this(id, desc, "MCScanNode", StatisticalType.MAX_COMPUTE_SCAN_NODE, needCheckColumnPriv); @@ -115,8 +116,6 @@ private void setScanParams(TFileRangeDesc rangeDesc, MaxComputeSplit maxComputeS } void createTableBatchReadSession() throws UserException { - Predicate filterPredicate = convertPredicate(); - List requiredPartitionColumns = new ArrayList<>(); List orderedRequiredDataColumns = new ArrayList<>(); @@ -164,9 +163,10 @@ void createTableBatchReadSession() throws UserException { } - protected Predicate convertPredicate() { + @Override + protected void convertPredicate() { if (conjuncts.isEmpty()) { - return Predicate.NO_PREDICATE; + this.filterPredicate = Predicate.NO_PREDICATE; } List odpsPredicates = new ArrayList<>(); @@ -180,9 +180,9 @@ protected Predicate convertPredicate() { } if (odpsPredicates.isEmpty()) { - return Predicate.NO_PREDICATE; + this.filterPredicate = Predicate.NO_PREDICATE; } else if (odpsPredicates.size() == 1) { - return odpsPredicates.get(0); + this.filterPredicate = odpsPredicates.get(0); } else { com.aliyun.odps.table.optimizer.predicate.CompoundPredicate filterPredicate = new com.aliyun.odps.table.optimizer.predicate.CompoundPredicate( @@ -191,7 +191,7 @@ protected Predicate convertPredicate() { for (Predicate odpsPredicate : odpsPredicates) { filterPredicate.addPredicate(odpsPredicate); } - return filterPredicate; + this.filterPredicate = filterPredicate; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java index e0d0f9a3ea2cd0..02f831ba37bc1c 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/paimon/source/PaimonScanNode.java @@ -111,6 +111,10 @@ protected void doInitialize() throws UserException { super.doInitialize(); source = new PaimonSource(desc); Preconditions.checkNotNull(source); + } + + @Override + protected void convertPredicate() { PaimonPredicateConverter paimonPredicateConverter = new PaimonPredicateConverter( source.getPaimonTable().rowType()); predicates = paimonPredicateConverter.convertToPaimonExpr(conjuncts); diff --git a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java index 0167e0c7c4298f..6f660993d6350a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/datasource/trinoconnector/source/TrinoConnectorScanNode.java @@ -105,10 +105,10 @@ public TrinoConnectorScanNode(PlanNodeId id, TupleDescriptor desc, boolean needC protected void doInitialize() throws UserException { super.doInitialize(); source = new TrinoConnectorSource(desc); - convertPredicate(); } - protected void convertPredicate() throws UserException { + @Override + protected void convertPredicate() { if (conjuncts.isEmpty()) { constraint = Constraint.alwaysTrue(); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java index 6e140fff13f87a..5bf36cce161ccf 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/glue/translator/PhysicalPlanTranslator.java @@ -624,7 +624,6 @@ public PlanFragment visitPhysicalEsScan(PhysicalEsScan esScan, PlanTranslatorCon EsScanNode esScanNode = new EsScanNode(context.nextPlanNodeId(), tupleDescriptor, table instanceof EsExternalTable); esScanNode.setNereidsId(esScan.getId()); - esScanNode.addConjuncts(translateToLegacyConjuncts(esScan.getConjuncts())); Utils.execWithUncheckedException(esScanNode::init); context.addScanNode(esScanNode, esScan); context.getRuntimeTranslator().ifPresent( @@ -666,7 +665,6 @@ private PlanFragment getPlanFragmentForPhysicalFileScan(PhysicalFileScan fileSca ScanNode scanNode, ExternalTable table, TupleDescriptor tupleDescriptor) { scanNode.setNereidsId(fileScan.getId()); - scanNode.addConjuncts(translateToLegacyConjuncts(fileScan.getConjuncts())); scanNode.setPushDownAggNoGrouping(context.getRelationPushAggOp(fileScan.getRelationId())); TableName tableName = new TableName(null, "", ""); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java index 099765b5160ab0..75f2ddc7b703fe 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/jobs/executor/Rewriter.java @@ -108,9 +108,6 @@ import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderApply; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderLimit; import org.apache.doris.nereids.rules.rewrite.PullUpProjectUnderTopN; -import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoEsScan; -import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoJdbcScan; -import org.apache.doris.nereids.rules.rewrite.PushConjunctsIntoOdbcScan; import org.apache.doris.nereids.rules.rewrite.PushCountIntoUnionAll; import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoin; import org.apache.doris.nereids.rules.rewrite.PushDownAggThroughJoinOnPkFk; @@ -410,10 +407,7 @@ public class Rewriter extends AbstractBatchJobExecutor { topDown( new PruneOlapScanPartition(), new PruneEmptyPartition(), - new PruneFileScanPartition(), - new PushConjunctsIntoJdbcScan(), - new PushConjunctsIntoOdbcScan(), - new PushConjunctsIntoEsScan() + new PruneFileScanPartition() ) ), topic("MV optimization", diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java index b0bf689d1e1612..c776e73c240f93 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindRelation.java @@ -432,7 +432,9 @@ private LogicalPlan getLogicalPlan(TableIf table, UnboundRelation unboundRelatio return hudiScan; } else { return new LogicalFileScan(unboundRelation.getRelationId(), (HMSExternalTable) table, - qualifierWithoutTableName, unboundRelation.getTableSample(), + qualifierWithoutTableName, + ((HMSExternalTable) table).getAllPartitions(), + unboundRelation.getTableSample(), unboundRelation.getTableSnapshot()); } case ICEBERG_EXTERNAL_TABLE: diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java index 9e83fec2a12666..1e4db2eba11810 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalEsScanToPhysicalEsScan.java @@ -37,8 +37,7 @@ public Rule build() { esScan.getQualifier(), DistributionSpecAny.INSTANCE, Optional.empty(), - esScan.getLogicalProperties(), - esScan.getConjuncts()) + esScan.getLogicalProperties()) ).toRule(RuleType.LOGICAL_ES_SCAN_TO_PHYSICAL_ES_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java index 4946dcd56c3fed..70ab9b1d502c0d 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalFileScanToPhysicalFileScan.java @@ -39,7 +39,6 @@ public Rule build() { DistributionSpecAny.INSTANCE, Optional.empty(), fileScan.getLogicalProperties(), - fileScan.getConjuncts(), fileScan.getSelectedPartitions(), fileScan.getTableSample(), fileScan.getTableSnapshot()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java index a5d676eab67c30..d8beef8d042ed9 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalHudiScanToPhysicalHudiScan.java @@ -38,7 +38,6 @@ public Rule build() { DistributionSpecAny.INSTANCE, Optional.empty(), fileScan.getLogicalProperties(), - fileScan.getConjuncts(), fileScan.getSelectedPartitions(), fileScan.getTableSample(), fileScan.getTableSnapshot(), diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java index 9c2c6518034ee9..99120c2b2eeb1e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalJdbcScanToPhysicalJdbcScan.java @@ -35,8 +35,7 @@ public Rule build() { jdbcScan.getTable(), jdbcScan.getQualifier(), Optional.empty(), - jdbcScan.getLogicalProperties(), - jdbcScan.getConjuncts()) + jdbcScan.getLogicalProperties()) ).toRule(RuleType.LOGICAL_JDBC_SCAN_TO_PHYSICAL_JDBC_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java index 22aea72f736122..441219d7a0d075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/implementation/LogicalOdbcScanToPhysicalOdbcScan.java @@ -35,8 +35,7 @@ public Rule build() { odbcScan.getTable(), odbcScan.getQualifier(), Optional.empty(), - odbcScan.getLogicalProperties(), - odbcScan.getConjuncts()) + odbcScan.getLogicalProperties()) ).toRule(RuleType.LOGICAL_ODBC_SCAN_TO_PHYSICAL_ODBC_SCAN_RULE); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java index 198b6363d9a1e2..808288b8fe3cab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/AdjustNullable.java @@ -31,7 +31,6 @@ import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.logical.LogicalAggregate; import org.apache.doris.nereids.trees.plans.logical.LogicalCTEConsumer; -import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalJoin; @@ -277,17 +276,6 @@ public Plan visitLogicalCTEConsumer(LogicalCTEConsumer cteConsumer, Map replaceMap) { - if (!relation.getConjuncts().isEmpty()) { - relation.getOutputSet().forEach(s -> replaceMap.put(s.getExprId(), s)); - Set conjuncts = updateExpressions(relation.getConjuncts(), replaceMap); - return relation.withConjuncts(conjuncts).recomputeLogicalProperties(); - } else { - return relation; - } - } - private T updateExpression(T input, Map replaceMap) { return (T) input.rewriteDownShortCircuit(e -> e.accept(SlotReferenceReplacer.INSTANCE, replaceMap)); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java index 9b2f4999013893..2de4efab2ff6ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PruneFileScanPartition.java @@ -17,13 +17,10 @@ package org.apache.doris.nereids.rules.rewrite; -import org.apache.doris.catalog.Env; import org.apache.doris.catalog.PartitionItem; import org.apache.doris.datasource.ExternalTable; -import org.apache.doris.datasource.hive.HMSExternalCatalog; import org.apache.doris.datasource.hive.HMSExternalTable; import org.apache.doris.datasource.hive.HMSExternalTable.DLAType; -import org.apache.doris.datasource.hive.HiveMetaStoreCache; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.rules.Rule; import org.apache.doris.nereids.rules.RuleType; @@ -72,8 +69,7 @@ public Rule build() { selectedPartitions = new SelectedPartitions(0, ImmutableMap.of(), true); } - LogicalFileScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()) - .withSelectedPartitions(selectedPartitions); + LogicalFileScan rewrittenScan = scan.withSelectedPartitions(selectedPartitions); return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); }).toRule(RuleType.FILE_SCAN_PARTITION_PRUNE); } @@ -95,11 +91,7 @@ private SelectedPartitions pruneHivePartitions(HMSExternalTable hiveTbl, .map(column -> scanOutput.get(column.getName().toLowerCase())) .collect(Collectors.toList()); - HiveMetaStoreCache cache = Env.getCurrentEnv().getExtMetaCacheMgr() - .getMetaStoreCache((HMSExternalCatalog) hiveTbl.getCatalog()); - HiveMetaStoreCache.HivePartitionValues hivePartitionValues = cache.getPartitionValues( - hiveTbl.getDbName(), hiveTbl.getName(), hiveTbl.getPartitionColumnTypes()); - Map idToPartitionItem = hivePartitionValues.getIdToPartitionItem(); + Map idToPartitionItem = scan.getSelectedPartitions().selectedPartitions; List prunedPartitions = new ArrayList<>(PartitionPruner.prune( partitionSlots, filter.getPredicate(), idToPartitionItem, ctx, PartitionTableType.HIVE)); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java deleted file mode 100644 index 9bd485716d2d0a..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoEsScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; - -/** - * Rewrite es plan to set the conjuncts. - */ -public class PushConjunctsIntoEsScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalEsScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalEsScan scan = filter.child(); - LogicalEsScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return rewrittenScan; - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ES_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java deleted file mode 100644 index 21d723f9561eec..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoJdbcScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; - -/** - * Rewrite jdbc plan to set the conjuncts. - */ -public class PushConjunctsIntoJdbcScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalJdbcScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalJdbcScan scan = filter.child(); - LogicalJdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_JDBC_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java deleted file mode 100644 index 8a1ba4d1b6790f..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/rewrite/PushConjunctsIntoOdbcScan.java +++ /dev/null @@ -1,39 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.rules.Rule; -import org.apache.doris.nereids.rules.RuleType; -import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; -import org.apache.doris.nereids.trees.plans.logical.LogicalOdbcScan; - -/** - * Rewrite odbc plan to set the conjuncts. - */ -public class PushConjunctsIntoOdbcScan extends OneRewriteRuleFactory { - - @Override - public Rule build() { - return logicalFilter(logicalOdbcScan()).thenApply(ctx -> { - LogicalFilter filter = ctx.root; - LogicalOdbcScan scan = filter.child(); - LogicalOdbcScan rewrittenScan = scan.withConjuncts(filter.getConjuncts()); - return new LogicalFilter<>(filter.getConjuncts(), rewrittenScan); - }).toRule(RuleType.PUSH_CONJUNCTS_INTO_ODBC_SCAN); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java index 7b7c95dd811d79..20e72b81aa7872 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/copier/LogicalPlanDeepCopier.java @@ -40,7 +40,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeTopN; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalExcept; -import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFilter; import org.apache.doris.nereids.trees.plans.logical.LogicalGenerate; import org.apache.doris.nereids.trees.plans.logical.LogicalHaving; @@ -189,22 +188,6 @@ public Plan visitLogicalDeferMaterializeOlapScan(LogicalDeferMaterializeOlapScan return new LogicalDeferMaterializeOlapScan(newScan, newSlotIds, newRowId); } - @Override - public Plan visitLogicalExternalRelation(LogicalExternalRelation relation, - DeepCopierContext context) { - if (context.getRelationReplaceMap().containsKey(relation.getRelationId())) { - return context.getRelationReplaceMap().get(relation.getRelationId()); - } - LogicalExternalRelation newRelation = relation.withRelationId(StatementScopeIdGenerator.newRelationId()); - updateReplaceMapWithOutput(relation, newRelation, context.exprIdReplaceMap); - Set conjuncts = relation.getConjuncts().stream() - .map(p -> ExpressionDeepCopier.INSTANCE.deepCopy(p, context)) - .collect(ImmutableSet.toImmutableSet()); - newRelation = newRelation.withConjuncts(conjuncts); - context.putRelation(relation.getRelationId(), newRelation); - return newRelation; - } - @Override public Plan visitLogicalProject(LogicalProject project, DeepCopierContext context) { Plan child = project.child().accept(this, context); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java index 589546944224ec..8fb916b8a5e526 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalEsScan.java @@ -20,35 +20,31 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; import org.apache.doris.nereids.util.Utils; -import com.google.common.collect.ImmutableSet; - import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external es catalog. */ -public class LogicalEsScan extends LogicalExternalRelation { +public class LogicalEsScan extends LogicalCatalogRelation { /** * Constructor for LogicalEsScan. */ public LogicalEsScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, Set conjuncts) { - super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalEsScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -62,26 +58,18 @@ public String toString() { @Override public LogicalEsScan withGroupExpression(Optional groupExpression) { return new LogicalEsScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties, - conjuncts); - } - - @Override - public LogicalEsScan withConjuncts(Set conjuncts) { - return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + return new LogicalEsScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalEsScan withRelationId(RelationId relationId) { - return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), - Optional.empty(), conjuncts); + return new LogicalEsScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java deleted file mode 100644 index bc6313f5291044..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalExternalRelation.java +++ /dev/null @@ -1,68 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.trees.plans.logical; - -import org.apache.doris.catalog.TableIf; -import org.apache.doris.nereids.memo.GroupExpression; -import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.plans.PlanType; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.visitor.PlanVisitor; - -import com.google.common.collect.ImmutableSet; - -import java.util.List; -import java.util.Objects; -import java.util.Optional; -import java.util.Set; - -/** - * abstract class catalog relation for logical relation - */ -public abstract class LogicalExternalRelation extends LogicalCatalogRelation { - - // TODO remove this conjuncts when old planner is removed - protected final Set conjuncts; - - public LogicalExternalRelation(RelationId relationId, PlanType type, TableIf table, List qualifier, - Set conjuncts, - Optional groupExpression, Optional logicalProperties) { - super(relationId, type, table, qualifier, groupExpression, logicalProperties); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); - } - - public abstract LogicalExternalRelation withConjuncts(Set conjuncts); - - @Override - public abstract LogicalExternalRelation withRelationId(RelationId relationId); - - public Set getConjuncts() { - return conjuncts; - } - - @Override - public boolean equals(Object o) { - return super.equals(o) && Objects.equals(conjuncts, ((LogicalExternalRelation) o).conjuncts); - } - - @Override - public R accept(PlanVisitor visitor, C context) { - return visitor.visitLogicalExternalRelation(this, context); - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java index 8dd47c44b15ab3..0a2c69b68c1d33 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalFileScan.java @@ -23,7 +23,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -32,18 +31,16 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Sets; import java.util.List; import java.util.Map; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Logical file scan for external catalog. */ -public class LogicalFileScan extends LogicalExternalRelation { +public class LogicalFileScan extends LogicalCatalogRelation { protected final SelectedPartitions selectedPartitions; protected final Optional tableSample; @@ -54,18 +51,25 @@ public class LogicalFileScan extends LogicalExternalRelation { */ protected LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { - super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + super(id, PlanType.LOGICAL_FILE_SCAN, table, qualifier, groupExpression, logicalProperties); this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; } + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, + SelectedPartitions selectedPartitions, + Optional tableSample, Optional tableSnapshot) { + this(id, table, qualifier, Optional.empty(), Optional.empty(), + selectedPartitions, tableSample, tableSnapshot); + } + public LogicalFileScan(RelationId id, ExternalTable table, List qualifier, Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); + SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot); } public SelectedPartitions getSelectedPartitions() { @@ -98,31 +102,25 @@ public String toString() { @Override public LogicalFileScan withGroupExpression(Optional groupExpression) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); - } - - @Override - public LogicalFileScan withConjuncts(Set conjuncts) { - return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot); } public LogicalFileScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot); } @Override public LogicalFileScan withRelationId(RelationId relationId) { return new LogicalFileScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + Optional.empty(), selectedPartitions, tableSample, tableSnapshot); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java index deeca65efcbd1e..629690889432b3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalHudiScan.java @@ -44,7 +44,6 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableSet; -import com.google.common.collect.Sets; import org.apache.hudi.common.table.HoodieTableMetaClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -71,10 +70,10 @@ public class LogicalHudiScan extends LogicalFileScan { */ protected LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, Optional groupExpression, Optional logicalProperties, - Set conjuncts, SelectedPartitions selectedPartitions, Optional tableSample, + SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { - super(id, table, qualifier, groupExpression, logicalProperties, conjuncts, + super(id, table, qualifier, groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); @@ -85,7 +84,7 @@ protected LogicalHudiScan(RelationId id, ExternalTable table, List quali public LogicalHudiScan(RelationId id, ExternalTable table, List qualifier, Optional tableSample, Optional tableSnapshot) { this(id, table, qualifier, Optional.empty(), Optional.empty(), - Sets.newHashSet(), SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot, + SelectedPartitions.NOT_PRUNED, tableSample, tableSnapshot, Optional.empty(), Optional.empty()); } @@ -136,7 +135,7 @@ public String toString() { @Override public LogicalHudiScan withGroupExpression(Optional groupExpression) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -144,27 +143,20 @@ public LogicalHudiScan withGroupExpression(Optional groupExpres public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, - groupExpression, logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot, - scanParams, incrementalRelation); - } - - @Override - public LogicalHudiScan withConjuncts(Set conjuncts) { - return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, logicalProperties, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } public LogicalHudiScan withSelectedPartitions(SelectedPartitions selectedPartitions) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.of(getLogicalProperties()), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @Override public LogicalHudiScan withRelationId(RelationId relationId) { return new LogicalHudiScan(relationId, (ExternalTable) table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.empty(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -223,7 +215,7 @@ public LogicalHudiScan withScanParams(HMSExternalTable table, TableScanParams sc } newScanParams = Optional.ofNullable(scanParams); return new LogicalHudiScan(relationId, table, qualifier, Optional.empty(), - Optional.empty(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + Optional.empty(), selectedPartitions, tableSample, tableSnapshot, newScanParams, newIncrementalRelation); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java index cde2b6be242a08..53176f72c53e95 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalJdbcScan.java @@ -22,7 +22,6 @@ import org.apache.doris.datasource.ExternalTable; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -30,29 +29,26 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external jdbc catalog and jdbc table. */ -public class LogicalJdbcScan extends LogicalExternalRelation { +public class LogicalJdbcScan extends LogicalCatalogRelation { /** * Constructor for LogicalJdbcScan. */ public LogicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, - Set conjuncts) { - super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalJdbcScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -73,24 +69,18 @@ public String toString() { @Override public LogicalJdbcScan withGroupExpression(Optional groupExpression) { return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); - } - - @Override - public LogicalJdbcScan withConjuncts(Set conjuncts) { - return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); + return new LogicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalJdbcScan withRelationId(RelationId relationId) { - return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts); + return new LogicalJdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java index 414cb335af15b6..3ce9b8d74035de 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/logical/LogicalOdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.catalog.TableIf; import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,26 +28,23 @@ import org.apache.doris.nereids.util.Utils; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableSet; import java.util.List; import java.util.Optional; -import java.util.Set; /** * Logical scan for external odbc table. */ -public class LogicalOdbcScan extends LogicalExternalRelation { +public class LogicalOdbcScan extends LogicalCatalogRelation { public LogicalOdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - Optional logicalProperties, - Set conjuncts) { - super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, conjuncts, groupExpression, logicalProperties); + Optional logicalProperties) { + super(id, PlanType.LOGICAL_ODBC_SCAN, table, qualifier, groupExpression, logicalProperties); } public LogicalOdbcScan(RelationId id, TableIf table, List qualifier) { - this(id, table, qualifier, Optional.empty(), Optional.empty(), ImmutableSet.of()); + this(id, table, qualifier, Optional.empty(), Optional.empty()); } @Override @@ -69,24 +65,18 @@ public String toString() { @Override public LogicalOdbcScan withGroupExpression(Optional groupExpression) { return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, - Optional.of(getLogicalProperties()), conjuncts); - } - - @Override - public LogicalOdbcScan withConjuncts(Set conjuncts) { - return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), - Optional.of(getLogicalProperties()), conjuncts); + Optional.of(getLogicalProperties())); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties, conjuncts); + return new LogicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties); } @Override public LogicalOdbcScan withRelationId(RelationId relationId) { - return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty(), conjuncts); + return new LogicalOdbcScan(relationId, table, qualifier, Optional.empty(), Optional.empty()); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java index 474436f1dbb5ff..8c6d348e6fb8ab 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalEsScan.java @@ -22,7 +22,6 @@ import org.apache.doris.nereids.properties.DistributionSpec; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -30,12 +29,8 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical es scan for external catalog. @@ -43,17 +38,15 @@ public class PhysicalEsScan extends PhysicalCatalogRelation { private final DistributionSpec distributionSpec; - private final Set conjuncts; /** * Constructor for PhysicalEsScan. */ public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts) { + LogicalProperties logicalProperties) { super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } /** @@ -61,12 +54,10 @@ public PhysicalEsScan(RelationId id, TableIf table, List qualifier, */ public PhysicalEsScan(RelationId id, TableIf table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_ES_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -86,24 +77,20 @@ public R accept(PlanVisitor visitor, C context) { @Override public PhysicalEsScan withGroupExpression(Optional groupExpression) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts); + groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts); + groupExpression, logicalProperties.get()); } @Override public PhysicalEsScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statsDeriveResult) { return new PhysicalEsScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + groupExpression, getLogicalProperties(), physicalProperties, statsDeriveResult); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java index c289f2beb358db..d1a0d9c6c4a7f2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalFileScan.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -35,7 +34,6 @@ import java.util.List; import java.util.Optional; -import java.util.Set; /** * Physical file scan for external catalog. @@ -43,7 +41,6 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { protected final DistributionSpec distributionSpec; - protected final Set conjuncts; protected final SelectedPartitions selectedPartitions; protected final Optional tableSample; protected final Optional tableSnapshot; @@ -53,11 +50,11 @@ public class PhysicalFileScan extends PhysicalCatalogRelation { */ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, - logicalProperties, conjuncts, selectedPartitions, tableSample, tableSnapshot); + logicalProperties, selectedPartitions, tableSample, tableSnapshot); } /** @@ -66,11 +63,10 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi public PhysicalFileScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { this(id, PlanType.PHYSICAL_FILE_SCAN, table, qualifier, distributionSpec, groupExpression, - logicalProperties, physicalProperties, statistics, conjuncts, selectedPartitions, tableSample, - tableSnapshot); + logicalProperties, physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); } /** @@ -78,12 +74,11 @@ public PhysicalFileScan(RelationId id, ExternalTable table, List qualifi */ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties); this.distributionSpec = distributionSpec; - this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; @@ -92,12 +87,11 @@ protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, Li protected PhysicalFileScan(RelationId id, PlanType type, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot) { super(id, type, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); this.distributionSpec = distributionSpec; - this.conjuncts = conjuncts; this.selectedPartitions = selectedPartitions; this.tableSample = tableSample; this.tableSnapshot = tableSnapshot; @@ -107,10 +101,6 @@ public DistributionSpec getDistributionSpec() { return distributionSpec; } - public Set getConjuncts() { - return conjuncts; - } - public SelectedPartitions getSelectedPartitions() { return selectedPartitions; } @@ -128,7 +118,6 @@ public String toString() { return Utils.toSqlString("PhysicalFileScan[" + table.getName() + "]" + getGroupIdWithPrefix(), "stats", statistics, "qualified", Utils.qualifiedName(qualifier, table.getName()), - "conjuncts", conjuncts, "selected partitions num", selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown" ); @@ -142,14 +131,14 @@ public R accept(PlanVisitor visitor, C context) { @Override public PhysicalFileScan withGroupExpression(Optional groupExpression) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot); + groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot); } @Override @@ -161,7 +150,7 @@ public ExternalTable getTable() { public PhysicalFileScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalFileScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, + groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java index d5bc299a2baf96..451c57233920f6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalHudiScan.java @@ -26,7 +26,6 @@ import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; import org.apache.doris.nereids.trees.TableSample; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -38,7 +37,6 @@ import java.util.List; import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical Hudi scan for Hudi table. @@ -54,12 +52,12 @@ public class PhysicalHudiScan extends PhysicalFileScan { */ public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, - LogicalProperties logicalProperties, Set conjuncts, + LogicalProperties logicalProperties, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, - conjuncts, selectedPartitions, tableSample, tableSnapshot); + selectedPartitions, tableSample, tableSnapshot); Objects.requireNonNull(scanParams, "scanParams should not null"); Objects.requireNonNull(incrementalRelation, "incrementalRelation should not null"); this.scanParams = scanParams; @@ -72,11 +70,11 @@ public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifi public PhysicalHudiScan(RelationId id, ExternalTable table, List qualifier, DistributionSpec distributionSpec, Optional groupExpression, LogicalProperties logicalProperties, PhysicalProperties physicalProperties, - Statistics statistics, Set conjuncts, SelectedPartitions selectedPartitions, + Statistics statistics, SelectedPartitions selectedPartitions, Optional tableSample, Optional tableSnapshot, Optional scanParams, Optional incrementalRelation) { super(id, PlanType.PHYSICAL_HUDI_SCAN, table, qualifier, distributionSpec, groupExpression, logicalProperties, - physicalProperties, statistics, conjuncts, selectedPartitions, tableSample, tableSnapshot); + physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot); this.scanParams = scanParams; this.incrementalRelation = incrementalRelation; } @@ -92,7 +90,7 @@ public Optional getIncrementalRelation() { @Override public PhysicalHudiScan withGroupExpression(Optional groupExpression) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, getLogicalProperties(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -100,7 +98,7 @@ groupExpression, getLogicalProperties(), conjuncts, selectedPartitions, tableSam public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, logicalProperties.get(), conjuncts, selectedPartitions, tableSample, tableSnapshot, + groupExpression, logicalProperties.get(), selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -108,7 +106,7 @@ public Plan withGroupExprLogicalPropChildren(Optional groupExpr public PhysicalHudiScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalHudiScan(relationId, getTable(), qualifier, distributionSpec, - groupExpression, getLogicalProperties(), physicalProperties, statistics, conjuncts, + groupExpression, getLogicalProperties(), physicalProperties, statistics, selectedPartitions, tableSample, tableSnapshot, scanParams, incrementalRelation); } @@ -124,7 +122,6 @@ public String toString() { "qualified", Utils.qualifiedName(qualifier, table.getName()), "output", getOutput(), "stats", statistics, - "conjuncts", conjuncts, "selected partitions num", selectedPartitions.isPruned ? selectedPartitions.selectedPartitions.size() : "unknown", "isIncremental", incrementalRelation.isPresent() diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java index d68a459bd3799c..7923c8af9f41e6 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalJdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,27 +28,21 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical jdbc scan for external catalog. */ public class PhysicalJdbcScan extends PhysicalCatalogRelation { - private final Set conjuncts; - /** * Constructor for PhysicalJdbcScan. */ public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, - Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { + Optional groupExpression, LogicalProperties logicalProperties) { this(id, table, qualifier, groupExpression, logicalProperties, - null, null, conjuncts); + null, null); } /** @@ -57,11 +50,9 @@ public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, */ public PhysicalJdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_JDBC_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -80,23 +71,19 @@ public R accept(PlanVisitor visitor, C context) { @Override public PhysicalJdbcScan withGroupExpression(Optional groupExpression) { - return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts); + return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get()); } @Override public PhysicalJdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalJdbcScan(relationId, table, qualifier, groupExpression, - getLogicalProperties(), physicalProperties, statistics, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + getLogicalProperties(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java index f63b5a8d420206..6ae6b931120d82 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/physical/PhysicalOdbcScan.java @@ -21,7 +21,6 @@ import org.apache.doris.nereids.memo.GroupExpression; import org.apache.doris.nereids.properties.LogicalProperties; import org.apache.doris.nereids.properties.PhysicalProperties; -import org.apache.doris.nereids.trees.expressions.Expression; import org.apache.doris.nereids.trees.plans.Plan; import org.apache.doris.nereids.trees.plans.PlanType; import org.apache.doris.nereids.trees.plans.RelationId; @@ -29,27 +28,20 @@ import org.apache.doris.nereids.util.Utils; import org.apache.doris.statistics.Statistics; -import com.google.common.collect.ImmutableSet; - import java.util.List; -import java.util.Objects; import java.util.Optional; -import java.util.Set; /** * Physical odbc scan for odbc table. */ public class PhysicalOdbcScan extends PhysicalCatalogRelation { - - private final Set conjuncts; - /** * Constructor for PhysicalOdbcScan. */ public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, - Optional groupExpression, LogicalProperties logicalProperties, Set conjuncts) { + Optional groupExpression, LogicalProperties logicalProperties) { this(id, table, qualifier, groupExpression, logicalProperties, - null, null, conjuncts); + null, null); } /** @@ -57,11 +49,9 @@ public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, */ public PhysicalOdbcScan(RelationId id, TableIf table, List qualifier, Optional groupExpression, - LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics, - Set conjuncts) { + LogicalProperties logicalProperties, PhysicalProperties physicalProperties, Statistics statistics) { super(id, PlanType.PHYSICAL_ODBC_SCAN, table, qualifier, groupExpression, logicalProperties, physicalProperties, statistics); - this.conjuncts = ImmutableSet.copyOf(Objects.requireNonNull(conjuncts, "conjuncts should not be null")); } @Override @@ -80,23 +70,19 @@ public R accept(PlanVisitor visitor, C context) { @Override public PhysicalOdbcScan withGroupExpression(Optional groupExpression) { - return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties(), conjuncts); + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, getLogicalProperties()); } @Override public Plan withGroupExprLogicalPropChildren(Optional groupExpression, Optional logicalProperties, List children) { - return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get(), conjuncts); + return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, logicalProperties.get()); } @Override public PhysicalOdbcScan withPhysicalPropertiesAndStats(PhysicalProperties physicalProperties, Statistics statistics) { return new PhysicalOdbcScan(relationId, table, qualifier, groupExpression, - getLogicalProperties(), physicalProperties, statistics, conjuncts); - } - - public Set getConjuncts() { - return this.conjuncts; + getLogicalProperties(), physicalProperties, statistics); } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java index 0871e3dca375bf..fef94ff52f97c3 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/RelationVisitor.java @@ -24,7 +24,6 @@ import org.apache.doris.nereids.trees.plans.logical.LogicalDeferMaterializeOlapScan; import org.apache.doris.nereids.trees.plans.logical.LogicalEmptyRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalEsScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalExternalRelation; import org.apache.doris.nereids.trees.plans.logical.LogicalFileScan; import org.apache.doris.nereids.trees.plans.logical.LogicalHudiScan; import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; @@ -95,16 +94,12 @@ default R visitLogicalEmptyRelation(LogicalEmptyRelation emptyRelation, C contex return visitLogicalRelation(emptyRelation, context); } - default R visitLogicalExternalRelation(LogicalExternalRelation relation, C context) { - return visitLogicalCatalogRelation(relation, context); - } - default R visitLogicalEsScan(LogicalEsScan esScan, C context) { - return visitLogicalExternalRelation(esScan, context); + return visitLogicalCatalogRelation(esScan, context); } default R visitLogicalFileScan(LogicalFileScan fileScan, C context) { - return visitLogicalExternalRelation(fileScan, context); + return visitLogicalCatalogRelation(fileScan, context); } default R visitLogicalHudiScan(LogicalHudiScan fileScan, C context) { @@ -112,11 +107,11 @@ default R visitLogicalHudiScan(LogicalHudiScan fileScan, C context) { } default R visitLogicalJdbcScan(LogicalJdbcScan jdbcScan, C context) { - return visitLogicalExternalRelation(jdbcScan, context); + return visitLogicalCatalogRelation(jdbcScan, context); } default R visitLogicalOdbcScan(LogicalOdbcScan odbcScan, C context) { - return visitLogicalExternalRelation(odbcScan, context); + return visitLogicalCatalogRelation(odbcScan, context); } default R visitLogicalOlapScan(LogicalOlapScan olapScan, C context) { diff --git a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java index fb6ac9859341d8..c875ef6bc2f141 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/external/hms/HmsCatalogTest.java @@ -49,6 +49,7 @@ public class HmsCatalogTest extends AnalyzeCheckTestBase { private static final String HMS_CATALOG = "hms_ctl"; + private static final long NOW = System.currentTimeMillis(); private Env env; private CatalogMgr mgr; @@ -101,6 +102,11 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(db, "initialized", true); Deencapsulation.setField(tbl, "objectCreated", true); + Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); + Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "catalog", hmsCatalog); + Deencapsulation.setField(tbl, "dbName", "hms_db"); + Deencapsulation.setField(tbl, "name", "hms_tbl"); new Expectations(tbl) { { tbl.getId(); @@ -131,6 +137,10 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { minTimes = 0; result = TableIf.TableType.HMS_EXTERNAL_TABLE; + // mock initSchemaAndUpdateTime and do nothing + tbl.initSchemaAndUpdateTime(); + minTimes = 0; + tbl.getDatabase(); minTimes = 0; result = db; @@ -142,6 +152,11 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view1, "objectCreated", true); + Deencapsulation.setField(view1, "schemaUpdateTime", NOW); + Deencapsulation.setField(view1, "eventUpdateTime", 0); + Deencapsulation.setField(view1, "catalog", hmsCatalog); + Deencapsulation.setField(view1, "dbName", "hms_db"); + Deencapsulation.setField(view1, "name", "hms_view1"); new Expectations(view1) { { @@ -188,6 +203,11 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view2, "objectCreated", true); + Deencapsulation.setField(view2, "schemaUpdateTime", NOW); + Deencapsulation.setField(view2, "eventUpdateTime", 0); + Deencapsulation.setField(view2, "catalog", hmsCatalog); + Deencapsulation.setField(view2, "dbName", "hms_db"); + Deencapsulation.setField(view2, "name", "hms_view2"); new Expectations(view2) { { @@ -234,6 +254,11 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view3, "objectCreated", true); + Deencapsulation.setField(view3, "schemaUpdateTime", NOW); + Deencapsulation.setField(view3, "eventUpdateTime", 0); + Deencapsulation.setField(view3, "catalog", hmsCatalog); + Deencapsulation.setField(view3, "dbName", "hms_db"); + Deencapsulation.setField(view3, "name", "hms_view3"); new Expectations(view3) { { @@ -280,6 +305,11 @@ private void createDbAndTableForHmsCatalog(HMSExternalCatalog hmsCatalog) { }; Deencapsulation.setField(view4, "objectCreated", true); + Deencapsulation.setField(view4, "schemaUpdateTime", NOW); + Deencapsulation.setField(view4, "eventUpdateTime", 0); + Deencapsulation.setField(view4, "catalog", hmsCatalog); + Deencapsulation.setField(view4, "dbName", "hms_db"); + Deencapsulation.setField(view4, "name", "hms_view4"); new Expectations(view4) { { diff --git a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java b/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java deleted file mode 100644 index 023f9c4f7ff33a..00000000000000 --- a/fe/fe-core/src/test/java/org/apache/doris/nereids/rules/rewrite/AdjustNullableTest.java +++ /dev/null @@ -1,74 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.rewrite; - -import org.apache.doris.nereids.trees.expressions.ExprId; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThan; -import org.apache.doris.nereids.trees.expressions.Slot; -import org.apache.doris.nereids.trees.expressions.SlotReference; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.plans.RelationId; -import org.apache.doris.nereids.trees.plans.logical.LogicalJdbcScan; -import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan; -import org.apache.doris.nereids.types.IntegerType; -import org.apache.doris.nereids.util.MemoPatternMatchSupported; -import org.apache.doris.nereids.util.PlanConstructor; - -import mockit.Mock; -import mockit.MockUp; -import org.junit.jupiter.api.Assertions; -import org.junit.jupiter.api.Test; - -import java.util.ArrayList; -import java.util.HashSet; -import java.util.Optional; -import java.util.Set; - -/** - * Tests for {@link AdjustNullableTest}. - */ -class AdjustNullableTest implements MemoPatternMatchSupported { - private final LogicalOlapScan scan = PlanConstructor.newLogicalOlapScan(0, "t1", 0); - - @Test - void testLogicalExternalRelation() { - new MockUp() { - @Mock - public Set getOutputSet() { - Set output = new HashSet<>(); - output.add(new SlotReference(new ExprId(1), "id", IntegerType.INSTANCE, false, - new ArrayList<>())); - return output; - } - }; - - GreaterThan gt = new GreaterThan(new SlotReference(new ExprId(1), "id", - IntegerType.INSTANCE, true, new ArrayList<>()), Literal.of("1")); - Set conjuncts = new HashSet<>(); - conjuncts.add(gt); - Assertions.assertTrue(conjuncts.iterator().next().nullable()); - LogicalJdbcScan jdbcScan = - new LogicalJdbcScan(new RelationId(1), PlanConstructor.newOlapTable(0, "t1", 0), - new ArrayList<>(), Optional.empty(), Optional.empty(), conjuncts); - AdjustNullable adjustNullable = new AdjustNullable(); - LogicalJdbcScan newJdbcScan = (LogicalJdbcScan) adjustNullable.rewriteRoot(jdbcScan, null); - conjuncts = newJdbcScan.getConjuncts(); - Assertions.assertFalse(conjuncts.iterator().next().nullable()); - } -} diff --git a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java index 86c460e6bfd603..bddb3c8185ae72 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/qe/HmsQueryCacheTest.java @@ -120,6 +120,9 @@ private void init(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(tbl, "objectCreated", true); Deencapsulation.setField(tbl, "schemaUpdateTime", NOW); Deencapsulation.setField(tbl, "eventUpdateTime", 0); + Deencapsulation.setField(tbl, "catalog", hmsCatalog); + Deencapsulation.setField(tbl, "dbName", "hms_db"); + Deencapsulation.setField(tbl, "name", "hms_tbl"); new Expectations(tbl) { { tbl.getId(); @@ -167,6 +170,9 @@ private void init(HMSExternalCatalog hmsCatalog) { Deencapsulation.setField(tbl2, "objectCreated", true); Deencapsulation.setField(tbl2, "schemaUpdateTime", NOW); Deencapsulation.setField(tbl2, "eventUpdateTime", 0); + Deencapsulation.setField(tbl2, "catalog", hmsCatalog); + Deencapsulation.setField(tbl2, "dbName", "hms_db"); + Deencapsulation.setField(tbl2, "name", "hms_tbl2"); new Expectations(tbl2) { { tbl2.getId();