Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -594,9 +594,7 @@ protected List<AutoAnalysisPendingJob> getPendingJobs(Map<TableNameInfo, Set<Pai
for (Entry<TableNameInfo, Set<Pair<String, String>>> entry : jobMap.entrySet()) {
TableNameInfo table = entry.getKey();
if (tableNameInfo == null
|| tableNameInfo.getCtl() == null && tableNameInfo.getDb() == null
&& tableNameInfo.getTbl() == null
|| tableNameInfo.equals(table)) {
|| matchesFilter(tableNameInfo, table)) {
result.add(new AutoAnalysisPendingJob(table.getCtl(),
table.getDb(), table.getTbl(), entry.getValue(), priority));
}
Expand All @@ -605,6 +603,29 @@ protected List<AutoAnalysisPendingJob> getPendingJobs(Map<TableNameInfo, Set<Pai
return result;
}

private boolean matchesFilter(TableNameInfo filter, TableNameInfo target) {
if (StringUtils.isEmpty(filter.getCtl())
&& StringUtils.isEmpty(filter.getDb())
&& StringUtils.isEmpty(filter.getTbl())) {
return true;
}

if (!StringUtils.isEmpty(filter.getCtl())
&& !filter.getCtl().equals(target.getCtl())) {
return false;
}
if (!StringUtils.isEmpty(filter.getDb())
&& !filter.getDb().equals(target.getDb())) {
return false;
}
if (!StringUtils.isEmpty(filter.getTbl())
&& !filter.getTbl().equals(target.getTbl())) {
return false;
}

return true;
}

public List<AnalysisInfo> findAnalysisJobs(String state, String ctl, String db,
String table, long jobId, boolean isAuto) {
TableIf tbl = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,9 @@
import org.apache.doris.common.DdlException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.util.MasterDaemon;
import org.apache.doris.datasource.ExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.persist.TableStatsDeletionLog;
import org.apache.doris.statistics.AnalysisInfo.AnalysisMethod;
Expand Down Expand Up @@ -148,6 +150,9 @@ protected void processOneJob(TableIf table, Set<Pair<String, String>> columns, J
if (StatisticsUtil.enablePartitionAnalyze() && table.isPartitionedTable()) {
analysisMethod = AnalysisMethod.FULL;
}
if (table instanceof ExternalTable) { // External table only support full analyze now
analysisMethod = AnalysisMethod.FULL;
}
boolean isSampleAnalyze = analysisMethod.equals(AnalysisMethod.SAMPLE);
OlapTable olapTable = table instanceof OlapTable ? (OlapTable) table : null;
AnalysisManager manager = Env.getServingEnv().getAnalysisManager();
Expand Down Expand Up @@ -229,9 +234,9 @@ protected boolean supportAutoAnalyze(TableIf tableIf) {
if (tableIf == null) {
return false;
}
return tableIf instanceof OlapTable
return tableIf instanceof OlapTable || tableIf instanceof IcebergExternalTable
|| tableIf instanceof HMSExternalTable
&& ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
&& ((HMSExternalTable) tableIf).getDlaType().equals(HMSExternalTable.DLAType.HIVE);
}

protected AnalysisInfo createAnalyzeJobForTbl(TableIf table, Set<Pair<String, String>> jobColumns,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -55,6 +55,7 @@
import org.apache.doris.datasource.InternalCatalog;
import org.apache.doris.datasource.hive.HMSExternalTable;
import org.apache.doris.datasource.hive.HMSExternalTable.DLAType;
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
import org.apache.doris.info.TableNameInfo;
import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral;
import org.apache.doris.nereids.trees.expressions.literal.IPv4Literal;
Expand Down Expand Up @@ -1125,15 +1126,17 @@ public static boolean needAnalyzeColumn(TableIf table, Pair<String, String> colu
// 3. Check partition
return needAnalyzePartition(olapTable, tableStatsStatus, columnStatsMeta);
} else {
// Now, we only support Hive external table auto analyze.
if (!(table instanceof HMSExternalTable)) {
if (!(table instanceof HMSExternalTable || (table instanceof IcebergExternalTable))) {
return false;
}
HMSExternalTable hmsTable = (HMSExternalTable) table;
if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
return false;
if (table instanceof HMSExternalTable) {
HMSExternalTable hmsTable = (HMSExternalTable) table;
if (!hmsTable.getDlaType().equals(DLAType.HIVE)) {
return false;
}
}
// External is hard to calculate change rate, use time interval to control analyze frequency.
// External is hard to calculate change rate, use time interval to control
// analyze frequency.
return System.currentTimeMillis()
- tableStatsStatus.lastAnalyzeTime > StatisticsUtil.getExternalTableAutoAnalyzeIntervalInMillis();
}
Expand Down
Loading