Skip to content

Support identical column name in different index. Cherry pick #32957

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Jibing-Li merged 1 commit into from
Mar 28, 2024
Merged

Support identical column name in different index. Cherry pick #32957

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 5 additions & 4 deletions fe/fe-core/src/main/java/org/apache/doris/analysis/ShowColumnStatsStmt.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@
import org.apache.doris.qe.ConnectContext;
import org.apache.doris.qe.ShowResultSet;
import org.apache.doris.qe.ShowResultSetMetaData;
import org.apache.doris.statistics.AnalysisManager;
import org.apache.doris.statistics.ColStatsMeta;
import org.apache.doris.statistics.ColumnStatistic;

Expand Down Expand Up @@ -140,23 +141,23 @@ public TableIf getTable() {

public ShowResultSet constructResultSet(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics) {
List<List<String>> result = Lists.newArrayList();
AnalysisManager analysisManager = Env.getCurrentEnv().getAnalysisManager();
columnStatistics.forEach(p -> {
if (p.second.isUnKnown) {
return;
}

List<String> row = Lists.newArrayList();
row.add(p.first.first);
// p data structure is Pair<Pair<IndexName, ColumnName>, ColumnStatistic>
row.add(p.first.second);
row.add(p.first.first);
row.add(String.valueOf(p.second.count));
row.add(String.valueOf(p.second.ndv));
row.add(String.valueOf(p.second.numNulls));
row.add(String.valueOf(p.second.dataSize));
row.add(String.valueOf(p.second.avgSizeByte));
row.add(String.valueOf(p.second.minExpr == null ? "N/A" : p.second.minExpr.toSql()));
row.add(String.valueOf(p.second.maxExpr == null ? "N/A" : p.second.maxExpr.toSql()));
ColStatsMeta colStatsMeta = Env.getCurrentEnv().getAnalysisManager().findColStatsMeta(table.getId(),
p.first.first);
ColStatsMeta colStatsMeta = analysisManager.findColStatsMeta(table.getId(), p.first.first, p.first.second);
row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.analysisMethod));
row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.analysisType));
row.add(String.valueOf(colStatsMeta == null ? "N/A" : colStatsMeta.jobType));
Expand Down
23 changes: 21 additions & 2 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,7 @@
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.HistogramTask;
import org.apache.doris.statistics.OlapAnalysisTask;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.system.Backend;
import org.apache.doris.system.SystemInfoService;
import org.apache.doris.thrift.TColumn;
Expand Down Expand Up @@ -93,6 +94,7 @@
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Optional;
Expand Down Expand Up @@ -754,8 +756,8 @@ public List<Column> getSchemaByIndexId(Long indexId, boolean full) {
}

@Override
public List<Column> getSchemaAllIndexes(boolean full) {
List<Column> columns = Lists.newArrayList();
public Set<Column> getSchemaAllIndexes(boolean full) {
Set<Column> columns = Sets.newHashSet();
for (Long indexId : indexIdToMeta.keySet()) {
columns.addAll(getSchemaByIndexId(indexId, full));
}
Expand Down Expand Up @@ -1284,6 +1286,23 @@ public BaseAnalysisTask createAnalysisTask(AnalysisInfo info) {
}
}

@Override
public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
Set<Pair<String, String>> ret = Sets.newHashSet();
// Check the schema of all indexes for each given column name,
// If the column name exists in the index, add the <IndexName, ColumnName> pair to return list.
for (String column : columns) {
for (MaterializedIndexMeta meta : indexIdToMeta.values()) {
Column col = meta.getColumnByName(column);
if (col == null || StatisticsUtil.isUnsupportedType(col.getType())) {
continue;
}
ret.add(Pair.of(getIndexNameById(meta.getIndexId()), column.toLowerCase(Locale.ROOT)));
}
}
return ret;
}

@Override
public long fetchRowCount() {
long rowCount = 0;
Expand Down
12 changes: 7 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/catalog/Table.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.doris.common.ErrorCode;
import org.apache.doris.common.FeMetaVersion;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.QueryableReentrantReadWriteLock;
Expand All @@ -39,6 +40,7 @@
import com.google.common.base.Strings;
import com.google.common.collect.Lists;
import com.google.common.collect.Maps;
import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import org.apache.commons.lang3.NotImplementedException;
import org.apache.commons.lang3.StringUtils;
Expand Down Expand Up @@ -384,11 +386,6 @@ public List<Column> getBaseSchema() {
return getBaseSchema(Util.showHiddenColumns());
}

@Override
public List<Column> getSchemaAllIndexes(boolean full) {
return getBaseSchema();
}

public List<Column> getBaseSchema(boolean full) {
if (full) {
return fullSchema;
Expand Down Expand Up @@ -650,4 +647,9 @@ public List<Long> getChunkSizes() {
public long fetchRowCount() {
return 0;
}

@Override
public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
return Sets.newHashSet();
}
}
13 changes: 12 additions & 1 deletion fe/fe-core/src/main/java/org/apache/doris/catalog/TableIf.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import org.apache.doris.cluster.ClusterNamespace;
import org.apache.doris.common.DdlException;
import org.apache.doris.common.MetaNotFoundException;
import org.apache.doris.common.Pair;
import org.apache.doris.nereids.exceptions.AnalysisException;
import org.apache.doris.persist.AlterConstraintLog;
import org.apache.doris.statistics.AnalysisInfo;
Expand Down Expand Up @@ -118,7 +119,11 @@ default boolean tryWriteLockIfExist(long timeout, TimeUnit unit) {

List<Column> getBaseSchema();

List<Column> getSchemaAllIndexes(boolean full);
default Set<Column> getSchemaAllIndexes(boolean full) {
Set<Column> ret = Sets.newHashSet();
ret.addAll(getBaseSchema());
return ret;
}

default List<Column> getBaseSchemaOrEmpty() {
try {
Expand Down Expand Up @@ -181,6 +186,12 @@ default long getRowCountForNereids() {

Optional<ColumnStatistic> getColumnStatistic(String colName);

/**
* @param columns Set of column names.
* @return List of pairs. Each pair is <IndexName, ColumnName>. For external table, index name is table name.
*/
Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns);

// Get all the chunk sizes of this table. Now, only HMS external table implemented this interface.
// For HMS external table, the return result is a list of all the files' size.
List<Long> getChunkSizes();
Expand Down
24 changes: 19 additions & 5 deletions fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalTable.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import org.apache.doris.catalog.TableIf;
import org.apache.doris.catalog.constraint.Constraint;
import org.apache.doris.common.AnalysisException;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.common.util.Util;
Expand All @@ -32,8 +33,10 @@
import org.apache.doris.statistics.AnalysisInfo;
import org.apache.doris.statistics.BaseAnalysisTask;
import org.apache.doris.statistics.ColumnStatistic;
import org.apache.doris.statistics.util.StatisticsUtil;
import org.apache.doris.thrift.TTableDescriptor;

import com.google.common.collect.Sets;
import com.google.gson.annotations.SerializedName;
import lombok.Getter;
import org.apache.commons.lang3.NotImplementedException;
Expand All @@ -44,8 +47,10 @@
import java.io.DataOutput;
import java.io.IOException;
import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.Optional;
import java.util.Set;

/**
* External table represent tables that are not self-managed by Doris.
Expand Down Expand Up @@ -145,11 +150,6 @@ public List<Column> getBaseSchema() {
return getFullSchema();
}

@Override
public List<Column> getSchemaAllIndexes(boolean full) {
return getBaseSchema();
}

@Override
public List<Column> getBaseSchema(boolean full) {
return getFullSchema();
Expand Down Expand Up @@ -311,6 +311,20 @@ public void gsonPostProcess() throws IOException {
objectCreated = false;
}

@Override
public Set<Pair<String, String>> getColumnIndexPairs(Set<String> columns) {
Set<Pair<String, String>> ret = Sets.newHashSet();
for (String column : columns) {
Column col = getColumn(column);
if (col == null || StatisticsUtil.isUnsupportedType(col.getType())) {
continue;
}
// External table put table name as index name.
ret.add(Pair.of(String.valueOf(name), column.toLowerCase(Locale.ROOT)));
}
return ret;
}

@Override
public List<Long> getChunkSizes() {
throw new NotImplementedException("getChunkSized not implemented");
Expand Down
37 changes: 20 additions & 17 deletions fe/fe-core/src/main/java/org/apache/doris/qe/ShowExecutor.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2536,16 +2536,18 @@ private void handleShowColumnStats() throws AnalysisException {
private void getStatsForAllColumns(List<Pair<Pair<String, String>, ColumnStatistic>> columnStatistics,
TableIf tableIf) throws AnalysisException {
List<ResultRow> resultRows = StatisticsRepository.queryColumnStatisticsForTable(tableIf.getId());
// row[4] is index id, row[5] is column name.
for (ResultRow row : resultRows) {
String indexName = "N/A";
String indexName = tableIf.getName();
long indexId = Long.parseLong(row.get(4));
if (indexId != -1) {
indexName = ((OlapTable) tableIf).getIndexNameById(indexId);
if (indexName == null) {
continue;
}
if (tableIf instanceof OlapTable) {
OlapTable olapTable = (OlapTable) tableIf;
indexName = olapTable.getIndexNameById(indexId == -1 ? olapTable.getBaseIndexId() : indexId);
}
if (indexName == null) {
continue;
}
columnStatistics.add(Pair.of(Pair.of(row.get(5), indexName), ColumnStatistic.fromResultRow(row)));
columnStatistics.add(Pair.of(Pair.of(indexName, row.get(5)), ColumnStatistic.fromResultRow(row)));
}
}

Expand All @@ -2562,28 +2564,29 @@ private void getStatsForSpecifiedColumns(List<Pair<Pair<String, String>, ColumnS
indexIds.add(-1L);
}
for (long indexId : indexIds) {
String indexName = "N/A";
if (indexId != -1) {
indexName = ((OlapTable) tableIf).getIndexNameById(indexId);
if (indexName == null) {
continue;
}
String indexName = tableIf.getName();
if (tableIf instanceof OlapTable) {
OlapTable olapTable = (OlapTable) tableIf;
indexName = olapTable.getIndexNameById(indexId == -1 ? olapTable.getBaseIndexId() : indexId);
}
if (indexName == null) {
continue;
}
// Show column statistics in columnStatisticsCache.
if (showCache) {
ColumnStatistic columnStatistic = Env.getCurrentEnv().getStatisticsCache().getColumnStatistics(
tableIf.getDatabase().getCatalog().getId(),
tableIf.getDatabase().getId(), tableIf.getId(), indexId, colName);
columnStatistics.add(Pair.of(Pair.of(colName, indexName), columnStatistic));
columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic));
} else if (partitionNames == null) {
ColumnStatistic columnStatistic =
StatisticsRepository.queryColumnStatisticsByName(tableIf.getId(), indexId, colName);
columnStatistics.add(Pair.of(Pair.of(colName, indexName), columnStatistic));
columnStatistics.add(Pair.of(Pair.of(indexName, colName), columnStatistic));
} else {
String finalIndexName = indexName;
columnStatistics.addAll(StatisticsRepository.queryColumnStatisticsByPartitions(tableName,
colName, partitionNames.getPartitionNames())
.stream().map(s -> Pair.of(Pair.of(colName, finalIndexName), s))
.stream().map(s -> Pair.of(Pair.of(finalIndexName, colName), s))
.collect(Collectors.toList()));
}
}
Expand Down Expand Up @@ -3017,7 +3020,7 @@ private void handleShowAnalyzeTaskStatus() {
if (table instanceof OlapTable && analysisInfo.indexId != -1) {
row.add(((OlapTable) table).getIndexNameById(analysisInfo.indexId));
} else {
row.add("N/A");
row.add(table.getName());
}
row.add(analysisInfo.message);
row.add(TimeUtils.DATETIME_FORMAT.format(
Expand Down
20 changes: 10 additions & 10 deletions fe/fe-core/src/main/java/org/apache/doris/statistics/AnalysisInfo.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
package org.apache.doris.statistics;

import org.apache.doris.catalog.TableIf;
import org.apache.doris.common.Pair;
import org.apache.doris.common.io.Text;
import org.apache.doris.common.io.Writable;
import org.apache.doris.persist.gson.GsonUtils;
Expand All @@ -35,7 +36,6 @@
import java.io.IOException;
import java.text.ParseException;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.StringJoiner;

Expand Down Expand Up @@ -95,8 +95,8 @@ public enum ScheduleType {
@SerializedName("tblId")
public final long tblId;

// TODO: Map here is wired, List is enough
public final Map<String, Set<String>> colToPartitions;
// Pair<IndexName, ColumnName>
public final Set<Pair<String, String>> jobColumns;

public final Set<String> partitionNames;

Expand Down Expand Up @@ -207,7 +207,7 @@ public enum ScheduleType {
public final JobPriority priority;

public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId, long dbId, long tblId,
Map<String, Set<String>> colToPartitions, Set<String> partitionNames, String colName, Long indexId,
Set<Pair<String, String>> jobColumns, Set<String> partitionNames, String colName, Long indexId,
JobType jobType, AnalysisMode analysisMode, AnalysisMethod analysisMethod, AnalysisType analysisType,
int samplePercent, long sampleRows, int maxBucketNum, long periodTimeInMs, String message,
long lastExecTimeInMs, long timeCostInMs, AnalysisState state, ScheduleType scheduleType,
Expand All @@ -221,7 +221,7 @@ public AnalysisInfo(long jobId, long taskId, List<Long> taskIds, long catalogId,
this.catalogId = catalogId;
this.dbId = dbId;
this.tblId = tblId;
this.colToPartitions = colToPartitions;
this.jobColumns = jobColumns;
this.partitionNames = partitionNames;
this.colName = colName;
this.indexId = indexId;
Expand Down Expand Up @@ -278,8 +278,8 @@ public String toString() {
if (maxBucketNum > 0) {
sj.add("MaxBucketNum: " + maxBucketNum);
}
if (colToPartitions != null) {
sj.add("colToPartitions: " + getColToPartitionStr());
if (jobColumns != null) {
sj.add("jobColumns: " + getJobColumns());
}
if (lastExecTimeInMs > 0) {
sj.add("LastExecTime: " + StatisticsUtil.getReadableTime(lastExecTimeInMs));
Expand Down Expand Up @@ -314,12 +314,12 @@ public void addTaskId(long taskId) {
taskIds.add(taskId);
}

public String getColToPartitionStr() {
if (colToPartitions == null || colToPartitions.isEmpty()) {
public String getJobColumns() {
if (jobColumns == null || jobColumns.isEmpty()) {
return "";
}
Gson gson = new Gson();
return gson.toJson(colToPartitions);
return gson.toJson(jobColumns);
}

@Override
Expand Down
Loading