diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java index 12fb8a513427..1ba7c24c0daa 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/SqlNestedDataBenchmark.java @@ -175,7 +175,31 @@ public String getFormatString() "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 1005.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0", // 28, 29 "SELECT SUM(long1) FROM foo WHERE double3 < 2000.0 AND double3 > 1000.0", - "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0" + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 2000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0", + // 30, 31 + "SELECT SUM(long1) FROM foo WHERE double3 < 3000.0 AND double3 > 1000.0", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 3000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0", + // 32,33 + "SELECT SUM(long1) FROM foo WHERE double3 < 5000.0 AND double3 > 1000.0", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) < 5000.0 AND JSON_VALUE(nested, '$.nesteder.double3' RETURNING DOUBLE) > 1000.0", + // 34,35 smaller cardinality like range filter + "SELECT SUM(long1) FROM foo WHERE string1 LIKE '1%'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '1%'", + // 36,37 smaller cardinality like predicate filter + "SELECT SUM(long1) FROM foo WHERE string1 LIKE '%1%'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string1') LIKE '%1%'", + // 38-39 moderate cardinality like range + "SELECT SUM(long1) FROM foo WHERE string5 LIKE '1%'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '1%'", + // 40, 41 big cardinality lex range + "SELECT SUM(long1) FROM foo WHERE string5 > '1'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') > '1'", + // 42, 43 big cardinality like predicate filter + "SELECT SUM(long1) FROM foo WHERE string5 LIKE '%1%'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%'", + // 44, 45 big cardinality like filter + selector filter + "SELECT SUM(long1) FROM foo WHERE string5 LIKE '%1%' AND string1 = '1000'", + "SELECT SUM(JSON_VALUE(nested, '$.long1' RETURNING BIGINT)) FROM foo WHERE JSON_VALUE(nested, '$.nesteder.string5') LIKE '%1%' AND JSON_VALUE(nested, '$.nesteder.string1') = '1000'" ); @Param({"5000000"}) @@ -187,7 +211,11 @@ public String getFormatString() }) private String vectorize; - @Param({"none", "front-coded-4", "front-coded-16"}) + @Param({ + "none", + "front-coded-4", + "front-coded-16" + }) private String stringEncoding; @Param({ @@ -220,7 +248,23 @@ public String getFormatString() "26", "27", "28", - "29" + "29", + "30", + "31", + "32", + "33", + "34", + "35", + "36", + "37", + "38", + "39", + "40", + "41", + "42", + "43", + "44", + "45" }) private String query; @@ -253,7 +297,7 @@ public void setup() ImmutableList.of( new ExpressionTransform( "nested", - "json_object('long1', long1, 'nesteder', json_object('string1', string1, 'long2', long2, 'double3',double3))", + "json_object('long1', long1, 'nesteder', json_object('string1', string1, 'long2', long2, 'double3',double3, 'string5', string5))", TestExprMacroTable.INSTANCE ) ) diff --git a/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java b/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java index d8e4cd731bae..ac6c270bd3dd 100644 --- a/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java +++ b/processing/src/main/java/org/apache/druid/query/DruidProcessingConfig.java @@ -149,6 +149,20 @@ public int columnCacheSizeBytes() return 0; } + @Override + @Config(value = "${base_path}.indexes.skipValueRangeIndexScale") + public double skipValueRangeIndexScale() + { + return ColumnConfig.super.skipValueRangeIndexScale(); + } + + @Override + @Config(value = "${base_path}.indexes.skipValuePredicateIndexScale") + public double skipValuePredicateIndexScale() + { + return ColumnConfig.super.skipValuePredicateIndexScale(); + } + @Config(value = "${base_path}.fifo") public boolean isFifo() { diff --git a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java index 060f6b68ccfb..b49207f73c95 100644 --- a/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java +++ b/processing/src/main/java/org/apache/druid/segment/AutoTypeColumnMerger.java @@ -77,7 +77,6 @@ public class AutoTypeColumnMerger implements DimensionMergerV9 private final String name; private final IndexSpec indexSpec; private final SegmentWriteOutMedium segmentWriteOutMedium; - private final ProgressIndicator progressIndicator; private final Closer closer; private NestedCommonFormatColumnSerializer serializer; @@ -87,7 +86,6 @@ public AutoTypeColumnMerger( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - ProgressIndicator progressIndicator, Closer closer ) { @@ -95,7 +93,6 @@ public AutoTypeColumnMerger( this.name = name; this.indexSpec = indexSpec; this.segmentWriteOutMedium = segmentWriteOutMedium; - this.progressIndicator = progressIndicator; this.closer = closer; } @@ -142,7 +139,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = longSerializer; @@ -152,7 +148,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = doubleSerializer; @@ -162,7 +157,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = stringSerializer; @@ -172,7 +166,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = arraySerializer; @@ -191,7 +184,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = defaultSerializer; diff --git a/processing/src/main/java/org/apache/druid/segment/NestedCommonFormatColumnHandler.java b/processing/src/main/java/org/apache/druid/segment/NestedCommonFormatColumnHandler.java index 9f6bd439efad..d267af8b85e5 100644 --- a/processing/src/main/java/org/apache/druid/segment/NestedCommonFormatColumnHandler.java +++ b/processing/src/main/java/org/apache/druid/segment/NestedCommonFormatColumnHandler.java @@ -80,7 +80,7 @@ public DimensionMergerV9 makeMerger( Closer closer ) { - return new AutoTypeColumnMerger(name, indexSpec, segmentWriteOutMedium, progress, closer); + return new AutoTypeColumnMerger(name, indexSpec, segmentWriteOutMedium, closer); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/NestedDataColumnMerger.java b/processing/src/main/java/org/apache/druid/segment/NestedDataColumnMerger.java index caa6552d53f9..f1a4ea7bc091 100644 --- a/processing/src/main/java/org/apache/druid/segment/NestedDataColumnMerger.java +++ b/processing/src/main/java/org/apache/druid/segment/NestedDataColumnMerger.java @@ -54,7 +54,6 @@ public class NestedDataColumnMerger implements DimensionMergerV9 private final String name; private final IndexSpec indexSpec; private final SegmentWriteOutMedium segmentWriteOutMedium; - private final ProgressIndicator progressIndicator; private final Closer closer; private ColumnDescriptor.Builder descriptorBuilder; @@ -64,7 +63,6 @@ public NestedDataColumnMerger( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - ProgressIndicator progressIndicator, Closer closer ) { @@ -72,7 +70,6 @@ public NestedDataColumnMerger( this.name = name; this.indexSpec = indexSpec; this.segmentWriteOutMedium = segmentWriteOutMedium; - this.progressIndicator = progressIndicator; this.closer = closer; } @@ -115,7 +112,6 @@ public void writeMergedValueDictionary(List adapters) throws I name, indexSpec, segmentWriteOutMedium, - progressIndicator, closer ); serializer = defaultSerializer; diff --git a/processing/src/main/java/org/apache/druid/segment/NestedDataDimensionHandler.java b/processing/src/main/java/org/apache/druid/segment/NestedDataDimensionHandler.java index 3eda293a7af7..a202155a0d04 100644 --- a/processing/src/main/java/org/apache/druid/segment/NestedDataDimensionHandler.java +++ b/processing/src/main/java/org/apache/druid/segment/NestedDataDimensionHandler.java @@ -80,7 +80,7 @@ public DimensionMergerV9 makeMerger( Closer closer ) { - return new NestedDataColumnMerger(name, indexSpec, segmentWriteOutMedium, progress, closer); + return new NestedDataColumnMerger(name, indexSpec, segmentWriteOutMedium, closer); } @Override diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnConfig.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnConfig.java index ed3fa6b44264..273f0dfb7650 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ColumnConfig.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnConfig.java @@ -22,4 +22,70 @@ public interface ColumnConfig { int columnCacheSizeBytes(); + + /** + * If the total number of rows in a column multiplied by this value is smaller than the total number of bitmap + * index operations required to perform to use a {@link LexicographicalRangeIndex} or {@link NumericRangeIndex}, + * then for any {@link ColumnIndexSupplier} which chooses to participate in this config it will skip computing the + * index, indicated by a return value of null from the 'forRange' methods, to force the filter to be processed + * with a scan using a {@link org.apache.druid.query.filter.ValueMatcher} instead. + *

+ * For range indexes on columns where every value has an index, the number of bitmap operations is determined by how + * many individual values fall in the range, a subset of the columns total cardinality. + *

+ * Currently only the {@link org.apache.druid.segment.nested.NestedCommonFormatColumn} implementations of + * {@link ColumnIndexSupplier} support this behavior. + *

+ * This can make some standalone filters faster in cases where the overhead of walking the value dictionary and + * combining bitmaps to construct a {@link org.apache.druid.segment.BitmapOffset} or + * {@link org.apache.druid.segment.vector.BitmapVectorOffset} can exceed the cost of just using doing a full scan + * and using a {@link org.apache.druid.query.filter.ValueMatcher}. + *

+ * Where this is especially useful is in cases where the range index is used as part of some + * {@link org.apache.druid.segment.filter.AndFilter}, which segment processing partitions into groups of 'pre' + * filters, composed of those which should use indexes, and 'post' filters, which should use a matcher on the offset + * created by the indexes to filter the remaining results. This value pushes what would have been expensive index + * computations to go into the 'pre' group into using a value matcher as part of the 'post' group instead, sometimes + * providing an order of magnitude or higher performance increase. + */ + default double skipValueRangeIndexScale() + { + // this value was chosen testing bound filters on double columns with a variety of ranges at which this ratio + // of number of bitmaps compared to total number of rows appeared to be around the threshold where indexes stopped + // performing consistently faster than a full scan + value matcher + return 0.08; + } + + /** + * If the total number of rows in a column multiplied by this value is smaller than the total number of bitmap + * index operations required to perform to use a {@link DruidPredicateIndex} then for any {@link ColumnIndexSupplier} + * which chooses to participate in this config it will skip computing the index, in favor of doing a full scan and + * using a {@link org.apache.druid.query.filter.ValueMatcher} instead. This is indicated returning null from + * {@link ColumnIndexSupplier#as(Class)} even though it would have otherwise been able to create a + * {@link BitmapColumnIndex}. For predicate indexes, this is determined by the total value cardinality of the column + * for columns with an index for every value. + *

+ * Currently only the {@link org.apache.druid.segment.nested.NestedCommonFormatColumn} implementations of + * {@link ColumnIndexSupplier} support this behavior. + *

+ * This can make some standalone filters faster in cases where the overhead of walking the value dictionary and + * combining bitmaps to construct a {@link org.apache.druid.segment.BitmapOffset} or + * {@link org.apache.druid.segment.vector.BitmapVectorOffset} can exceed the cost of just using doing a full scan + * and using a {@link org.apache.druid.query.filter.ValueMatcher}. + *

+ * Where this is especially useful is in cases where the predicate index is used as part of some + * {@link org.apache.druid.segment.filter.AndFilter}, which segment processing partitions into groups of 'pre' + * filters, composed of those which should use indexes, and 'post' filters, which should use a matcher on the offset + * created by the indexes to filter the remaining results. This value pushes what would have been expensive index + * computations to go into the 'pre' group into using a value matcher as part of the 'post' group instead, sometimes + * providing an order of magnitude or higher performance increase. + *

+ * This value is separate from {@link #skipValueRangeIndexScale()} since the dynamics of computing predicate indexes + * is potentially different than the much cheaper range calculations (especially for numeric values), so having a + * separate control knob allows for corrections to be done to tune things separately from ranges. + */ + default double skipValuePredicateIndexScale() + { + return 0.08; + } } diff --git a/processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java index 7e2b21a25d06..4534745b9ef8 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/column/ColumnIndexSupplier.java @@ -44,4 +44,14 @@ public interface ColumnIndexSupplier */ @Nullable T as(Class clazz); + + static boolean skipComputingRangeIndexes(ColumnConfig columnConfig, int numRows, int rangeSize) + { + return rangeSize > (int) Math.ceil(columnConfig.skipValueRangeIndexScale() * numRows); + } + + static boolean skipComputingPredicateIndexes(ColumnConfig columnConfig, int numRowsToScan, int dictionaryCardinality) + { + return dictionaryCardinality > (int) Math.ceil(columnConfig.skipValuePredicateIndexScale() * numRowsToScan); + } } diff --git a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java index f75d9e2926a4..6913ac998948 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/DictionaryEncodedStringValueIndex.java @@ -19,6 +19,8 @@ package org.apache.druid.segment.column; +import org.apache.druid.collections.bitmap.BitmapFactory; + import javax.annotation.Nullable; /** @@ -41,4 +43,7 @@ public interface DictionaryEncodedStringValueIndex extends DictionaryEncodedValu */ @Nullable String getValue(int index); + + @SuppressWarnings({"unreachable", "unused"}) + BitmapFactory getBitmapFactory(); } diff --git a/processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java b/processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java index 7fec02b9e36e..44f69279aa9e 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/DruidPredicateIndex.java @@ -21,10 +21,19 @@ import org.apache.druid.query.filter.DruidPredicateFactory; +import javax.annotation.Nullable; + /** * Uses a {@link DruidPredicateFactory} to construct a {@link BitmapColumnIndex} */ public interface DruidPredicateIndex { + /** + * Get a {@link BitmapColumnIndex} corresponding to all the rows that match the supplied {@link DruidPredicateFactory} + *

+ * If this method returns null it indicates that there was no index that matched the respective values and a + * {@link org.apache.druid.query.filter.ValueMatcher} must be used instead. + */ + @Nullable BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory); } diff --git a/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDictionaryEncodedStringValueIndex.java b/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDictionaryEncodedStringValueIndex.java index 49ef97ea6c23..668694bb4a13 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDictionaryEncodedStringValueIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDictionaryEncodedStringValueIndex.java @@ -56,6 +56,12 @@ public String getValue(int index) return dictionary.get(index); } + @Override + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } + @Override public ImmutableBitmap getBitmap(int idx) { diff --git a/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDruidPredicateIndex.java b/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDruidPredicateIndex.java index c6c8941e5254..49badda066c8 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDruidPredicateIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/IndexedStringDruidPredicateIndex.java @@ -31,24 +31,63 @@ public final class IndexedStringDruidPredicateIndex> implements DruidPredicateIndex { + static final ColumnConfig ALWAYS_USE_INDEXES = new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + + @Override + public double skipValueRangeIndexScale() + { + return 1.0; + } + + @Override + public double skipValuePredicateIndexScale() + { + return 1.0; + } + }; private final BitmapFactory bitmapFactory; private final TDictionary dictionary; private final Indexed bitmaps; + private final ColumnConfig columnConfig; + private final int numRows; public IndexedStringDruidPredicateIndex( BitmapFactory bitmapFactory, TDictionary dictionary, Indexed bitmaps ) + { + this(bitmapFactory, dictionary, bitmaps, ALWAYS_USE_INDEXES, Integer.MAX_VALUE); + } + + public IndexedStringDruidPredicateIndex( + BitmapFactory bitmapFactory, + TDictionary dictionary, + Indexed bitmaps, + @Nullable ColumnConfig columnConfig, + int numRows + ) { this.bitmapFactory = bitmapFactory; this.dictionary = dictionary; this.bitmaps = bitmaps; + this.columnConfig = columnConfig; + this.numRows = numRows; } @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, dictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override diff --git a/processing/src/main/java/org/apache/druid/segment/column/IndexedUtf8LexicographicalRangeIndex.java b/processing/src/main/java/org/apache/druid/segment/column/IndexedUtf8LexicographicalRangeIndex.java index 7b1086ba8aaa..3f5c121e9ed2 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/IndexedUtf8LexicographicalRangeIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/IndexedUtf8LexicographicalRangeIndex.java @@ -44,21 +44,46 @@ public final class IndexedUtf8LexicographicalRangeIndex bitmaps; private final boolean hasNull; + private final ColumnConfig columnConfig; + private final int numRows; + public IndexedUtf8LexicographicalRangeIndex( BitmapFactory bitmapFactory, TDictionary dictionary, Indexed bitmaps, boolean hasNull ) + { + this( + bitmapFactory, + dictionary, + bitmaps, + hasNull, + IndexedStringDruidPredicateIndex.ALWAYS_USE_INDEXES, + Integer.MAX_VALUE + ); + } + + public IndexedUtf8LexicographicalRangeIndex( + BitmapFactory bitmapFactory, + TDictionary dictionary, + Indexed bitmaps, + boolean hasNull, + @Nullable ColumnConfig columnConfig, + int numRows + ) { Preconditions.checkArgument(dictionary.isSorted(), "Dictionary must be sorted"); this.bitmapFactory = bitmapFactory; this.dictionary = dictionary; this.bitmaps = bitmaps; this.hasNull = hasNull; + this.columnConfig = columnConfig; + this.numRows = numRows; } @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -66,6 +91,11 @@ public BitmapColumnIndex forRange( boolean endStrict ) { + final IntIntPair range = getRange(startValue, startStrict, endValue, endStrict); + final int start = range.leftInt(), end = range.rightInt(); + if (ColumnIndexSupplier.skipComputingRangeIndexes(columnConfig, numRows, end - start)) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -94,6 +124,7 @@ public ImmutableBitmap next() } @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, diff --git a/processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java b/processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java index 872eadf40dae..83c8fcfabd54 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/LexicographicalRangeIndex.java @@ -34,7 +34,11 @@ public interface LexicographicalRangeIndex * Get a {@link BitmapColumnIndex} corresponding to the values supplied in the specified range. If supplied starting * value is null, the range will begin at the first non-null value in the underlying value dictionary. If the end * value is null, the range will extend to the last value in the underlying value dictionary. + *

+ * If this method returns null it indicates that there is no index available that matches the requested range and a + * {@link org.apache.druid.query.filter.ValueMatcher} must be used instead. */ + @Nullable BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -47,10 +51,14 @@ BitmapColumnIndex forRange( * also match some predicate, such as to match a prefix. If supplied starting value is null, the range will begin at * the first non-null value in the underlying value dictionary that matches the predicate. If the end value is null, * the range will extend to the last value in the underlying value dictionary that matches the predicate. - * + *

* If the provided {@code} matcher is always true, it's better to use the other * {@link #forRange(String, boolean, String, boolean)} method. + *

+ * If this method returns null it indicates that there is no index available that matches the requested range and a + * {@link org.apache.druid.query.filter.ValueMatcher} must be used instead. */ + @Nullable BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, diff --git a/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java index bb9bf5c62541..cebece48b2cc 100644 --- a/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java +++ b/processing/src/main/java/org/apache/druid/segment/column/NumericRangeIndex.java @@ -32,7 +32,11 @@ public interface NumericRangeIndex * Get a {@link BitmapColumnIndex} corresponding to the values supplied in the specified range. If supplied starting * value is null, the range will begin at the first non-null value in the underlying value dictionary. If the end * value is null, the range will extend to the last value in the underlying value dictionary. + *

+ * If this method returns null it indicates that there is no index available that matches the requested range and a + * {@link org.apache.druid.query.filter.ValueMatcher} must be used instead. */ + @Nullable BitmapColumnIndex forRange( @Nullable Number startValue, boolean startStrict, diff --git a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java index b4c1463b0785..73ae5b6b0b83 100644 --- a/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java +++ b/processing/src/main/java/org/apache/druid/segment/filter/BoundFilter.java @@ -89,11 +89,13 @@ public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) boundDimFilter.getUpper(), boundDimFilter.isUpperStrict() ); - // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set - if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { - return rangeBitmaps; - } else { - return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); + if (rangeBitmaps != null) { + // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set + if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { + return rangeBitmaps; + } else { + return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); + } } } } @@ -112,16 +114,20 @@ public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) upper, boundDimFilter.isUpperStrict() ); - // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set - if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { - return rangeBitmaps; - } else { - return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); + if (rangeBitmaps != null) { + // preserve sad backwards compatible behavior where bound filter matches 'null' if the lower bound is not set + if (boundDimFilter.hasLowerBound() && !NullHandling.isNullOrEquivalent(boundDimFilter.getLower())) { + return rangeBitmaps; + } else { + return wrapRangeIndexWithNullValueIndex(indexSupplier, rangeBitmaps); + } } } } + // fall back to predicate based index if it is available return Filters.makePredicateIndex(boundDimFilter.getDimension(), selector, getPredicateFactory()); + } @Nullable diff --git a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java index fbee9f8d6099..d71f3fa9fc30 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/CompressedNestedDataComplexColumn.java @@ -95,6 +95,7 @@ public abstract class CompressedNestedDataComplexColumn parsePath(String path); @@ -974,17 +976,24 @@ private ColumnHolder readNestedFieldColumn(String field, int fieldIndex) columnBuilder.setHasMultipleValues(false) .setHasNulls(hasNull) .setDictionaryEncodedColumnSupplier(columnSupplier); + + final int size; + try (ColumnarInts throwAway = ints.get()) { + size = throwAway.size(); + } columnBuilder.setIndexSupplier( new NestedFieldColumnIndexSupplier( types, bitmapSerdeFactory.getBitmapFactory(), + columnConfig, rBitmaps, localDictionarySupplier, stringDictionarySupplier, longDictionarySupplier, doubleDictionarySupplier, arrayElementDictionarySupplier, - arrayElementBitmaps + arrayElementBitmaps, + size ), true, false diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java index 526e9f6e6ef1..3b8590c01784 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializer.java @@ -34,7 +34,6 @@ import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexMerger; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.StringEncodingStrategies; import org.apache.druid.segment.column.Types; @@ -146,7 +145,6 @@ public NestedDataColumnSerializer( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java index 881147a1cd8b..034c7bc64f5d 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedDataColumnSerializerV4.java @@ -36,7 +36,6 @@ import org.apache.druid.segment.GenericColumnSerializer; import org.apache.druid.segment.IndexMerger; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.StringEncodingStrategies; import org.apache.druid.segment.column.Types; @@ -140,7 +139,6 @@ public NestedDataColumnSerializerV4( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java index 7cf2655440c8..465d8cb30c55 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplier.java @@ -45,6 +45,7 @@ import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.segment.IntListUtils; import org.apache.druid.segment.column.BitmapColumnIndex; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex; @@ -92,17 +93,21 @@ public class NestedFieldColumnIndexSupplier bitmaps, Supplier> localDictionarySupplier, Supplier globalStringDictionarySupplier, Supplier> globalLongDictionarySupplier, Supplier> globalDoubleDictionarySupplier, @Nullable Supplier> arrayElementDictionarySupplier, - @Nullable GenericIndexed arrayElementBitmaps + @Nullable GenericIndexed arrayElementBitmaps, + int numRows ) { this.singleType = types.getSingleType(); @@ -116,6 +121,8 @@ public NestedFieldColumnIndexSupplier( this.arrayElementBitmaps = arrayElementBitmaps; this.adjustLongId = globalStringDictionarySupplier.get().size(); this.adjustDoubleId = adjustLongId + globalLongDictionarySupplier.get().size(); + this.columnConfig = columnConfig; + this.numRows = numRows; } @Nullable @@ -255,6 +262,7 @@ private IntIntPair getLocalRangeFromDictionary( } + @Nullable private BitmapColumnIndex makeRangeIndex( @Nullable T startValue, boolean startStrict, @@ -276,6 +284,10 @@ private BitmapColumnIndex makeRangeIndex( ); final int startIndex = localRange.leftInt(); final int endIndex = localRange.rightInt(); + final int size = endIndex - startIndex; + if (ColumnIndexSupplier.skipComputingRangeIndexes(columnConfig, numRows, size)) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -328,6 +340,12 @@ public String getValue(int index) } } + @Override + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } + @Override public ImmutableBitmap getBitmap(int idx) { @@ -420,6 +438,7 @@ private void findNext() private class NestedStringLexicographicalRangeIndex implements LexicographicalRangeIndex { @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -439,6 +458,7 @@ public BitmapColumnIndex forRange( } @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -447,24 +467,26 @@ public BitmapColumnIndex forRange( Predicate matcher ) { + final FixedIndexed localDictionary = localDictionarySupplier.get(); + final Indexed stringDictionary = globalStringDictionarySupplier.get(); + final IntIntPair range = getLocalRangeFromDictionary( + StringUtils.toUtf8ByteBuffer(startValue), + startStrict, + StringUtils.toUtf8ByteBuffer(endValue), + endStrict, + localDictionary, + stringDictionary, + 0 + ); + final int start = range.leftInt(), end = range.rightInt(); + if (ColumnIndexSupplier.skipComputingRangeIndexes(columnConfig, numRows, end - start)) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override public Iterable getBitmapIterable() { - - final FixedIndexed localDictionary = localDictionarySupplier.get(); - final Indexed stringDictionary = globalStringDictionarySupplier.get(); - final IntIntPair range = getLocalRangeFromDictionary( - StringUtils.toUtf8ByteBuffer(startValue), - startStrict, - StringUtils.toUtf8ByteBuffer(endValue), - endStrict, - localDictionary, - stringDictionary, - 0 - ); - final int start = range.leftInt(), end = range.rightInt(); return () -> new Iterator() { int currIndex = start; @@ -514,18 +536,20 @@ public ImmutableBitmap next() private class NestedStringPredicateIndex implements DruidPredicateIndex { @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + final FixedIndexed localDictionary = localDictionarySupplier.get(); + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, localDictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override public Iterable getBitmapIterable() { - return () -> new Iterator() { - - final FixedIndexed localDictionary = localDictionarySupplier.get(); final Indexed stringDictionary = globalStringDictionarySupplier.get(); final Predicate stringPredicate = matcherFactory.makeStringPredicate(); @@ -697,6 +721,7 @@ private void findNext() private class NestedLongNumericRangeIndex implements NumericRangeIndex { @Override + @Nullable public BitmapColumnIndex forRange( @Nullable Number startValue, boolean startStrict, @@ -719,8 +744,13 @@ public BitmapColumnIndex forRange( private class NestedLongPredicateIndex implements DruidPredicateIndex { @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + final FixedIndexed localDictionary = localDictionarySupplier.get(); + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, localDictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -728,7 +758,6 @@ public Iterable getBitmapIterable() { return () -> new Iterator() { - final FixedIndexed localDictionary = localDictionarySupplier.get(); final FixedIndexed longDictionary = globalLongDictionarySupplier.get(); final DruidLongPredicate longPredicate = matcherFactory.makeLongPredicate(); @@ -904,6 +933,7 @@ private void findNext() private class NestedDoubleNumericRangeIndex implements NumericRangeIndex { @Override + @Nullable public BitmapColumnIndex forRange( @Nullable Number startValue, boolean startStrict, @@ -926,8 +956,13 @@ public BitmapColumnIndex forRange( private class NestedDoublePredicateIndex implements DruidPredicateIndex { @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + final FixedIndexed localDictionary = localDictionarySupplier.get(); + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, localDictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -1121,8 +1156,12 @@ private void findNext() private class NestedVariantPredicateIndex extends NestedVariantLiteralIndex implements DruidPredicateIndex { @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, localDictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java index e7862bb645f0..b9cc9c39a555 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnAndIndexSupplier.java @@ -38,6 +38,7 @@ import org.apache.druid.segment.IntListUtils; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex; @@ -71,7 +72,8 @@ public static ScalarDoubleColumnAndIndexSupplier read( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, ByteBuffer bb, - ColumnBuilder columnBuilder + ColumnBuilder columnBuilder, + ColumnConfig columnConfig ) { final byte version = bb.get(); @@ -115,11 +117,17 @@ public static ScalarDoubleColumnAndIndexSupplier read( bitmapSerdeFactory.getObjectStrategy(), columnBuilder.getFileMapper() ); + final int size; + try (ColumnarDoubles throwAway = doubles.get()) { + size = throwAway.size(); + } return new ScalarDoubleColumnAndIndexSupplier( doubleDictionarySupplier, doubles, rBitmaps, - bitmapSerdeFactory.getBitmapFactory() + bitmapSerdeFactory.getBitmapFactory(), + columnConfig, + size ); } catch (IOException ex) { @@ -130,6 +138,8 @@ public static ScalarDoubleColumnAndIndexSupplier read( } } + + private final Supplier> doubleDictionarySupplier; private final Supplier valueColumnSupplier; @@ -138,12 +148,16 @@ public static ScalarDoubleColumnAndIndexSupplier read( private final BitmapFactory bitmapFactory; private final ImmutableBitmap nullValueBitmap; + private final ColumnConfig columnConfig; + private final int numRows; private ScalarDoubleColumnAndIndexSupplier( Supplier> longDictionary, Supplier valueColumnSupplier, GenericIndexed valueIndexes, - BitmapFactory bitmapFactory + BitmapFactory bitmapFactory, + ColumnConfig columnConfig, + int numRows ) { this.doubleDictionarySupplier = longDictionary; @@ -151,6 +165,8 @@ private ScalarDoubleColumnAndIndexSupplier( this.valueIndexes = valueIndexes; this.bitmapFactory = bitmapFactory; this.nullValueBitmap = valueIndexes.get(0) == null ? bitmapFactory.makeEmptyImmutableBitmap() : valueIndexes.get(0); + this.columnConfig = columnConfig; + this.numRows = numRows; } @Override @@ -314,6 +330,7 @@ private void findNext() private class DoubleNumericRangeIndex implements NumericRangeIndex { + @Nullable @Override public BitmapColumnIndex forRange( @Nullable Number startValue, @@ -332,6 +349,9 @@ public BitmapColumnIndex forRange( final int startIndex = range.leftInt(); final int endIndex = range.rightInt(); + if (ColumnIndexSupplier.skipComputingRangeIndexes(columnConfig, numRows, endIndex - startIndex)) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -360,9 +380,14 @@ public ImmutableBitmap next() private class DoublePredicateIndex implements DruidPredicateIndex { + @Nullable @Override public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + final FixedIndexed dictionary = doubleDictionarySupplier.get(); + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, dictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -443,5 +468,11 @@ public String getValue(int index) final Double value = dictionary.get(index); return value == null ? null : String.valueOf(value); } + + @Override + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } } } diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java index 1cd7e16c58a9..f467613c1d55 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarDoubleColumnSerializer.java @@ -30,7 +30,6 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.data.ColumnarDoublesSerializer; import org.apache.druid.segment.data.CompressedVSizeColumnarIntsSerializer; @@ -71,7 +70,6 @@ public ScalarDoubleColumnSerializer( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java index 583ef677cbff..10d07bcf308b 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnAndIndexSupplier.java @@ -37,6 +37,7 @@ import org.apache.druid.segment.IntListUtils; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex; @@ -70,7 +71,8 @@ public static ScalarLongColumnAndIndexSupplier read( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, ByteBuffer bb, - ColumnBuilder columnBuilder + ColumnBuilder columnBuilder, + ColumnConfig columnConfig ) { final byte version = bb.get(); @@ -114,11 +116,17 @@ public static ScalarLongColumnAndIndexSupplier read( longsValueColumn, byteOrder ); + final int size; + try (ColumnarLongs throwAway = longs.get()) { + size = throwAway.size(); + } return new ScalarLongColumnAndIndexSupplier( longDictionarySupplier, longs, rBitmaps, - bitmapSerdeFactory.getBitmapFactory() + bitmapSerdeFactory.getBitmapFactory(), + columnConfig, + size ); } catch (IOException ex) { @@ -129,6 +137,7 @@ public static ScalarLongColumnAndIndexSupplier read( } } + private final Supplier> longDictionarySupplier; private final Supplier valueColumnSupplier; @@ -138,12 +147,16 @@ public static ScalarLongColumnAndIndexSupplier read( private final BitmapFactory bitmapFactory; private final ImmutableBitmap nullValueBitmap; + private final ColumnConfig columnConfig; + private final int numRows; private ScalarLongColumnAndIndexSupplier( Supplier> longDictionarySupplier, Supplier valueColumnSupplier, GenericIndexed valueIndexes, - BitmapFactory bitmapFactory + BitmapFactory bitmapFactory, + ColumnConfig columnConfig, + int numRows ) { this.longDictionarySupplier = longDictionarySupplier; @@ -151,6 +164,8 @@ private ScalarLongColumnAndIndexSupplier( this.valueIndexes = valueIndexes; this.bitmapFactory = bitmapFactory; this.nullValueBitmap = valueIndexes.get(0) == null ? bitmapFactory.makeEmptyImmutableBitmap() : valueIndexes.get(0); + this.columnConfig = columnConfig; + this.numRows = numRows; } @Override @@ -314,6 +329,7 @@ private void findNext() private class LongNumericRangeIndex implements NumericRangeIndex { + @Nullable @Override public BitmapColumnIndex forRange( @Nullable Number startValue, @@ -332,6 +348,9 @@ public BitmapColumnIndex forRange( final int startIndex = range.leftInt(); final int endIndex = range.rightInt(); + if (ColumnIndexSupplier.skipComputingRangeIndexes(columnConfig, numRows, endIndex - startIndex)) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -360,9 +379,14 @@ public ImmutableBitmap next() private class LongPredicateIndex implements DruidPredicateIndex { + @Nullable @Override public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { + FixedIndexed dictionary = longDictionarySupplier.get(); + if (ColumnIndexSupplier.skipComputingPredicateIndexes(columnConfig, numRows, dictionary.size())) { + return null; + } return new SimpleImmutableBitmapIterableIndex() { @Override @@ -370,7 +394,7 @@ public Iterable getBitmapIterable() { return () -> new Iterator() { - final Iterator iterator = longDictionarySupplier.get().iterator(); + final Iterator iterator = dictionary.iterator(); final DruidLongPredicate longPredicate = matcherFactory.makeLongPredicate(); int next; @@ -444,5 +468,11 @@ public String getValue(int index) final Long value = dictionary.get(index); return value == null ? null : String.valueOf(value); } + + @Override + public BitmapFactory getBitmapFactory() + { + return bitmapFactory; + } } } diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java index 6148d183b83a..53e2b393c399 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarLongColumnSerializer.java @@ -30,7 +30,6 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.data.ColumnarLongsSerializer; import org.apache.druid.segment.data.CompressedVSizeColumnarIntsSerializer; @@ -71,7 +70,6 @@ public ScalarLongColumnSerializer( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java index a1842adf3447..8c1aa67c9987 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnAndIndexSupplier.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex; import org.apache.druid.segment.column.DictionaryEncodedValueIndex; @@ -51,6 +52,7 @@ import org.apache.druid.segment.data.GenericIndexed; import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.data.VByte; +import org.apache.druid.segment.serde.NestedCommonFormatColumnPartSerde; import javax.annotation.Nullable; import java.io.IOException; @@ -63,7 +65,8 @@ public static ScalarStringColumnAndIndexSupplier read( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, ByteBuffer bb, - ColumnBuilder columnBuilder + ColumnBuilder columnBuilder, + ColumnConfig columnConfig ) { final byte version = bb.get(); @@ -77,7 +80,7 @@ public static ScalarStringColumnAndIndexSupplier read( final GenericIndexed stringDictionary; final Supplier frontCodedStringDictionarySupplier; - final ByteBuffer stringDictionaryBuffer = loadInternalFile( + final ByteBuffer stringDictionaryBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile( mapper, columnName, NestedCommonFormatColumnSerializer.STRING_DICTIONARY_FILE_NAME @@ -111,7 +114,7 @@ public static ScalarStringColumnAndIndexSupplier read( stringDictionary = GenericIndexed.read(stringDictionaryBuffer, GenericIndexed.UTF8_STRATEGY, mapper); frontCodedStringDictionarySupplier = null; } - final ByteBuffer encodedValueColumn = loadInternalFile( + final ByteBuffer encodedValueColumn = NestedCommonFormatColumnPartSerde.loadInternalFile( mapper, columnName, NestedCommonFormatColumnSerializer.ENCODED_VALUE_COLUMN_FILE_NAME @@ -120,7 +123,7 @@ public static ScalarStringColumnAndIndexSupplier read( encodedValueColumn, byteOrder ); - final ByteBuffer valueIndexBuffer = loadInternalFile( + final ByteBuffer valueIndexBuffer = NestedCommonFormatColumnPartSerde.loadInternalFile( mapper, columnName, NestedCommonFormatColumnSerializer.BITMAP_INDEX_FILE_NAME @@ -130,12 +133,18 @@ public static ScalarStringColumnAndIndexSupplier read( bitmapSerdeFactory.getObjectStrategy(), columnBuilder.getFileMapper() ); + final int size; + try (ColumnarInts throwAway = ints.get()) { + size = throwAway.size(); + } return new ScalarStringColumnAndIndexSupplier( stringDictionary, frontCodedStringDictionarySupplier, ints, valueIndexes, - bitmapSerdeFactory + bitmapSerdeFactory, + columnConfig, + size ); } catch (IOException ex) { @@ -147,19 +156,24 @@ public static ScalarStringColumnAndIndexSupplier read( } + private final GenericIndexed stringDictionary; private final Supplier frontCodedStringDictionarySupplier; private final Supplier encodedColumnSupplier; private final GenericIndexed valueIndexes; private final ImmutableBitmap nullValueBitmap; private final BitmapFactory bitmapFactory; + private final ColumnConfig columnConfig; + private final int numRows; private ScalarStringColumnAndIndexSupplier( GenericIndexed stringDictionary, Supplier frontCodedStringDictionarySupplier, Supplier encodedColumnSupplier, GenericIndexed valueIndexes, - BitmapSerdeFactory serdeFactory + BitmapSerdeFactory serdeFactory, + ColumnConfig columnConfig, + int numRows ) { this.stringDictionary = stringDictionary; @@ -168,6 +182,8 @@ private ScalarStringColumnAndIndexSupplier( this.valueIndexes = valueIndexes; this.bitmapFactory = serdeFactory.getBitmapFactory(); this.nullValueBitmap = valueIndexes.get(0) == null ? bitmapFactory.makeEmptyImmutableBitmap() : valueIndexes.get(0); + this.columnConfig = columnConfig; + this.numRows = numRows; } @Override @@ -183,17 +199,6 @@ public NestedCommonFormatColumn get() return new ScalarStringDictionaryEncodedColumn<>(encodedColumnSupplier.get(), stringDictionary.singleThreaded()); } - private static ByteBuffer loadInternalFile( - SmooshedFileMapper fileMapper, - String filenameBase, - String internalFileName - ) throws IOException - { - return fileMapper.mapFile( - NestedCommonFormatColumnSerializer.getInternalFileName(filenameBase, internalFileName) - ); - } - @Nullable @Override public T as(Class clazz) @@ -216,14 +221,18 @@ public T as(Class clazz) return (T) new IndexedStringDruidPredicateIndex<>( bitmapFactory, new StringEncodingStrategies.Utf8ToStringIndexed(utf8Dictionary), - singleThreadedBitmaps + singleThreadedBitmaps, + columnConfig, + numRows ); } else if (clazz.equals(LexicographicalRangeIndex.class)) { return (T) new IndexedUtf8LexicographicalRangeIndex<>( bitmapFactory, utf8Dictionary, singleThreadedBitmaps, - utf8Dictionary.get(0) == null + utf8Dictionary.get(0) == null, + columnConfig, + numRows ); } else if (clazz.equals(DictionaryEncodedStringValueIndex.class) || clazz.equals(DictionaryEncodedValueIndex.class)) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java index 788880934cb7..2e40eba91cbe 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/ScalarStringColumnSerializer.java @@ -32,7 +32,6 @@ import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.StringEncodingStrategies; import org.apache.druid.segment.data.CompressedVSizeColumnarIntsSerializer; import org.apache.druid.segment.data.CompressionStrategy; @@ -69,7 +68,6 @@ public ScalarStringColumnSerializer( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnAndIndexSupplier.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnAndIndexSupplier.java index 3a2dcb2eb278..28b808b09f8a 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnAndIndexSupplier.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnAndIndexSupplier.java @@ -28,6 +28,7 @@ import org.apache.druid.java.util.common.io.smoosh.SmooshedFileMapper; import org.apache.druid.segment.column.BitmapColumnIndex; import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.NullValueIndex; @@ -56,7 +57,8 @@ public static VariantArrayColumnAndIndexSupplier read( ByteOrder byteOrder, BitmapSerdeFactory bitmapSerdeFactory, ByteBuffer bb, - ColumnBuilder columnBuilder + ColumnBuilder columnBuilder, + ColumnConfig columnConfig ) { final byte version = bb.get(); @@ -157,6 +159,10 @@ public static VariantArrayColumnAndIndexSupplier read( arrayDictionarybuffer, byteOrder ); + final int size; + try (ColumnarInts throwAway = ints.get()) { + size = throwAway.size(); + } return new VariantArrayColumnAndIndexSupplier( logicalType, stringDictionary, @@ -166,7 +172,9 @@ public static VariantArrayColumnAndIndexSupplier read( arrayDictionarySupplier, ints, valueIndexes, - bitmapSerdeFactory.getBitmapFactory() + bitmapSerdeFactory.getBitmapFactory(), + columnConfig, + size ); } catch (IOException ex) { @@ -199,7 +207,9 @@ public VariantArrayColumnAndIndexSupplier( Supplier arrayDictionarySupplier, Supplier encodedValueColumnSupplier, GenericIndexed valueIndexes, - @SuppressWarnings("unused") BitmapFactory bitmapFactory + @SuppressWarnings("unused") BitmapFactory bitmapFactory, + @SuppressWarnings("unused") ColumnConfig columnConfig, + @SuppressWarnings("unused") int numRows ) { this.logicalType = logicalType; diff --git a/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnSerializer.java b/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnSerializer.java index 541c3552206c..702af36453ef 100644 --- a/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnSerializer.java +++ b/processing/src/main/java/org/apache/druid/segment/nested/VariantArrayColumnSerializer.java @@ -33,7 +33,6 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; -import org.apache.druid.segment.ProgressIndicator; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.StringEncodingStrategies; import org.apache.druid.segment.data.CompressedVSizeColumnarIntsSerializer; @@ -83,7 +82,6 @@ public VariantArrayColumnSerializer( String name, IndexSpec indexSpec, SegmentWriteOutMedium segmentWriteOutMedium, - @SuppressWarnings("unused") ProgressIndicator progressIndicator, Closer closer ) { diff --git a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java index 7c1fea26abe6..4585364fae7a 100644 --- a/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java +++ b/processing/src/main/java/org/apache/druid/segment/serde/NestedCommonFormatColumnPartSerde.java @@ -110,7 +110,8 @@ public Deserializer getDeserializer() byteOrder, bitmapSerdeFactory, buffer, - builder + builder, + columnConfig ); ColumnCapabilitiesImpl capabilitiesBuilder = builder.getCapabilitiesBuilder(); capabilitiesBuilder.setDictionaryEncoded(true); @@ -128,7 +129,8 @@ public Deserializer getDeserializer() byteOrder, bitmapSerdeFactory, buffer, - builder + builder, + columnConfig ); ColumnCapabilitiesImpl capabilitiesBuilder = builder.getCapabilitiesBuilder(); capabilitiesBuilder.setDictionaryEncoded(true); @@ -146,7 +148,8 @@ public Deserializer getDeserializer() byteOrder, bitmapSerdeFactory, buffer, - builder + builder, + columnConfig ); ColumnCapabilitiesImpl capabilitiesBuilder = builder.getCapabilitiesBuilder(); capabilitiesBuilder.setDictionaryEncoded(true); @@ -165,7 +168,8 @@ public Deserializer getDeserializer() byteOrder, bitmapSerdeFactory, buffer, - builder + builder, + columnConfig ); ColumnCapabilitiesImpl capabilitiesBuilder = builder.getCapabilitiesBuilder(); capabilitiesBuilder.setDictionaryEncoded(true); diff --git a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java index 3cdaf24508ff..db72ec56d7f6 100644 --- a/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java +++ b/processing/src/main/java/org/apache/druid/segment/virtual/ListFilteredVirtualColumn.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.base.Predicate; import com.google.common.base.Predicates; +import org.apache.druid.collections.bitmap.BitmapFactory; import org.apache.druid.collections.bitmap.ImmutableBitmap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.query.BitmapResultFactory; @@ -503,6 +504,7 @@ private ListFilteredDruidPredicateIndex(DictionaryEncodedStringValueIndex delega } @Override + @Nullable public BitmapColumnIndex forPredicate(DruidPredicateFactory matcherFactory) { return new SimpleBitmapColumnIndex() @@ -542,6 +544,7 @@ private ListFilteredLexicographicalRangeIndex( } @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -553,6 +556,7 @@ public BitmapColumnIndex forRange( } @Override + @Nullable public BitmapColumnIndex forRange( @Nullable String startValue, boolean startStrict, @@ -621,6 +625,12 @@ public String getValue(int index) return delegate.getValue(idMapping.getReverseId(index)); } + @Override + public BitmapFactory getBitmapFactory() + { + return delegate.getBitmapFactory(); + } + @Override public ImmutableBitmap getBitmap(int idx) { diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java index 31a1afdf6dde..d8c385232aae 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierTest.java @@ -40,7 +40,6 @@ import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; import org.apache.druid.segment.IndexSpec; @@ -49,6 +48,7 @@ import org.apache.druid.segment.SimpleAscendingOffset; import org.apache.druid.segment.TestHelper; import org.apache.druid.segment.column.ColumnBuilder; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DruidPredicateIndex; @@ -94,6 +94,27 @@ public class NestedDataColumnSupplierTest extends InitializedNullHandlingTest private static final String NO_MATCH = "no"; + private static final ColumnConfig ALWAYS_USE_INDEXES = new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + + @Override + public double skipValueRangeIndexScale() + { + return 1.0; + } + + @Override + public double skipValuePredicateIndexScale() + { + return 1.0; + } + }; + @Rule public final TemporaryFolder tempFolder = new TemporaryFolder(); @@ -172,7 +193,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -230,7 +250,7 @@ public void testBasicFunctionality() throws IOException false, baseBuffer, bob, - () -> 0, + ALWAYS_USE_INDEXES, bitmapSerdeFactory, ByteOrder.nativeOrder() ); @@ -267,7 +287,7 @@ public void testConcurrency() throws ExecutionException, InterruptedException false, baseBuffer, bob, - () -> 0, + ALWAYS_USE_INDEXES, bitmapSerdeFactory, ByteOrder.nativeOrder() ); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java index d335a041b13f..4ac538f6612d 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedDataColumnSupplierV4Test.java @@ -37,7 +37,6 @@ import org.apache.druid.query.DefaultBitmapResultFactory; import org.apache.druid.query.filter.SelectorPredicateFactory; import org.apache.druid.query.monomorphicprocessing.RuntimeShapeInspector; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; import org.apache.druid.segment.IndexSpec; @@ -167,7 +166,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -219,7 +217,7 @@ public void testBasicFunctionality() throws IOException NestedDataColumnSupplierV4 supplier = NestedDataColumnSupplierV4.read( baseBuffer, bob, - () -> 0, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES, NestedDataComplexTypeSerde.OBJECT_MAPPER, new OnlyPositionalReadsTypeStrategy<>(ColumnType.LONG.getStrategy()), new OnlyPositionalReadsTypeStrategy<>(ColumnType.DOUBLE.getStrategy()) @@ -238,7 +236,7 @@ public void testConcurrency() throws ExecutionException, InterruptedException NestedDataColumnSupplierV4 supplier = NestedDataColumnSupplierV4.read( baseBuffer, bob, - () -> 0, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES, NestedDataComplexTypeSerde.OBJECT_MAPPER ); final String expectedReason = "none"; diff --git a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplierTest.java index 27dfd79bbcd5..fbe51d298dc6 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/NestedFieldColumnIndexSupplierTest.java @@ -29,6 +29,7 @@ import org.apache.druid.query.filter.DruidPredicateFactory; import org.apache.druid.query.filter.InDimFilter; import org.apache.druid.segment.column.BitmapColumnIndex; +import org.apache.druid.segment.column.ColumnConfig; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.DictionaryEncodedStringValueIndex; import org.apache.druid.segment.column.DictionaryEncodedValueIndex; @@ -62,6 +63,27 @@ public class NestedFieldColumnIndexSupplierTest extends InitializedNullHandlingTest { + private static final int ROW_COUNT = 10; + static final ColumnConfig ALWAYS_USE_INDEXES = new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + + @Override + public double skipValueRangeIndexScale() + { + return 1.0; + } + + @Override + public double skipValuePredicateIndexScale() + { + return 1.0; + } + }; BitmapSerdeFactory roaringFactory = RoaringBitmapSerdeFactory.getInstance(); BitmapResultFactory bitmapResultFactory = new DefaultBitmapResultFactory( roaringFactory.getBitmapFactory() @@ -70,6 +92,7 @@ public class NestedFieldColumnIndexSupplierTest extends InitializedNullHandlingT Supplier> globalLongs; Supplier> globalDoubles; + @Before public void setup() throws IOException { @@ -153,7 +176,7 @@ public void testSingleTypeStringColumnValueIndex() throws IOException BitmapColumnIndex columnIndex = nullIndex.forNull(); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); Assert.assertEquals(0, bitmap.size()); } @@ -172,21 +195,21 @@ public void testSingleTypeStringColumnValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 7, 8); // non-existent in local column columnIndex = valueSetIndex.forValue("fo"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "fooo", "z"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.8, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 2, 3, 4, 5, 6, 7, 8); } @@ -206,144 +229,144 @@ public void testSingleTypeStringColumnRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange(null, false, "a", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, true, "a", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, false, "b", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, false, "b", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 7, 8); forRange = rangeIndex.forRange("a", false, "b", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange("a", true, "b", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 7, 8); forRange = rangeIndex.forRange("b", false, "fon", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 7, 8); forRange = rangeIndex.forRange("bb", false, "fon", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange("b", true, "foo", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 9); forRange = rangeIndex.forRange("f", true, "g", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 5, 9); forRange = rangeIndex.forRange(null, false, "g", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); forRange = rangeIndex.forRange("f", false, null, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); forRange = rangeIndex.forRange("b", true, "fooo", true); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 9); forRange = rangeIndex.forRange("b", true, "fooo", false); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 5, 9); forRange = rangeIndex.forRange(null, true, "fooo", true); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 3, 7, 8, 9); forRange = rangeIndex.forRange("b", true, null, false); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); forRange = rangeIndex.forRange("b", false, null, true); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(null, true, "fooo", false); - Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(null, true, "foa", false); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 7, 8); forRange = rangeIndex.forRange(null, true, "foooa", false); - Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 5, 7, 8, 9); forRange = rangeIndex.forRange("foooa", true, "ggg", false); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange("g", true, "gg", false); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange("z", true, "zz", false); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange("z", false, "zz", false); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 4, 6); } @@ -367,7 +390,7 @@ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOExcepti true, s -> !"fooo".equals(s) ); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 9); @@ -378,7 +401,7 @@ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOExcepti true, s -> "fooo".equals(s) ); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 5); @@ -389,7 +412,7 @@ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOExcepti false, s -> !"fooo".equals(s) ); - Assert.assertEquals(0.8, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 8, 9); @@ -400,7 +423,7 @@ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOExcepti true, s -> !"fooo".equals(s) ); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 3, 7, 8, 9); @@ -411,7 +434,7 @@ public void testSingleTypeStringColumnRangeIndexWithPredicate() throws IOExcepti true, s -> true ); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); } @@ -434,7 +457,7 @@ public void testSingleTypeStringColumnPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 6, 7, 8); } @@ -453,7 +476,7 @@ public void testSingleTypeStringColumnWithNullValueIndex() throws IOException BitmapColumnIndex columnIndex = nullIndex.forNull(); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 7, 8); } @@ -472,21 +495,21 @@ public void testSingleTypeStringColumnWithNullValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 3); // non-existent in local column columnIndex = valueSetIndex.forValue("fo"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.0, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "fooo", "z"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 3, 4, 5, 6); } @@ -505,78 +528,78 @@ public void testSingleValueStringWithNullRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange("f", true, "g", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 5, 9); forRange = rangeIndex.forRange(null, false, "g", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 3, 5, 9); forRange = rangeIndex.forRange(null, false, "a", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, false, "b", true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, false, "b", false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 3); forRange = rangeIndex.forRange("f", false, null, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); forRange = rangeIndex.forRange("b", true, "fooo", true); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 9); forRange = rangeIndex.forRange("b", true, "fooo", false); - Assert.assertEquals(0.4, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 5, 9); forRange = rangeIndex.forRange(null, true, "fooo", true); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 3, 9); forRange = rangeIndex.forRange("b", true, null, false); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 6, 9); forRange = rangeIndex.forRange("b", false, null, true); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); forRange = rangeIndex.forRange(null, true, "fooo", false); - Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 3, 5, 9); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 3, 4, 5, 6, 9); } @@ -599,7 +622,7 @@ public void testSingleValueStringWithNullPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 3, 4, 6); } @@ -621,14 +644,14 @@ public void testSingleTypeLongColumnValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("1"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 9); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1", "300", "700"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 2, 3, 7, 8, 9); } @@ -647,57 +670,57 @@ public void testSingleTypeLongColumnRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange(10L, true, 400L, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 6, 7, 8); forRange = rangeIndex.forRange(1, true, 3, true); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(1, false, 3, true); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 9); forRange = rangeIndex.forRange(1, false, 3, false); - Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 5, 9); forRange = rangeIndex.forRange(100L, true, 300L, true); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(100L, true, 300L, false); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 7, 8); forRange = rangeIndex.forRange(100L, false, 300L, true); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 6); forRange = rangeIndex.forRange(100L, false, 300L, false); - Assert.assertEquals(0.5, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 6, 7, 8); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); } @@ -720,7 +743,7 @@ public void testSingleTypeLongColumnPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 5, 9); } @@ -739,7 +762,7 @@ public void testSingleTypeLongColumnWithNullValueIndex() throws IOException BitmapColumnIndex columnIndex = nullIndex.forNull(); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 5, 8); } @@ -758,14 +781,14 @@ public void testSingleTypeLongColumnWithNullValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("3"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 4); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1", "3", "300"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 7, 9); @@ -777,14 +800,14 @@ public void testSingleTypeLongColumnWithNullValueSetIndex() throws IOException treeSet.add("300"); columnIndex = valueSetIndex.forSortedValues(treeSet); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.8, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.8, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 2, 3, 4, 5, 7, 8, 9); // null value should really use NullValueIndex, but this works for classic reasons columnIndex = valueSetIndex.forValue(null); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 5, 8); } @@ -803,53 +826,53 @@ public void testSingleValueLongWithNullRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange(100, false, 700, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 6, 7); forRange = rangeIndex.forRange(100, true, 300, true); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(100, false, 300, true); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 6); forRange = rangeIndex.forRange(100, true, 300, false); - Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 7); forRange = rangeIndex.forRange(100, false, 300, false); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 6, 7); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 3, 4, 6, 7, 9); forRange = rangeIndex.forRange(null, false, 0, false); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, false, 1, false); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 9); forRange = rangeIndex.forRange(null, false, 1, true); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); } @@ -872,7 +895,7 @@ public void testSingleValueLongWithNullPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 4, 6); } @@ -894,14 +917,14 @@ public void testSingleTypeDoubleColumnValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("1.2"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 4, 7); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1.2", "3.3", "6.6"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.7, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 3, 4, 5, 6, 7, 9); } @@ -920,80 +943,80 @@ public void testSingleTypeDoubleColumnRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange(1.0, true, 5.0, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.9, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9); forRange = rangeIndex.forRange(1.1, false, 3.3, false); Assert.assertNotNull(forRange); - Assert.assertEquals(0.9, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.9, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 6, 7, 8, 9); forRange = rangeIndex.forRange(1.1, true, 3.3, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.3, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 4, 7); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(1.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(1.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9); forRange = rangeIndex.forRange(1.111, true, 1.19, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(1.01, true, 1.09, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(0.05, true, 0.98, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(0.05, true, 1.1, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(8.99, true, 10.10, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(8.99, true, 10.10, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(10.00, true, 10.10, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); @@ -1017,7 +1040,7 @@ public void testSingleTypeDoubleColumnPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 3, 4, 6, 7, 9); } @@ -1036,7 +1059,7 @@ public void testSingleTypeDoubleColumnWithNullValueIndex() throws IOException BitmapColumnIndex columnIndex = nullIndex.forNull(); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 6); } @@ -1055,14 +1078,14 @@ public void testSingleTypeDoubleColumnWithNullValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("6.6"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 5); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("1.2", "3.3", "7.7"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 4, 7, 9); @@ -1074,14 +1097,14 @@ public void testSingleTypeDoubleColumnWithNullValueSetIndex() throws IOException treeSet.add("7.7"); columnIndex = valueSetIndex.forSortedValues(treeSet); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.7, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 2, 3, 4, 6, 7, 9); // null value should really use NullValueIndex, but this works for classic reasons columnIndex = valueSetIndex.forValue(null); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.3, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.3, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 6); } @@ -1100,38 +1123,38 @@ public void testSingleValueDoubleWithNullRangeIndex() throws IOException BitmapColumnIndex forRange = rangeIndex.forRange(1.1, false, 5.0, true); Assert.assertNotNull(forRange); - Assert.assertEquals(0.6, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, forRange.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 7, 8, 9); forRange = rangeIndex.forRange(null, true, null, true); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9); forRange = rangeIndex.forRange(null, false, null, false); - Assert.assertEquals(0.7, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.7, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 2, 4, 5, 7, 8, 9); forRange = rangeIndex.forRange(null, true, 1.0, true); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); forRange = rangeIndex.forRange(null, true, 1.1, false); - Assert.assertEquals(0.2, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 8); forRange = rangeIndex.forRange(6.6, false, null, false); - Assert.assertEquals(0.1, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 5); forRange = rangeIndex.forRange(6.6, true, null, false); - Assert.assertEquals(0.0, forRange.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.0, forRange.estimateSelectivity(ROW_COUNT), 0.0); bitmap = forRange.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap); } @@ -1154,7 +1177,7 @@ public void testSingleValueDoubleWithNullPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 4, 7, 9); } @@ -1176,7 +1199,7 @@ public void testVariantNullValueIndex() throws IOException BitmapColumnIndex columnIndex = nullIndex.forNull(); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 7); } @@ -1195,26 +1218,26 @@ public void testVariantValueSetIndex() throws IOException BitmapColumnIndex columnIndex = valueSetIndex.forValue("b"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 9); columnIndex = valueSetIndex.forValue("1"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 0, 5); columnIndex = valueSetIndex.forValue("1.1"); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.1, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.1, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 8); // set index columnIndex = valueSetIndex.forSortedValues(new TreeSet<>(ImmutableSet.of("b", "300", "9.9", "1.6"))); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.4, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.4, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 9); @@ -1227,14 +1250,14 @@ public void testVariantValueSetIndex() throws IOException treeSet.add("1.6"); columnIndex = valueSetIndex.forSortedValues(treeSet); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.6, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.6, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 2, 3, 4, 7, 9); // null value should really use NullValueIndex, but this works for classic reasons columnIndex = valueSetIndex.forValue(null); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.2, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.2, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 2, 7); } @@ -1269,7 +1292,7 @@ public void testVariantPredicateIndex() throws IOException BitmapColumnIndex columnIndex = predicateIndex.forPredicate(predicateFactory); Assert.assertNotNull(columnIndex); - Assert.assertEquals(0.5, columnIndex.estimateSelectivity(10), 0.0); + Assert.assertEquals(0.5, columnIndex.estimateSelectivity(ROW_COUNT), 0.0); ImmutableBitmap bitmap = columnIndex.computeBitmapResult(bitmapResultFactory); checkBitmap(bitmap, 1, 3, 4, 6, 9); } @@ -1405,13 +1428,15 @@ public void testEnsureNoImproperSelectionFromAdjustedGlobals() throws IOExceptio .getByteValue() ), roaringFactory.getBitmapFactory(), + ALWAYS_USE_INDEXES, bitmaps, dictionarySupplier, stringIndexed, longIndexed, doubleIndexed, null, - null + null, + ROW_COUNT ); StringValueSetIndex valueSetIndex = indexSupplier.as(StringValueSetIndex.class); @@ -1440,7 +1465,116 @@ public void testEnsureNoImproperSelectionFromAdjustedGlobals() throws IOExceptio checkBitmap(bitmap); } + @Test + public void testSkipIndexThresholds() throws IOException + { + ColumnConfig twentyPercent = new ColumnConfig() + { + @Override + public int columnCacheSizeBytes() + { + return 0; + } + + @Override + public double skipValueRangeIndexScale() + { + return 0.2; + } + + @Override + public double skipValuePredicateIndexScale() + { + return 0.2; + } + }; + NestedFieldColumnIndexSupplier singleTypeStringSupplier = makeSingleTypeStringSupplier(twentyPercent); + NestedFieldColumnIndexSupplier singleTypeLongSupplier = makeSingleTypeLongSupplier(twentyPercent); + NestedFieldColumnIndexSupplier singleTypeDoubleSupplier = makeSingleTypeDoubleSupplier(twentyPercent); + NestedFieldColumnIndexSupplier variantSupplierWithNull = makeVariantSupplierWithNull(twentyPercent); + + // value cardinality of all of these dictionaries is bigger than the skip threshold, so predicate index short + // circuit early and return nothing + DruidPredicateFactory predicateFactory = new InDimFilter.InFilterDruidPredicateFactory( + null, + new InDimFilter.ValuesSet(ImmutableSet.of("0")) + ); + Assert.assertNull(singleTypeStringSupplier.as(DruidPredicateIndex.class).forPredicate(predicateFactory)); + Assert.assertNull(singleTypeLongSupplier.as(DruidPredicateIndex.class).forPredicate(predicateFactory)); + Assert.assertNull(singleTypeDoubleSupplier.as(DruidPredicateIndex.class).forPredicate(predicateFactory)); + Assert.assertNull(variantSupplierWithNull.as(DruidPredicateIndex.class).forPredicate(predicateFactory)); + + // range index computation is a bit more complicated and done inside of the index maker gizmo because we don't know + // the range up front + LexicographicalRangeIndex stringRange = singleTypeStringSupplier.as(LexicographicalRangeIndex.class); + NumericRangeIndex longRange = singleTypeLongSupplier.as(NumericRangeIndex.class); + NumericRangeIndex doubleRange = singleTypeDoubleSupplier.as(NumericRangeIndex.class); + + // string: [b, foo, fooo, z] + // small enough should be cool + Assert.assertNotNull(stringRange.forRange("fo", false, "fooo", false)); + Assert.assertNotNull(stringRange.forRange("fo", false, "fooo", false, (s) -> true)); + // range too big, no index + Assert.assertNull(stringRange.forRange("fo", false, "z", false)); + Assert.assertNull(stringRange.forRange("fo", false, "z", false, (s) -> true)); + + // long: [1, 3, 100, 300] + // small enough should be cool + Assert.assertNotNull(longRange.forRange(1, false, 100, true)); + // range too big, no index + Assert.assertNull(longRange.forRange(1, false, null, false)); + + // double: [1.1, 1.2, 3.3, 6.6] + // small enough should be cool + Assert.assertNotNull(doubleRange.forRange(null, false, 1.2, false)); + // range too big, no index + Assert.assertNull(doubleRange.forRange(null, false, 3.3, false)); + + // other index types should not be impacted + Assert.assertNotNull(singleTypeStringSupplier.as(DictionaryEncodedStringValueIndex.class)); + Assert.assertNotNull(singleTypeStringSupplier.as(DictionaryEncodedValueIndex.class)); + Assert.assertNotNull(singleTypeStringSupplier.as(StringValueSetIndex.class).forValue("foo")); + Assert.assertNotNull( + singleTypeStringSupplier.as(StringValueSetIndex.class) + .forSortedValues(new TreeSet<>(ImmutableSet.of("foo", "fooo", "z"))) + ); + Assert.assertNotNull(singleTypeStringSupplier.as(NullValueIndex.class)); + + Assert.assertNotNull(singleTypeLongSupplier.as(DictionaryEncodedStringValueIndex.class)); + Assert.assertNotNull(singleTypeLongSupplier.as(DictionaryEncodedValueIndex.class)); + Assert.assertNotNull(singleTypeLongSupplier.as(StringValueSetIndex.class).forValue("1")); + Assert.assertNotNull( + singleTypeLongSupplier.as(StringValueSetIndex.class) + .forSortedValues(new TreeSet<>(ImmutableSet.of("1", "3", "100"))) + ); + Assert.assertNotNull(singleTypeLongSupplier.as(NullValueIndex.class)); + + Assert.assertNotNull(singleTypeDoubleSupplier.as(DictionaryEncodedStringValueIndex.class)); + Assert.assertNotNull(singleTypeDoubleSupplier.as(DictionaryEncodedValueIndex.class)); + Assert.assertNotNull(singleTypeDoubleSupplier.as(StringValueSetIndex.class).forValue("1.1")); + Assert.assertNotNull( + singleTypeDoubleSupplier.as(StringValueSetIndex.class) + .forSortedValues(new TreeSet<>(ImmutableSet.of("1.1", "1.2", "3.3"))) + ); + Assert.assertNotNull(singleTypeDoubleSupplier.as(NullValueIndex.class)); + + // variant: [null, b, z, 1, 300, 1.1, 9.9] + Assert.assertNotNull(variantSupplierWithNull.as(DictionaryEncodedStringValueIndex.class)); + Assert.assertNotNull(variantSupplierWithNull.as(DictionaryEncodedValueIndex.class)); + Assert.assertNotNull(variantSupplierWithNull.as(StringValueSetIndex.class).forValue("b")); + Assert.assertNotNull( + variantSupplierWithNull.as(StringValueSetIndex.class) + .forSortedValues(new TreeSet<>(ImmutableSet.of("b", "1", "9.9"))) + ); + Assert.assertNotNull(variantSupplierWithNull.as(NullValueIndex.class)); + } + private NestedFieldColumnIndexSupplier makeSingleTypeStringSupplier() throws IOException + { + return makeSingleTypeStringSupplier(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeStringSupplier(ColumnConfig columnConfig) throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1503,17 +1637,25 @@ private NestedFieldColumnIndexSupplier makeSingleTypeStringSupplier() throws new FieldTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeSingleTypeStringWithNullsSupplier() throws IOException + { + return makeSingleTypeStringWithNullsSupplier(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeStringWithNullsSupplier(ColumnConfig columnConfig) + throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1579,17 +1721,24 @@ private NestedFieldColumnIndexSupplier makeSingleTypeStringWithNullsSupplier( new FieldTypeInfo.MutableTypeSet().add(ColumnType.STRING).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplier() throws IOException + { + return makeSingleTypeLongSupplier(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplier(ColumnConfig columnConfig) throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1652,17 +1801,25 @@ private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplier() throws IO new FieldTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplierWithNull() throws IOException + { + return makeSingleTypeLongSupplierWithNull(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplierWithNull(ColumnConfig columnConfig) + throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1729,17 +1886,24 @@ private NestedFieldColumnIndexSupplier makeSingleTypeLongSupplierWithNull() t new FieldTypeInfo.MutableTypeSet().add(ColumnType.LONG).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplier() throws IOException + { + return makeSingleTypeDoubleSupplier(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplier(ColumnConfig columnConfig) throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1802,17 +1966,25 @@ private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplier() throws new FieldTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplierWithNull() throws IOException + { + return makeSingleTypeDoubleSupplierWithNull(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplierWithNull(ColumnConfig columnConfig) + throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1879,17 +2051,24 @@ private NestedFieldColumnIndexSupplier makeSingleTypeDoubleSupplierWithNull() new FieldTypeInfo.MutableTypeSet().add(ColumnType.DOUBLE).getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } private NestedFieldColumnIndexSupplier makeVariantSupplierWithNull() throws IOException + { + return makeVariantSupplierWithNull(ALWAYS_USE_INDEXES); + } + + private NestedFieldColumnIndexSupplier makeVariantSupplierWithNull(ColumnConfig columnConfig) throws IOException { ByteBuffer localDictionaryBuffer = ByteBuffer.allocate(1 << 12).order(ByteOrder.nativeOrder()); ByteBuffer bitmapsBuffer = ByteBuffer.allocate(1 << 12); @@ -1967,13 +2146,15 @@ private NestedFieldColumnIndexSupplier makeVariantSupplierWithNull() throws I .getByteValue() ), roaringFactory.getBitmapFactory(), + columnConfig, bitmaps, dictionarySupplier, globalStrings, globalLongs, globalDoubles, null, - null + null, + ROW_COUNT ); } diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java index bf2ec04890de..9d25216936f7 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarDoubleColumnSupplierTest.java @@ -35,7 +35,6 @@ import org.apache.druid.query.filter.SelectorPredicateFactory; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.IndexableAdapter; @@ -125,7 +124,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -182,7 +180,8 @@ public void testBasicFunctionality() ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); try (ScalarDoubleColumn column = (ScalarDoubleColumn) supplier.get()) { smokeTest(supplier, column); @@ -199,7 +198,8 @@ public void testConcurrency() throws ExecutionException, InterruptedException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java index eb834da77e4d..16c2cfbd0114 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarLongColumnSupplierTest.java @@ -35,7 +35,6 @@ import org.apache.druid.query.filter.SelectorPredicateFactory; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.IndexableAdapter; @@ -125,7 +124,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -182,7 +180,8 @@ public void testBasicFunctionality() ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); try (ScalarLongColumn column = (ScalarLongColumn) supplier.get()) { smokeTest(supplier, column); @@ -199,7 +198,8 @@ public void testConcurrency() throws ExecutionException, InterruptedException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java index d3a0e9093a16..926b2a8f7960 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/ScalarStringColumnSupplierTest.java @@ -36,7 +36,6 @@ import org.apache.druid.query.filter.SelectorPredicateFactory; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.DimensionSelector; import org.apache.druid.segment.IndexSpec; @@ -128,7 +127,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -185,7 +183,8 @@ public void testBasicFunctionality() throws IOException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); try (ScalarStringDictionaryEncodedColumn column = (ScalarStringDictionaryEncodedColumn) supplier.get()) { smokeTest(supplier, column); @@ -202,7 +201,8 @@ public void testConcurrency() throws ExecutionException, InterruptedException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); diff --git a/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java b/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java index f70b8c79ddea..77658495c47a 100644 --- a/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java +++ b/processing/src/test/java/org/apache/druid/segment/nested/VariantArrayColumnSupplierTest.java @@ -33,7 +33,6 @@ import org.apache.druid.query.DefaultBitmapResultFactory; import org.apache.druid.segment.AutoTypeColumnIndexer; import org.apache.druid.segment.AutoTypeColumnMerger; -import org.apache.druid.segment.BaseProgressIndicator; import org.apache.druid.segment.ColumnValueSelector; import org.apache.druid.segment.IndexSpec; import org.apache.druid.segment.IndexableAdapter; @@ -122,7 +121,6 @@ private SmooshedFileMapper smooshify( fileNameBase, new IndexSpec(), writeOutMediumFactory.makeSegmentWriteOutMedium(tempFolder.newFolder()), - new BaseProgressIndicator(), closer ); @@ -180,7 +178,8 @@ public void testBasicFunctionality() throws IOException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); try (VariantArrayColumn column = (VariantArrayColumn) supplier.get()) { smokeTest(supplier, column); @@ -198,7 +197,8 @@ public void testConcurrency() throws ExecutionException, InterruptedException ByteOrder.nativeOrder(), bitmapSerdeFactory, baseBuffer, - bob + bob, + NestedFieldColumnIndexSupplierTest.ALWAYS_USE_INDEXES ); final String expectedReason = "none"; final AtomicReference failureReason = new AtomicReference<>(expectedReason); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index 6cbdd6015085..99b15fdc3040 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -2661,6 +2661,7 @@ public void testGroupByPathBoundFilterLongNoLower() .build() ), ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"100", 2L} ), RowSignature.builder() @@ -2922,6 +2923,7 @@ public void testGroupByPathBoundFilterDoubleNoLower() .build() ), ImmutableList.of( + new Object[]{NullHandling.defaultStringValue(), 4L}, new Object[]{"2.02", 2L} ), RowSignature.builder()