diff --git a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java index f345d1367a031..ecb50dfc50ddf 100644 --- a/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java +++ b/server/src/test/java/org/elasticsearch/search/aggregations/bucket/terms/TermsAggregatorTests.java @@ -58,7 +58,12 @@ import org.elasticsearch.index.query.MatchAllQueryBuilder; import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.indices.breaker.NoneCircuitBreakerService; +import org.elasticsearch.script.MockScriptEngine; import org.elasticsearch.script.Script; +import org.elasticsearch.script.ScriptEngine; +import org.elasticsearch.script.ScriptModule; +import org.elasticsearch.script.ScriptService; +import org.elasticsearch.script.ScriptType; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregationBuilders; @@ -100,6 +105,7 @@ import java.util.function.BiFunction; import java.util.function.Function; +import static java.util.Collections.singleton; import static org.elasticsearch.index.mapper.SeqNoFieldMapper.PRIMARY_TERM_NAME; import static org.elasticsearch.search.aggregations.AggregationBuilders.terms; import static org.elasticsearch.search.aggregations.PipelineAggregatorBuilders.bucketScript; @@ -111,6 +117,26 @@ public class TermsAggregatorTests extends AggregatorTestCase { private boolean randomizeAggregatorImpl = true; + /** Script to return the {@code _value} provided by aggs framework. */ + private static final String VALUE_SCRIPT = "_value"; + private static final String SINGLE_SCRIPT = "single"; + + @Override + protected ScriptService getMockScriptService() { + Map, Object>> scripts = new HashMap<>(); + + scripts.put(VALUE_SCRIPT, vars -> (Double.valueOf((Double) vars.get("_value")) + 1)); + scripts.put(SINGLE_SCRIPT, vars -> 1); + + MockScriptEngine scriptEngine = new MockScriptEngine(MockScriptEngine.NAME, + scripts, + Collections.emptyMap()); + Map engines = Collections.singletonMap(scriptEngine.getType(), scriptEngine); + + return new ScriptService(Settings.EMPTY, engines, ScriptModule.CORE_CONTEXTS); + } + + protected A createAggregator(AggregationBuilder aggregationBuilder, IndexSearcher indexSearcher, MappedFieldType... fieldTypes) throws IOException { try { @@ -227,11 +253,8 @@ public void testSimple() throws Exception { fieldType.setName("string"); fieldType.setHasDocValues(true ); - TermsAggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); + assertEquals(5, result.getBuckets().size()); assertEquals("", result.getBuckets().get(0).getKeyAsString()); assertEquals(2L, result.getBuckets().get(0).getDocCount()); @@ -298,11 +321,8 @@ public void testStringIncludeExclude() throws Exception { .size(12) .order(BucketOrder.key(true)); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); + assertEquals(10, result.getBuckets().size()); assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -335,11 +355,7 @@ public void testStringIncludeExclude() throws Exception { .field("sv_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType, fieldType2); assertEquals(5, result.getBuckets().size()); assertEquals("val001", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -359,11 +375,7 @@ public void testStringIncludeExclude() throws Exception { .field("mv_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(8, result.getBuckets().size()); assertEquals("val002", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -388,11 +400,7 @@ public void testStringIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, "val00.+")) .field("mv_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2, result.getBuckets().size()); assertEquals("val010", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -405,11 +413,7 @@ public void testStringIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new String[]{"val000", "val010"}, null)) .field("mv_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2, result.getBuckets().size()); assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -423,11 +427,7 @@ public void testStringIncludeExclude() throws Exception { "val005", "val006", "val007", "val008", "val009", "val011"})) .field("mv_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2, result.getBuckets().size()); assertEquals("val000", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -479,11 +479,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new long[]{0, 5}, null)) .field("long_field") .order(BucketOrder.key(true)); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2, result.getBuckets().size()); assertEquals(0L, result.getBuckets().get(0).getKey()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -496,11 +492,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, new long[]{0, 5})) .field("long_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(4, result.getBuckets().size()); assertEquals(1L, result.getBuckets().get(0).getKey()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -520,11 +512,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(new double[]{0.0, 5.0}, null)) .field("double_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(2, result.getBuckets().size()); assertEquals(0.0, result.getBuckets().get(0).getKey()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -537,11 +525,7 @@ public void testNumericIncludeExclude() throws Exception { .includeExclude(new IncludeExclude(null, new double[]{0.0, 5.0})) .field("double_field") .order(BucketOrder.key(true)); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(4, result.getBuckets().size()); assertEquals(1.0, result.getBuckets().get(0).getKey()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -703,11 +687,7 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, fieldType.setName("field"); fieldType.setHasDocValues(true); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals(size, result.getBuckets().size()); for (int i = 0; i < size; i++) { Map.Entry expected = expectedBuckets.get(i); @@ -730,11 +710,8 @@ private void termsAggregator(ValueType valueType, MappedFieldType fieldType, .size(numTerms) .collectMode(randomFrom(Aggregator.SubAggCollectionMode.values())) .field("field")); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, filterFieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = ((Filter) aggregator.buildAggregation(0L)).getAggregations().get("_name2"); + result = ((Filter) searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, + fieldType, filterFieldType)).getAggregations().get("_name2"); int expectedFilteredCounts = 0; for (Integer count : filteredCounts.values()) { if (count > 0) { @@ -806,11 +783,7 @@ private void termsAggregatorWithNestedMaxAgg(ValueType valueType, MappedFiel MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); fieldType2.setName("value"); fieldType2.setHasDocValues(true); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType, fieldType2); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType, fieldType2); assertEquals(size, result.getBuckets().size()); for (int i = 0; i < size; i++) { Map.Entry expected = expectedBuckets.get(i); @@ -841,33 +814,18 @@ public void testEmpty() throws Exception { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(ValueType.STRING) .field("string"); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); - assertEquals("_name", result.getName()); - assertEquals(0, result.getBuckets().size()); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType1); + assertNull(result); aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.LONG) .field("long"); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType2); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); - assertEquals("_name", result.getName()); - assertEquals(0, result.getBuckets().size()); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType2); + assertNull(result); aggregationBuilder = new TermsAggregationBuilder("_name").userValueTypeHint(ValueType.DOUBLE) .field("double"); - aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType3); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - result = (Terms) aggregator.buildAggregation(0L); - assertEquals("_name", result.getName()); - assertEquals(0, result.getBuckets().size()); + result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType3); + assertNull(result); } } } @@ -876,19 +834,25 @@ public void testEmpty() throws Exception { public void testUnmapped() throws Exception { try (Directory directory = newDirectory()) { try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + + indexWriter.addDocument(singleton(new NumericDocValuesField("other", 7))); + indexWriter.commit(); + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { IndexSearcher indexSearcher = newIndexSearcher(indexReader); ValueType[] valueTypes = new ValueType[]{ValueType.STRING, ValueType.LONG, ValueType.DOUBLE}; String[] fieldNames = new String[]{"string", "long", "double"}; + + MappedFieldType fieldType = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType.setName("other"); + fieldType.setHasDocValues(true); + for (int i = 0; i < fieldNames.length; i++) { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, (MappedFieldType) null); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); + assertEquals("_name", result.getName()); assertEquals(0, result.getBuckets().size()); assertFalse(AggregationInspectionHelper.hasValue((InternalTerms)result)); @@ -922,11 +886,49 @@ public void testUnmappedWithMissing() throws Exception { TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .userValueTypeHint(valueTypes[i]) .field(fieldNames[i]).missing(missingValues[i]); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType1); + assertEquals("_name", result.getName()); + assertEquals(1, result.getBuckets().size()); + assertEquals(missingValues[i], result.getBuckets().get(0).getKey()); + assertEquals(1, result.getBuckets().get(0).getDocCount()); + } + } + } + } + } + + public void testMissing() throws Exception { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + + Document document = new Document(); + document.add(new NumericDocValuesField("field", 100)); + indexWriter.addDocument(document); + + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + + MappedFieldType fieldType1 = new KeywordFieldMapper.KeywordFieldType(); + fieldType1.setName("string"); + fieldType1.setHasDocValues(true); + + MappedFieldType fieldType2 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType2.setName("long"); + fieldType2.setHasDocValues(true); + + MappedFieldType fieldType3 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.DOUBLE); + fieldType3.setName("double"); + fieldType3.setHasDocValues(true); + + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + String[] fieldNames = new String[]{"string", "long", "double"}; + Object[] missingValues = new Object[]{"abc", 19L, 19.2}; + + + for (int i = 0; i < fieldNames.length; i++) { + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .field(fieldNames[i]).missing(missingValues[i]); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, + fieldType1, fieldType2, fieldType3); assertEquals("_name", result.getName()); assertEquals(1, result.getBuckets().size()); assertEquals(missingValues[i], result.getBuckets().get(0).getKey()); @@ -1003,11 +1005,7 @@ public void testIpField() throws Exception { IndexSearcher indexSearcher = newIndexSearcher(indexReader); TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") .field(field); // Note - other places we throw IllegalArgumentException - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType); assertEquals("_name", result.getName()); assertEquals(1, result.getBuckets().size()); assertEquals("192.168.100.42", result.getBuckets().get(0).getKey()); @@ -1055,11 +1053,7 @@ public void testNestedTermsAgg() throws Exception { fieldType2.setName("field2"); fieldType2.setHasDocValues(true); - Aggregator aggregator = createAggregator(aggregationBuilder, indexSearcher, fieldType1, fieldType2); - aggregator.preCollection(); - indexSearcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - Terms result = (Terms) aggregator.buildAggregation(0L); + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType1, fieldType2); assertEquals(3, result.getBuckets().size()); assertEquals("a", result.getBuckets().get(0).getKeyAsString()); assertEquals(1L, result.getBuckets().get(0).getDocCount()); @@ -1289,6 +1283,65 @@ public void testOrderByPipelineAggregation() throws Exception { } } + public void testSingleValuedFieldWithScript() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + + Document document = new Document(); + document.add(new NumericDocValuesField("value", 100)); + indexWriter.addDocument(document); + + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + + MappedFieldType fieldType1 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType1.setName("value"); + fieldType1.setHasDocValues(true); + + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, SINGLE_SCRIPT, Collections.emptyMap())); + + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType1); + assertEquals("_name", result.getName()); + assertEquals(1, result.getBuckets().size()); + assertEquals("1", result.getBuckets().get(0).getKey()); + assertEquals(1, result.getBuckets().get(0).getDocCount()); + } + } + } + } + + public void testSingleValuedFieldWithValueScript() throws IOException { + try (Directory directory = newDirectory()) { + try (RandomIndexWriter indexWriter = new RandomIndexWriter(random(), directory)) { + + Document document = new Document(); + document.add(new NumericDocValuesField("value", 100)); + indexWriter.addDocument(document); + + try (IndexReader indexReader = maybeWrapReaderEs(indexWriter.getReader())) { + + MappedFieldType fieldType1 = new NumberFieldMapper.NumberFieldType(NumberFieldMapper.NumberType.LONG); + fieldType1.setName("value"); + fieldType1.setHasDocValues(true); + + IndexSearcher indexSearcher = newIndexSearcher(indexReader); + + TermsAggregationBuilder aggregationBuilder = new TermsAggregationBuilder("_name") + .field("value") + .script(new Script(ScriptType.INLINE, MockScriptEngine.NAME, VALUE_SCRIPT, Collections.emptyMap())); + + Terms result = searchAndReduce(indexSearcher, new MatchAllDocsQuery(), aggregationBuilder, fieldType1); + assertEquals("_name", result.getName()); + assertEquals(1, result.getBuckets().size()); + assertEquals(101.0, result.getBuckets().get(0).getKey()); + assertEquals(1, result.getBuckets().get(0).getDocCount()); + } + } + } + } + private final SeqNoFieldMapper.SequenceIDFields sequenceIDFields = SeqNoFieldMapper.SequenceIDFields.emptySeqID(); private List generateDocsWithNested(String id, int value, int[] nestedValues) { List documents = new ArrayList<>(); @@ -1352,11 +1405,7 @@ private IndexReader createIndexWithDoubles() throws IOException { private InternalAggregation buildInternalAggregation(TermsAggregationBuilder builder, MappedFieldType fieldType, IndexSearcher searcher) throws IOException { - TermsAggregator aggregator = createAggregator(builder, searcher, fieldType); - aggregator.preCollection(); - searcher.search(new MatchAllDocsQuery(), aggregator); - aggregator.postCollection(); - return aggregator.buildAggregation(0L); + return searchAndReduce(searcher, new MatchAllDocsQuery(), builder, fieldType); } }