Initial elasticsearch 6.x support

.gitignore

@@ -0,0 +1,3 @@
+build/
+*.log
+.gradle/

build.gradle

@@ -0,0 +1,51 @@
+def elasticVersion = '6.2.4'
+
+buildscript {
+    def elasticVersion = '6.2.4'
+    repositories {
+        mavenLocal()
+        mavenCentral()
+        jcenter()
+    }
+
+    dependencies {
+        classpath "org.elasticsearch.gradle:build-tools:${elasticVersion}"
+    }
+}
+
+group = 'org.elasticsearch.plugin'
+version = elasticVersion
+
+apply plugin: 'java'
+apply plugin: 'elasticsearch.esplugin'
+apply plugin: 'idea'
+
+esplugin {
+    name 'envelope-aggregation'
+    description 'Returns envelope of geo points'
+    classname 'org.opendatasoft.elasticsearch.plugin.EnvelopeAggregation'
+}
+
+dependencies {
+    compile "org.elasticsearch:elasticsearch:${elasticVersion}"
+    // We need jts and spatial4j in the same version as what ES uses, to avoid
+    // conflicts
+    //compile 'com.vividsolutions:jts:1.13'
+    //compile 'org.locationtech.spatial4j:spatial4j:0.6'
+}
+
+test.enabled = true
+
+// Skip license headers check
+licenseHeaders.enabled = false
+
+// Use elasticsearch checkstyle rules
+checkstyleTest.enabled = true
+
+// FIXME Dependency license check needs to be enabled
+licenseFile = rootProject.file('../LICENSE')
+noticeFile = rootProject.file('README.md')
+dependencyLicenses.enabled = false
+
+// FIXME thirdparty audit needs to be enabled
+thirdPartyAudit.enabled = false

src/main/java/org/opendatasoft/elasticsearch/plugin/EnvelopeAggregation.java

@@ -0,0 +1,24 @@
+package org.opendatasoft.elasticsearch.plugin;
+
+import java.util.ArrayList;
+import org.elasticsearch.plugins.Plugin;
+import org.elasticsearch.plugins.SearchPlugin;
+import org.opendatasoft.elasticsearch.search.aggregations.metric.ConvexHullAggregationBuilder;
+import org.opendatasoft.elasticsearch.search.aggregations.metric.InternalConvexHull;
+
+public class EnvelopeAggregation extends Plugin implements SearchPlugin {
+    @Override
+    public ArrayList<SearchPlugin.AggregationSpec> getAggregations() {
+        ArrayList<SearchPlugin.AggregationSpec> r = new ArrayList<>();
+
+        r.add(
+                new AggregationSpec(
+                        ConvexHullAggregationBuilder.NAME,
+                        ConvexHullAggregationBuilder::new,
+                        ConvexHullAggregationBuilder::parse)
+                .addResultReader(InternalConvexHull::new)
+        );
+
+        return r;
+    }
+}

src/main/java/org/opendatasoft/elasticsearch/search/aggregations/metric/ConvexHull.java

@@ -0,0 +1,8 @@
+package org.opendatasoft.elasticsearch.search.aggregations.metric;
+
+import com.vividsolutions.jts.geom.Geometry;
+import org.elasticsearch.search.aggregations.Aggregation;
+
+public interface ConvexHull extends Aggregation {
+    Geometry getShape();
+}

src/main/java/org/opendatasoft/elasticsearch/search/aggregations/metric/ConvexHullAggregationBuilder.java

@@ -0,0 +1,78 @@
+package org.opendatasoft.elasticsearch.search.aggregations.metric;
+
+import org.elasticsearch.common.io.stream.StreamInput;
+import org.elasticsearch.common.io.stream.StreamOutput;
+import org.elasticsearch.common.xcontent.ObjectParser;
+import org.elasticsearch.common.xcontent.XContentBuilder;
+import org.elasticsearch.common.xcontent.XContentParser;
+import org.elasticsearch.search.aggregations.AggregationBuilder;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValueType;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.aggregations.support.ValuesSourceParserHelper;
+import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+
+
+public class ConvexHullAggregationBuilder extends ValuesSourceAggregationBuilder<ValuesSource.GeoPoint, ConvexHullAggregationBuilder> {
+
+    public static final String NAME = "envelope";
+
+    private static final ObjectParser<ConvexHullAggregationBuilder, Void> PARSER;
+    static {
+        PARSER = new ObjectParser<>(ConvexHullAggregationBuilder.NAME);
+        ValuesSourceParserHelper.declareGeoFields(PARSER, false, false);
+    }
+
+    public static AggregationBuilder parse(String aggregationName, XContentParser parser) throws IOException {
+        return PARSER.parse(parser, new ConvexHullAggregationBuilder(aggregationName), null);
+    }
+
+    public ConvexHullAggregationBuilder(String name) {
+        super(name, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+    }
+
+    public ConvexHullAggregationBuilder(StreamInput in) throws IOException {
+        super(in, ValuesSourceType.GEOPOINT, ValueType.GEOPOINT);
+    }
+
+    @Override
+    public String getType() {
+        return NAME;
+    }
+
+    @Override
+    protected void innerWriteTo(StreamOutput out) throws IOException {
+
+    }
+
+    @Override
+    protected ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ?> innerBuild(
+            SearchContext context,
+            ValuesSourceConfig<ValuesSource.GeoPoint> config,
+            AggregatorFactory<?> parent,
+            AggregatorFactories.Builder subFactoriesBuilder) throws IOException {
+        return new ConvexHullAggregator.Factory(name, config, context, parent, subFactoriesBuilder, metaData);
+    }
+
+    @Override
+    protected XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException {
+        return builder;
+    }
+
+    @Override
+    protected int innerHashCode() {
+        return 0;
+    }
+
+    @Override
+    protected boolean innerEquals(Object obj) {
+        return false;
+    }
+}

src/main/java/org/opendatasoft/elasticsearch/search/aggregations/metric/ConvexHullAggregator.java

@@ -0,0 +1,137 @@
+package org.opendatasoft.elasticsearch.search.aggregations.metric;
+
+import org.apache.logging.log4j.Logger;
+import com.vividsolutions.jts.geom.Coordinate;
+import com.vividsolutions.jts.geom.Geometry;
+import org.apache.lucene.index.LeafReaderContext;
+import org.elasticsearch.common.geo.GeoPoint;
+import org.elasticsearch.common.geo.builders.ShapeBuilder;
+import org.elasticsearch.common.logging.ESLoggerFactory;
+import org.elasticsearch.common.util.BigArrays;
+import org.elasticsearch.common.util.ObjectArray;
+import org.elasticsearch.index.fielddata.MultiGeoPointValues;
+import org.elasticsearch.search.aggregations.Aggregator;
+import org.elasticsearch.search.aggregations.AggregatorFactories;
+import org.elasticsearch.search.aggregations.AggregatorFactory;
+import org.elasticsearch.search.aggregations.InternalAggregation;
+import org.elasticsearch.search.aggregations.LeafBucketCollector;
+import org.elasticsearch.search.aggregations.LeafBucketCollectorBase;
+import org.elasticsearch.search.aggregations.metrics.MetricsAggregator;
+import org.elasticsearch.search.aggregations.pipeline.PipelineAggregator;
+import org.elasticsearch.search.aggregations.support.ValuesSource;
+import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory;
+import org.elasticsearch.search.aggregations.support.ValuesSourceConfig;
+import org.elasticsearch.search.internal.SearchContext;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+public class ConvexHullAggregator extends MetricsAggregator {
+    private static final Logger logger = ESLoggerFactory.getLogger(ConvexHullAggregator.class);
+
+    private final ValuesSource.GeoPoint valuesSource;
+    private MultiGeoPointValues values;
+    private ObjectArray<Set<Coordinate>> geoPoints;
+
+    protected ConvexHullAggregator(
+            String name, SearchContext context, Aggregator parent,
+            ValuesSource.GeoPoint valuesSource, List<PipelineAggregator> pipelineAggregators,
+            Map<String, Object> metaData) throws IOException {
+        super(name, context, parent, pipelineAggregators, metaData);
+        this.valuesSource = valuesSource;
+        geoPoints = context.bigArrays().newObjectArray(10);
+    }
+
+    @Override
+    public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) {
+        logger.info("Calling getLeafCollector()");
+        if (valuesSource == null) {
+            return LeafBucketCollector.NO_OP_COLLECTOR;
+        }
+        final BigArrays bigArrays = context.bigArrays();
+        final MultiGeoPointValues values = valuesSource.geoPointValues(ctx);
+        return new LeafBucketCollectorBase(sub, values) {
+            @Override
+            public void collect(int doc, long bucket) throws IOException {
+                logger.info("Calling collect");
+                if (bucket >= geoPoints.size()) {
+                    geoPoints = bigArrays.grow(geoPoints, bucket + 1);
+                }
+
+                Set<Coordinate> polygon = geoPoints.get(bucket);
+
+                if (polygon == null) {
+                    polygon = new HashSet<Coordinate>();
+                    geoPoints.set(bucket, polygon);
+                }
+
+                values.advanceExact(doc);
+                final int valuesCount = values.docValueCount();
+
+                for (int i=0; i < valuesCount; i++) {
+                    GeoPoint value = values.nextValue();
+                    polygon.add(new Coordinate(value.getLon(), value.getLat()));
+                }
+            }
+        };
+    }
+
+    @Override
+    public InternalAggregation buildAggregation(long bucket) throws IOException {
+        logger.info("Calling buildAggregation(" + bucket + ")");
+        if (valuesSource == null) {
+            logger.info("valuesSource is null");
+            return buildEmptyAggregation();
+        }
+        logger.info("valuesSources: " + valuesSource);
+        Set<Coordinate> points = geoPoints.get(bucket);
+
+        if (points == null) {
+            logger.info("points is null");
+            return buildEmptyAggregation();
+        }
+
+        Geometry convexHull = new com.vividsolutions.jts.algorithm.ConvexHull(
+                points.toArray(new Coordinate[points.size()]),
+                ShapeBuilder.FACTORY
+        ).getConvexHull();
+        logger.info(convexHull);
+
+        return new InternalConvexHull(name, convexHull, pipelineAggregators(), metaData());
+    }
+
+    @Override
+    public InternalAggregation buildEmptyAggregation() {
+        logger.info("Calling buildEmptyAggregation()");
+        return new InternalConvexHull(name, null, pipelineAggregators(), metaData());
+    }
+
+    public static class Factory extends ValuesSourceAggregatorFactory<ValuesSource.GeoPoint, ConvexHullAggregator.Factory> {
+
+        protected Factory(
+                String name, ValuesSourceConfig<ValuesSource.GeoPoint> config,
+                SearchContext context, AggregatorFactory<?> parent,
+                AggregatorFactories.Builder subFactoriesBuilder,
+                Map<String, Object> metaData) throws IOException {
+            super(name, config, context, parent, subFactoriesBuilder, metaData);
+        }
+
+        @Override
+        protected Aggregator createUnmapped(
+                Aggregator parent, List<PipelineAggregator> list,
+                Map<String, Object> metaData) throws IOException {
+            return new ConvexHullAggregator(name, context, parent, null, list, metaData);
+        }
+
+        @Override
+        protected Aggregator doCreateInternal(
+                ValuesSource.GeoPoint valuesSource, Aggregator parent,
+                boolean collectsFromSingleBucket, List<PipelineAggregator> list,
+                Map<String, Object> metaData) throws IOException {
+            return new ConvexHullAggregator(name, context, parent, valuesSource, list, metaData);
+        }
+    }
+}