From e0e8d6e5d12e84b9db385c3d7939b40d847460a6 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 30 Mar 2021 19:09:07 +0200 Subject: [PATCH 01/15] initial skeleton of vector tiles prototype implementation (#69338) --- x-pack/plugin/spatial/build.gradle | 23 ++ .../mapbox-vector-tile-3.1.0.jar.sha1 | 1 + .../licenses/mapbox-vector-tile-LICENSE.txt | 177 +++++++++++ .../licenses/mapbox-vector-tile-NOTICE.txt | 0 .../licenses/protobuf-java-3.14.0.jar.sha1 | 1 + .../licenses/protobuf-java-LICENSE.txt | 33 +++ .../spatial/licenses/protobuf-java-NOTICE.txt | 0 .../xpack/spatial/SpatialPlugin.java | 51 +++- .../AbstractVectorTileAggregator.java | 161 ++++++++++ .../aggregations/InternalVectorTile.java | 228 ++++++++++++++ .../VectorTileAggregationBuilder.java | 189 ++++++++++++ .../VectorTileAggregatorFactory.java | 68 +++++ .../VectorTileAggregatorSupplier.java | 30 ++ .../VectorTileGeoPointAggregator.java | 58 ++++ .../VectorTileGeoShapeAggregator.java | 133 +++++++++ .../spatial/vectortile/FeatureFactory.java | 185 ++++++++++++ .../spatial/vectortile/PointFactory.java | 105 +++++++ .../RestAggregatedVectorTileAction.java | 186 ++++++++++++ .../vectortile/RestVectorTileAction.java | 107 +++++++ .../vectortile/VectorTileAggConfig.java | 97 ++++++ .../vectortile/VectorTileGeometryBuilder.java | 67 +++++ .../spatial/vectortile/VectorTileUtils.java | 69 +++++ .../aggregations/InternalVectorTileTests.java | 182 ++++++++++++ .../VectorTileAggregationBuilderTests.java | 45 +++ .../VectorTileAggregatorTests.java | 239 +++++++++++++++ .../xpack/spatial/VectorTileGridRestIT.java | 279 ++++++++++++++++++ .../xpack/spatial/VectorTileRestIT.java | 105 +++++++ 27 files changed, 2814 insertions(+), 5 deletions(-) create mode 100644 x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 create mode 100644 x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt create mode 100644 x-pack/plugin/spatial/licenses/mapbox-vector-tile-NOTICE.txt create mode 100644 x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 create mode 100644 x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt create mode 100644 x-pack/plugin/spatial/licenses/protobuf-java-NOTICE.txt create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java create mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java create mode 100644 x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java create mode 100644 x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 8a196b50b9cb9..bef0731cf8a29 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -15,6 +15,10 @@ dependencies { yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) api project(path: ':modules:geo') restTestConfig project(path: ':modules:geo', configuration: 'restTests') + api "com.wdtinc:mapbox-vector-tile:3.1.0" + api "com.google.protobuf:protobuf-java:3.14.0" + yamlRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") + yamlRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0") } restResources { @@ -30,3 +34,22 @@ testClusters.all { setting 'xpack.license.self_generated.type', 'trial' testDistribution = 'DEFAULT' } + +tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // uses internal java api: sun.misc.Unsafe + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor' + ) + + ignoreMissingClasses( + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory' + ) +} + diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 b/x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 new file mode 100644 index 0000000000000..c98d2861a1bb8 --- /dev/null +++ b/x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 @@ -0,0 +1 @@ +06c4432c7885a3938571a57e73cc1444d7a39f12 \ No newline at end of file diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt b/x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt new file mode 100644 index 0000000000000..f433b1a53f5b8 --- /dev/null +++ b/x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt @@ -0,0 +1,177 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-NOTICE.txt b/x-pack/plugin/spatial/licenses/mapbox-vector-tile-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 b/x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 new file mode 100644 index 0000000000000..a8a1dde74b47d --- /dev/null +++ b/x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 @@ -0,0 +1 @@ +bb6430f70647fc349fffd1690ddb889dc3ea6699 \ No newline at end of file diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt b/x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt new file mode 100644 index 0000000000000..fc4865a95ffdc --- /dev/null +++ b/x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt @@ -0,0 +1,33 @@ + +Copyright 2008 Google Inc. All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are +met: + + * Redistributions of source code must retain the above copyright +notice, this list of conditions and the following disclaimer. + * Redistributions in binary form must reproduce the above +copyright notice, this list of conditions and the following disclaimer +in the documentation and/or other materials provided with the +distribution. + * Neither the name of Google Inc. nor the names of its +contributors may be used to endorse or promote products derived from +this software without specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS +"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT +LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR +A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT +OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, +SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT +LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, +DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY +THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + +Code generated by the Protocol Buffer compiler is owned by the owner +of the input file used when generating it. This code is not +standalone and requires a support library to be linked with it. This +support library is itself covered by the above license. diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-NOTICE.txt b/x-pack/plugin/spatial/licenses/protobuf-java-NOTICE.txt new file mode 100644 index 0000000000000..e69de29bb2d1d diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index e119ff0d25c86..8b2fe7793fe84 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -8,6 +8,12 @@ import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.geo.GeoPlugin; import org.elasticsearch.index.mapper.Mapper; @@ -18,6 +24,8 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; @@ -27,6 +35,8 @@ import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregator; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -34,6 +44,8 @@ import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction; +import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; +import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; @@ -53,6 +65,8 @@ import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import org.elasticsearch.xpack.spatial.vectortile.RestAggregatedVectorTileAction; +import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction; import java.util.Arrays; import java.util.Collections; @@ -60,6 +74,7 @@ import java.util.List; import java.util.Map; import java.util.function.Consumer; +import java.util.function.Supplier; import static java.util.Collections.singletonList; @@ -72,13 +87,31 @@ protected XPackLicenseState getLicenseState() { } @Override - public List> getActions() { + public List> getActions() { return Arrays.asList( - new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.SPATIAL, SpatialUsageTransportAction.class), - new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.SPATIAL, SpatialInfoTransportAction.class), - new ActionPlugin.ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class)); + new ActionHandler<>(XPackUsageFeatureAction.SPATIAL, SpatialUsageTransportAction.class), + new ActionHandler<>(XPackInfoFeatureAction.SPATIAL, SpatialInfoTransportAction.class), + new ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class)); } + @Override + public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster) { + return Arrays.asList( + new RestVectorTileAction(), + new RestAggregatedVectorTileAction()); + } + + @Override + public List> getExecutorBuilders(Settings settings) { + FixedExecutorBuilder indexing = + new FixedExecutorBuilder(settings, "vector_tile_generation", 1, -1, "thread_pool.vectortile", false); + return Collections.singletonList(indexing); + } + + @Override public Map getMappers() { Map mappers = new HashMap<>(super.getMappers()); @@ -113,7 +146,15 @@ public List getAggregations() { usage.track(SpatialStatsAction.Item.GEOLINE, checkLicense(GeoLineAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) .addResultReader(InternalGeoLine::new) - .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage)); + .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage), + new AggregationSpec( + VectorTileAggregationBuilder.NAME, + VectorTileAggregationBuilder::new, + VectorTileAggregationBuilder.PARSER) + // usage.track(SpatialStatsAction.Item.GEOLINE, + // checkLicense(VectorTileAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) + .addResultReader(InternalVectorTile::new) + .setAggregatorRegistrar(VectorTileAggregationBuilder::registerAggregators)); } @Override diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java new file mode 100644 index 0000000000000..13cf03f91f7e8 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java @@ -0,0 +1,161 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.apache.lucene.search.ScoreMode; +import org.elasticsearch.common.lease.Releasables; +import org.elasticsearch.common.util.LongArray; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.search.aggregations.metrics.MetricsAggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.xpack.spatial.vectortile.VectorTileUtils; + +import java.io.IOException; +import java.util.List; +import java.util.Map; + +/** + * Base Vector tile aggregator class. It might generate up to three layers: If there is any point it generates a POINT layer + * that cluster points in a cluster them in a 256 X 256 grid. If there is any line, it generated a LINE layer with a 4096 extent + * contains ing all lines. If there is any polygon, it generated a POLYGON layer with a 4096 extent contains ing all lines. + */ +public abstract class AbstractVectorTileAggregator extends MetricsAggregator { + + protected static final int POINT_EXTENT = 256; + protected static final String POINT_LAYER = "POINT"; + protected static final int LINE_EXTENT = 4096; + protected static final String LINE_LAYER = "LINE"; + protected static final int POLYGON_EXTENT = 4096; + protected static final String POLYGON_LAYER = "POLYGON"; + + protected static final String ID_TAG = "id"; + + protected final ValuesSource valuesSource; + protected final int x; + protected final int y; + protected final int z; + private VectorTile.Tile.Layer.Builder polygonLayerBuilder; + private int numPolygons; + private VectorTile.Tile.Layer.Builder lineLayerBuilder; + private int numLines; + private LongArray clusters; + private final double pointXScale; + private final double pointYScale; + protected final Rectangle rectangle; + final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); + + public AbstractVectorTileAggregator( + String name, + ValuesSourceConfig valuesSourceConfig, + int z, + int x, + int y, + AggregationContext context, + Aggregator parent, + Map metadata + ) throws IOException { + super(name, context, parent, metadata); + this.valuesSource = valuesSourceConfig.getValuesSource(); + this.z = z; + this.x = x; + this.y = y; + this.rectangle = VectorTileUtils.getTileBounds(z, x, y); + this.pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) POINT_EXTENT); + this.pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) POINT_EXTENT); + } + + @Override + public ScoreMode scoreMode() { + return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; + } + + protected void addPoint(double lat, double lon) { + final int x = (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + final int y = (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + POINT_EXTENT; + if (x >= 0 && x < POINT_EXTENT && y >= 0 && y < POINT_EXTENT) { + if (clusters == null) { + clusters = bigArrays().newLongArray(POINT_EXTENT * POINT_EXTENT); + } + final int pos = POINT_EXTENT * x + y; + clusters.increment(pos, 1); + } + } + + protected void addLineFeatures(String id, List features) { + for (VectorTile.Tile.Feature feature : features) { + if (lineLayerBuilder == null) { + lineLayerBuilder = VectorTile.Tile.Layer.newBuilder(); + lineLayerBuilder.setVersion(2); + lineLayerBuilder.setName(LINE_LAYER); + lineLayerBuilder.setExtent(LINE_EXTENT); + lineLayerBuilder.addKeys(ID_TAG); + } + valueBuilder.clear(); + valueBuilder.setStringValue(id); + lineLayerBuilder.addValues(valueBuilder); + VectorTile.Tile.Feature.Builder builder = feature.toBuilder(); + builder.addTags(0); + builder.addTags(numLines++); + lineLayerBuilder.addFeatures(builder); + } + } + + protected void addPolygonFeatures(String id, List features) { + for (VectorTile.Tile.Feature feature : features) { + if (polygonLayerBuilder == null) { + polygonLayerBuilder = VectorTile.Tile.Layer.newBuilder(); + polygonLayerBuilder.setVersion(2); + polygonLayerBuilder.setName(POLYGON_LAYER); + polygonLayerBuilder.setExtent(POLYGON_EXTENT); + polygonLayerBuilder.addKeys(ID_TAG); + } + valueBuilder.clear(); + valueBuilder.setStringValue(id); + polygonLayerBuilder.addValues(valueBuilder); + VectorTile.Tile.Feature.Builder builder = feature.toBuilder(); + builder.addTags(0); + builder.addTags(numPolygons++); + polygonLayerBuilder.addFeatures(builder); + } + } + + + + @Override + public InternalAggregation buildAggregation(long bucket) { + if (valuesSource == null) { + return buildEmptyAggregation(); + } + final VectorTile.Tile.Layer polygons = polygonLayerBuilder != null ? polygonLayerBuilder.build() : null; + final VectorTile.Tile.Layer lines = lineLayerBuilder != null ? lineLayerBuilder.build() : null; + final long[] points; + if (clusters != null) { + points = new long[POINT_EXTENT * POINT_EXTENT]; + for (int i = 0; i < POINT_EXTENT * POINT_EXTENT; i++) { + points[i] = clusters.get(i); + } + } else { + points = null; + } + return new InternalVectorTile(name, polygons, lines, points, metadata()); + } + + @Override + public InternalAggregation buildEmptyAggregation() { + return new InternalVectorTile(name, null, null, null, metadata()); + } + + @Override + public void doClose() { + Releasables.close(clusters); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java new file mode 100644 index 0000000000000..591f900c4433a --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java @@ -0,0 +1,228 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmdHdr; +import org.apache.lucene.util.BitUtil; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.aggregations.InternalAggregation; +import org.elasticsearch.xpack.spatial.vectortile.VectorTileUtils; + +import java.io.IOException; +import java.io.OutputStream; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; + +public class InternalVectorTile extends InternalAggregation { + private static String COUNT_KEY = "count"; + protected final VectorTile.Tile.Layer polygons; + protected final VectorTile.Tile.Layer lines; + protected final long[] points; + + public InternalVectorTile(String name, VectorTile.Tile.Layer polygons, + VectorTile.Tile.Layer lines, long[] points, Map metadata) { + super(name, metadata); + this.polygons = polygons; + this.lines = lines; + this.points = points; + } + + /** + * Read from a stream. + */ + public InternalVectorTile(StreamInput in) throws IOException { + super(in); + this.polygons = in.readBoolean() ? VectorTile.Tile.Layer.parseDelimitedFrom(in) : null; + this.lines = in.readBoolean() ? VectorTile.Tile.Layer.parseDelimitedFrom(in) : null; + this.points = in.readBoolean() ? in.readLongArray() : null; + } + + @Override + protected void doWriteTo(StreamOutput out) throws IOException { + if (polygons != null) { + out.writeBoolean(true); + polygons.writeDelimitedTo(out); + } else { + out.writeBoolean(false); + } + if (lines != null) { + out.writeBoolean(true); + lines.writeDelimitedTo(out); + } else { + out.writeBoolean(false); + } + if (points != null) { + out.writeBoolean(true); + out.writeLongArray(points); + } else { + out.writeBoolean(false); + } + } + + @Override + public String getWriteableName() { + return VectorTileAggregationBuilder.NAME; + } + + public byte[] getVectorTileBytes() { + return getVectorTile().toByteArray(); + } + + public void writeTileToStream(OutputStream stream) throws IOException { + getVectorTile().writeTo(stream); + } + + private VectorTile.Tile getVectorTile() { + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + if (polygons != null) { + tileBuilder.addLayers(polygons); + } + if (lines != null) { + tileBuilder.addLayers(lines); + } + if (points != null) { + tileBuilder.addLayers(buildPointLayer()); + } + return tileBuilder.build(); + } + + private VectorTile.Tile.Layer buildPointLayer() { + final VectorTile.Tile.Layer.Builder pointLayerBuilder = + VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.POINT_LAYER, AbstractVectorTileAggregator.POINT_EXTENT); + pointLayerBuilder.addKeys(COUNT_KEY); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); + final HashMap values = new HashMap<>(); + for (int i = 0; i < AbstractVectorTileAggregator.POINT_EXTENT; i++) { + final int xVal = i * AbstractVectorTileAggregator.POINT_EXTENT; + for (int j = 0; j < AbstractVectorTileAggregator.POINT_EXTENT; j++) { + final long count = points[xVal + j]; + if (count > 0) { + featureBuilder.clear(); + featureBuilder.setType(VectorTile.Tile.GeomType.POINT); + // create geometry commands + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(i)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(j)); + // Add count as key value pair + featureBuilder.addTags(0); + final int tagValue; + if (values.containsKey(count)) { + tagValue = values.get(count); + } else { + valueBuilder.clear(); + valueBuilder.setIntValue(count); + tagValue = values.size(); + pointLayerBuilder.addValues(valueBuilder); + values.put(count, tagValue); + } + featureBuilder.addTags(tagValue); + pointLayerBuilder.addFeatures(featureBuilder); + } + } + } + return pointLayerBuilder.build(); + } + + @Override + public InternalAggregation reduce(List aggregations, ReduceContext reduceContext) { + VectorTile.Tile.Layer.Builder polygons = null; + VectorTile.Tile.Layer.Builder lines = null; + long[] points = null; + for (InternalAggregation aggregation : aggregations) { + InternalVectorTile internalVectorTile = (InternalVectorTile) aggregation; + if (internalVectorTile.polygons != null) { + if (polygons == null) { + polygons = + VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.POLYGON_LAYER, + AbstractVectorTileAggregator.POLYGON_EXTENT + ); + polygons.addKeys(AbstractVectorTileAggregator.ID_TAG); + } + mergeLayer(polygons, internalVectorTile.polygons); + } + if (internalVectorTile.lines != null) { + if (lines == null) { + lines = VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.LINE_LAYER, + AbstractVectorTileAggregator.LINE_EXTENT + ); + lines.addKeys(AbstractVectorTileAggregator.ID_TAG); + } + mergeLayer(lines, internalVectorTile.lines); + } + if (internalVectorTile.points != null) { + if (points == null) { + points = new long[AbstractVectorTileAggregator.POINT_EXTENT * AbstractVectorTileAggregator.POINT_EXTENT]; + } + mergePoints(points, internalVectorTile.points); + } + } + final VectorTile.Tile.Layer polygonLayer = polygons != null ? polygons.build() : null; + final VectorTile.Tile.Layer lineLayer = lines != null ? lines.build() : null; + return new InternalVectorTile(name, polygonLayer, lineLayer, points, getMetadata()); + } + + private void mergeLayer(VectorTile.Tile.Layer.Builder layerBuilder, VectorTile.Tile.Layer layer) { + layerBuilder.addAllKeys(layer.getKeysList()); + layerBuilder.addAllValues(layer.getValuesList()); + for(int i = 0; i < layer.getFeaturesCount(); i++) { + layerBuilder.addFeatures(layer.getFeatures(i)); + } + } + + private void mergePoints(long[] pointsBuilder, long[] layer) { + for(int i = 0; i < pointsBuilder.length; i++) { + pointsBuilder[i] += layer[i]; + } + } + + @Override + protected boolean mustReduceOnSingleInternalAgg() { + return false; + } + + @Override + public Object getProperty(List path) { + if (path.isEmpty()) { + return this; + } else if (path.size() == 1 && "value".equals(path.get(0))) { + return getVectorTileBytes(); + } else { + throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path); + } + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(CommonFields.VALUE.getPreferredName(), getVectorTileBytes()); + return builder; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), polygons, lines, Arrays.hashCode(points)); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) return true; + if (obj == null || getClass() != obj.getClass()) return false; + if (super.equals(obj) == false) return false; + + InternalVectorTile that = (InternalVectorTile) obj; + return Arrays.equals(points, that.points) && + Objects.equals(polygons, that.polygons) && + Objects.equals(lines, that.lines); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java new file mode 100644 index 0000000000000..09bab2a7741b7 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java @@ -0,0 +1,189 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.io.stream.StreamInput; +import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.search.DocValueFormat; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; + +import java.io.IOException; +import java.util.Collections; +import java.util.Map; +import java.util.Objects; + +/** + * Aggregates geo data in a vector tile.It currently only supports to be o the top-level (e.g cannot have a parent) + * but it can change. + */ +public class VectorTileAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly { + public static final String NAME = "vector-tile"; + public static final ParseField ZOOM_FIELD = new ParseField("z"); + public static final ParseField X_FIELD = new ParseField("x"); + public static final ParseField Y_FIELD = new ParseField("y"); + public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = + new ValuesSourceRegistry.RegistryKey<>( + NAME, + VectorTileAggregatorSupplier.class + ); + public static final ObjectParser PARSER = + ObjectParser.fromBuilder(NAME, VectorTileAggregationBuilder::new); + + static { + ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false, false); + PARSER.declareInt(VectorTileAggregationBuilder::z, ZOOM_FIELD); + PARSER.declareInt(VectorTileAggregationBuilder::x, X_FIELD); + PARSER.declareInt(VectorTileAggregationBuilder::x, Y_FIELD); + } + + private int z; + private int x; + private int y; + + public static void registerAggregators(ValuesSourceRegistry.Builder builder) { + builder.register( + VectorTileAggregationBuilder.REGISTRY_KEY, + Collections.singletonList(CoreValuesSourceType.GEOPOINT), + VectorTileGeoPointAggregator::new, + true + ); + builder.register( + VectorTileAggregationBuilder.REGISTRY_KEY, + Collections.singletonList(GeoShapeValuesSourceType.instance()), + VectorTileGeoShapeAggregator::new, + true + ); + } + + public VectorTileAggregationBuilder(String name) { + super(name); + } + + protected VectorTileAggregationBuilder( + VectorTileAggregationBuilder clone, + AggregatorFactories.Builder factoriesBuilder, + Map metadata + ) { + super(clone, factoriesBuilder, metadata); + this.x = clone.x; + this.y = clone.y; + this.z = clone.z; + } + + @Override + protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { + return new VectorTileAggregationBuilder(this, factoriesBuilder, metadata); + } + + /** + * Read from a stream. + */ + public VectorTileAggregationBuilder(StreamInput in) throws IOException { + super(in); + z = in.readVInt(); + x = in.readVInt(); + y = in.readVInt(); + } + + @Override + protected ValuesSourceType defaultValueSourceType() { + return CoreValuesSourceType.GEOPOINT; + } + + @Override + protected void innerWriteTo(StreamOutput out) throws IOException { + out.writeVInt(z); + out.writeVInt(x); + out.writeVInt(y); + } + + @Override + protected ValuesSourceRegistry.RegistryKey getRegistryKey() { + return REGISTRY_KEY; + } + + @Override + protected VectorTileAggregatorFactory innerBuild( + AggregationContext context, + ValuesSourceConfig config, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder) throws IOException { + if (parent != null) { + // we don't allow vector-tile aggregations to be the child of a bucketing aggregation + throw new IllegalArgumentException(NAME + " aggregation must be at the top level"); + } + VectorTileAggregatorSupplier aggregatorSupplier = + context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); + return new VectorTileAggregatorFactory(name, config, z, x, y, context, parent, + subFactoriesBuilder, metadata, aggregatorSupplier); + } + + @Override + public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { + builder.field(ZOOM_FIELD.getPreferredName(), z); + builder.field(X_FIELD.getPreferredName(), x); + builder.field(Y_FIELD.getPreferredName(), y); + return builder; + } + + @Override + public String getType() { + return NAME; + } + + public VectorTileAggregationBuilder z(int z) { + this.z = z; + return this; + } + + public VectorTileAggregationBuilder x(int x) { + this.x = x; + return this; + } + + public VectorTileAggregationBuilder y(int y) { + this.y = y; + return this; + } + + @Override + protected ValuesSourceConfig resolveConfig(AggregationContext context) { + // TODO: make this behavior right + if (field() == null && script() == null) { + return new ValuesSourceConfig(CoreValuesSourceType.GEOPOINT, null, true, null, null, 1.0, null, DocValueFormat.RAW, context); + } else { + return super.resolveConfig(context); + } + } + + @Override + public boolean equals(Object o) { + if (this == o) return true; + if (o == null || getClass() != o.getClass()) return false; + if (super.equals(o) == false) return false; + VectorTileAggregationBuilder that = (VectorTileAggregationBuilder) o; + return z == that.z && x == that.x && y == that.y; + } + + @Override + public int hashCode() { + return Objects.hash(super.hashCode(), z, x, y); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java new file mode 100644 index 0000000000000..5b4a6c45bb043 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.AggregatorFactory; +import org.elasticsearch.search.aggregations.CardinalityUpperBound; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.Map; + +class VectorTileAggregatorFactory extends ValuesSourceAggregatorFactory { + + private final VectorTileAggregatorSupplier aggregatorSupplier; + private final int x; + private final int y; + private final int z; + + VectorTileAggregatorFactory( + String name, + ValuesSourceConfig config, + int z, + int x, + int y, + AggregationContext context, + AggregatorFactory parent, + AggregatorFactories.Builder subFactoriesBuilder, + Map metadata, + VectorTileAggregatorSupplier aggregatorSupplier + ) throws IOException { + super(name, config, context, parent, subFactoriesBuilder, metadata); + this.aggregatorSupplier = aggregatorSupplier; + this.z = z; + this.x = x; + this.y = y; + } + + @Override + protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { + return new AbstractVectorTileAggregator(name, config, z, x, y, context, parent, metadata) { + @Override + protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { + return LeafBucketCollector.NO_OP_COLLECTOR; + } + }; + } + + @Override + protected Aggregator doCreateInternal( + Aggregator parent, + CardinalityUpperBound bucketCardinality, + Map metadata + ) throws IOException { + return aggregatorSupplier + .build(name, config, z, x, y, context, parent, metadata); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java new file mode 100644 index 0000000000000..e2e02bcef51d1 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java @@ -0,0 +1,30 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.Map; + +@FunctionalInterface +public interface VectorTileAggregatorSupplier { + + AbstractVectorTileAggregator build( + String name, + ValuesSourceConfig valuesSourceConfig, + int z, + int x, + int y, + AggregationContext context, + Aggregator parent, + Map metadata + ) throws IOException; +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java new file mode 100644 index 0000000000000..4a961c3dddd30 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java @@ -0,0 +1,58 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.index.fielddata.MultiGeoPointValues; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSource; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; + +import java.io.IOException; +import java.util.Map; + +/** + * Aggregator over a geo point field. + */ +public class VectorTileGeoPointAggregator extends AbstractVectorTileAggregator { + + public VectorTileGeoPointAggregator( + String name, + ValuesSourceConfig valuesSourceConfig, + int z, + int x, + int y, + AggregationContext context, + Aggregator parent, + Map metadata + ) throws IOException { + super(name, valuesSourceConfig, z, x, y, context, parent, metadata); + } + + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) { + final MultiGeoPointValues values = ((ValuesSource.GeoPoint) valuesSource).geoPointValues(ctx); + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + if (values.advanceExact(doc)) { + final int numPoints = values.docValueCount(); + for (int i = 0; i < numPoints; i++) { + final GeoPoint point = values.nextValue(); + addPoint(point.lat(), point.lon()); + } + } + } + }; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java new file mode 100644 index 0000000000000..3a976fa885285 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java @@ -0,0 +1,133 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.apache.lucene.index.LeafReaderContext; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.search.aggregations.Aggregator; +import org.elasticsearch.search.aggregations.LeafBucketCollector; +import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.support.AggregationContext; +import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; +import org.elasticsearch.search.lookup.SourceLookup; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; +import org.elasticsearch.xpack.spatial.vectortile.FeatureFactory; +import org.elasticsearch.xpack.spatial.vectortile.PointFactory; + +import java.io.IOException; +import java.util.List; +import java.util.Map; +import java.util.Set; + +/** + * Aggregator over a geo shape field. It skips shapes where bounding box is smaller + * than the distance between two pixels. It reads Geometry from source which is slow. + */ +public class VectorTileGeoShapeAggregator extends AbstractVectorTileAggregator { + + private final String fieldName; + private final double latPointPrecision; + private final double lonPointPrecision; + private final MappedFieldType sourceField; + private final GeometryParser parser = new GeometryParser(true, false, false); + + public VectorTileGeoShapeAggregator( + String name, + ValuesSourceConfig valuesSourceConfig, + int z, + int x, + int y, + AggregationContext context, + Aggregator parent, + Map metadata + ) throws IOException { + super(name, valuesSourceConfig, z, x, y, context, parent, metadata); + this.sourceField = context.getFieldType(SourceFieldMapper.NAME); + this.fieldName = valuesSourceConfig.fieldType().name(); + Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + // TODO: Reason the logic for when we should skip a complex geometry + this.latPointPrecision = 2 * (rectangle.getMaxLat() - rectangle.getMinLat()) / POINT_EXTENT; + this.lonPointPrecision = 2 * (rectangle.getMaxLon() - rectangle.getMinLon()) / POINT_EXTENT; + } + + @Override + public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) { + final GeoShapeValues values = ((GeoShapeValuesSource) valuesSource).geoShapeValues(ctx); + final FeatureFactory featureFactory = new FeatureFactory(z, x, y, POLYGON_EXTENT); + final PointFactory pointFactory = new PointFactory(); + final CustomFieldsVisitor visitor = new CustomFieldsVisitor(Set.of(), true); + return new LeafBucketCollectorBase(sub, values) { + @Override + public void collect(int doc, long bucket) throws IOException { + if (values.advanceExact(doc)) { + GeoShapeValues.GeoShapeValue shapeValue = values.value(); + switch (shapeValue.dimensionalShapeType()) { + case POINT: { + visitor.reset(); + ctx.reader().document(doc, visitor); + final SourceLookup lookup = new SourceLookup(); + lookup.setSource(visitor.source()); + final Object pointObject = lookup.get(fieldName); + if (pointObject != null) { + final List points = pointFactory.getPoints(parser.parseGeometry(pointObject)); + for (Point point : points) { + addPoint(point.getLat(), point.getLon()); + } + } + break; + } + case LINE: { + if (skip(shapeValue)) { + addPoint(shapeValue.lat(), shapeValue.lon()); + } else { + visitor.reset(); + ctx.reader().document(doc, visitor); + final SourceLookup lookup = new SourceLookup(); + lookup.setSource(visitor.source()); + final Object lines = lookup.get(fieldName); + if (lines != null) { + addLineFeatures(visitor.id(), + featureFactory.getFeatures(parser.parseGeometry(lines))); + } + } + break; + } + case POLYGON: { + if (skip(shapeValue)) { + addPoint(shapeValue.lat(), shapeValue.lon()); + } else { + visitor.reset(); + ctx.reader().document(doc, visitor); + final SourceLookup lookup = new SourceLookup(); + lookup.setSource(visitor.source()); + final Object polygons = lookup.get(fieldName); + if (polygons != null) { + addPolygonFeatures(visitor.id(), featureFactory.getFeatures(parser.parseGeometry(polygons))); + } + } + break; + } + } + } + } + }; + } + + private boolean skip(GeoShapeValues.GeoShapeValue shapeValue) { + final double width = shapeValue.boundingBox().maxX() - shapeValue.boundingBox().minX(); + final double height = shapeValue.boundingBox().maxY() - shapeValue.boundingBox().minY(); + return width <= this.lonPointPrecision && height <= this.latPointPrecision; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java new file mode 100644 index 0000000000000..29077ca2d3258 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java @@ -0,0 +1,185 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.vectortile; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IGeometryFilter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Envelope; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.LineString; + +import java.util.List; + +public class FeatureFactory { + + private final IGeometryFilter acceptAllGeomFilter = geometry -> true; + private final IUserDataConverter ignoreUserData = new UserDataIgnoreConverter(); + private final MvtLayerParams layerParams; + private final GeometryFactory geomFactory = new GeometryFactory(); + private final MvtLayerProps layerProps = new MvtLayerProps(); + private final JTSGeometryBuilder builder; + + private final Envelope tileEnvelope; + private final Envelope clipEnvelope; + + public FeatureFactory(int z, int x, int y, int extent) { + this.tileEnvelope = VectorTileUtils.getJTSTileBounds(z, x, y); + this.clipEnvelope = VectorTileUtils.getJTSTileBounds(z, x, y); + this.clipEnvelope.expandBy(tileEnvelope.getWidth() * 0.1d, tileEnvelope.getHeight() * 0.1d); + this.builder = new JTSGeometryBuilder(geomFactory); + // TODO: Not sure what is the difference between extent and tile size? + this.layerParams = new MvtLayerParams(extent, extent); + } + + public List getFeatures(Geometry geometry) { + TileGeomResult tileGeom = + JtsAdapter.createTileGeom(JtsAdapter.flatFeatureList(geometry.visit(builder)), + tileEnvelope, clipEnvelope, geomFactory, layerParams, acceptAllGeomFilter); + // MVT tile geometry to MVT features + return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, ignoreUserData); + } + + private static class JTSGeometryBuilder implements GeometryVisitor { + + private final GeometryFactory geomFactory; + + JTSGeometryBuilder(GeometryFactory geomFactory) { + this.geomFactory = geomFactory; + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Circle circle) { + throw new IllegalArgumentException("Circle is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(GeometryCollection collection) { + throw new IllegalArgumentException("Circle is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(LinearRing ring) throws RuntimeException { + throw new IllegalArgumentException("LinearRing is not supported"); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Point point) throws RuntimeException { + return buildPoint(point); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiPoint multiPoint) throws RuntimeException { + final org.locationtech.jts.geom.Point[] points = new org.locationtech.jts.geom.Point[multiPoint.size()]; + for (int i = 0; i < multiPoint.size(); i++) { + points[i] = buildPoint(multiPoint.get(i)); + } + return geomFactory.createMultiPoint(points); + } + + private org.locationtech.jts.geom.Point buildPoint(Point point) { + final double x = VectorTileUtils.lonToSphericalMercator(point.getX()); + final double y = VectorTileUtils.latToSphericalMercator(point.getY()); + return geomFactory.createPoint(new Coordinate(x, y)); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Line line) { + return buildLine(line); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiLine multiLine) throws RuntimeException { + LineString[] lineStrings = new LineString[multiLine.size()]; + for (int i = 0; i < multiLine.size(); i++) { + lineStrings[i] = buildLine(multiLine.get(i)); + } + return geomFactory.createMultiLineString(lineStrings); + } + + private LineString buildLine(Line line) { + final Coordinate[] coordinates = new Coordinate[line.length()]; + for (int i = 0; i < line.length(); i++) { + final double x = VectorTileUtils.lonToSphericalMercator(line.getX(i)); + final double y = VectorTileUtils.latToSphericalMercator(line.getY(i)); + coordinates[i] = new Coordinate(x, y); + } + return geomFactory.createLineString(coordinates); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Polygon polygon) throws RuntimeException { + return buildPolygon(polygon); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(MultiPolygon multiPolygon) throws RuntimeException { + org.locationtech.jts.geom.Polygon[] polygons = new org.locationtech.jts.geom.Polygon[multiPolygon.size()]; + for (int i = 0; i < multiPolygon.size(); i++) { + polygons[i] = buildPolygon(multiPolygon.get(i)); + } + return geomFactory.createMultiPolygon(polygons); + } + + private org.locationtech.jts.geom.Polygon buildPolygon(Polygon polygon) { + final org.locationtech.jts.geom.LinearRing outerShell = buildLinearRing(polygon.getPolygon()); + if (polygon.getNumberOfHoles() == 0) { + return geomFactory.createPolygon(outerShell); + } + org.locationtech.jts.geom.LinearRing[] holes = new org.locationtech.jts.geom.LinearRing[polygon.getNumberOfHoles()]; + for (int i = 0; i < polygon.getNumberOfHoles(); i++) { + holes[i] = buildLinearRing(polygon.getHole(i)); + } + return geomFactory.createPolygon(outerShell, holes); + } + + private org.locationtech.jts.geom.LinearRing buildLinearRing(LinearRing ring) throws RuntimeException { + final Coordinate[] coordinates = new Coordinate[ring.length()]; + for (int i = 0; i < ring.length(); i++) { + final double x = VectorTileUtils.lonToSphericalMercator(ring.getX(i)); + final double y = VectorTileUtils.latToSphericalMercator(ring.getY(i)); + coordinates[i] = new Coordinate(x, y); + } + return geomFactory.createLinearRing(coordinates); + } + + @Override + public org.locationtech.jts.geom.Geometry visit(Rectangle rectangle) throws RuntimeException { + // TODO: handle degenerated rectangles? + final double xMin = VectorTileUtils.lonToSphericalMercator(rectangle.getMinX()); + final double yMin = VectorTileUtils.latToSphericalMercator(rectangle.getMinY()); + final double xMax = VectorTileUtils.lonToSphericalMercator(rectangle.getMaxX()); + final double yMax = VectorTileUtils.latToSphericalMercator(rectangle.getMaxY()); + final Coordinate[] coordinates = new Coordinate[5]; + coordinates[0] = new Coordinate(xMin, yMin); + coordinates[1] = new Coordinate(xMax, yMin); + coordinates[2] = new Coordinate(xMax, yMax); + coordinates[3] = new Coordinate(xMin, yMax); + coordinates[4] = new Coordinate(xMin, yMin); + return geomFactory.createPolygon(coordinates); + } + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java new file mode 100644 index 0000000000000..7824342ed16a4 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.vectortile; + +import org.elasticsearch.geometry.Circle; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.GeometryCollection; +import org.elasticsearch.geometry.GeometryVisitor; +import org.elasticsearch.geometry.Line; +import org.elasticsearch.geometry.LinearRing; +import org.elasticsearch.geometry.MultiLine; +import org.elasticsearch.geometry.MultiPoint; +import org.elasticsearch.geometry.MultiPolygon; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Polygon; +import org.elasticsearch.geometry.Rectangle; + +import java.util.ArrayList; +import java.util.List; + +public class PointFactory { + + private PointExtractor pointExtractor = new PointExtractor(); + + public PointFactory() { + } + + public List getPoints(Geometry geometry) { + pointExtractor.points.clear(); + geometry.visit(pointExtractor); + return pointExtractor.points; + } + + private static class PointExtractor implements GeometryVisitor { + + List points = new ArrayList<>(); + + PointExtractor() { + } + + @Override + public Void visit(Point point) throws IllegalArgumentException { + points.add(point); + return null; + } + + @Override + public Void visit(MultiPoint multiPoint) throws IllegalArgumentException { + for (Point p : multiPoint) { + visit(p); + } + return null; + } + + + @Override + public Void visit(GeometryCollection collection) throws IllegalArgumentException { + for (Geometry geometry : collection) { + geometry.visit(this); + } + return null; + } + + @Override + public Void visit(Circle circle) { + return null; + } + + @Override + public Void visit(Line line) throws IllegalArgumentException { + return null; + } + + @Override + public Void visit(LinearRing ring) throws IllegalArgumentException { + return null; + } + + @Override + public Void visit(MultiLine multiLine) throws IllegalArgumentException { + return null; + } + + + @Override + public Void visit(MultiPolygon multiPolygon) throws IllegalArgumentException { + return null; + } + + @Override + public Void visit(Polygon polygon) throws IllegalArgumentException { + return null; + } + + @Override + public Void visit(Rectangle rectangle) throws IllegalArgumentException { + return null; + } + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java new file mode 100644 index 0000000000000..47f1646f2b5ec --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java @@ -0,0 +1,186 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.vectortile; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; + +import java.io.IOException; +import java.util.HashMap; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestAggregatedVectorTileAction extends BaseRestHandler { + + private static final String INDEX_PARAM = "index"; + private static final String FIELD_PARAM = "field"; + private static final String Z_PARAM = "z"; + private static final String X_PARAM = "x"; + private static final String Y_PARAM = "y"; + private static final String TYPE_PARAM = "type"; + private static final String GRID_TYPE = "grid"; + + private static final String GRID_FIELD = "grid"; + private static final String BOUNDS_FIELD = "bounds"; + + @Override + public List routes() { + return List.of(new Route(GET, "{index}/_agg_mvt/{field}/{z}/{x}/{y}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + final String index = restRequest.param(INDEX_PARAM); + final String field = restRequest.param(FIELD_PARAM); + final int z = Integer.parseInt(restRequest.param(Z_PARAM)); + final int x = Integer.parseInt(restRequest.param(X_PARAM)); + final int y = Integer.parseInt(restRequest.param(Y_PARAM)); + final boolean isGrid = restRequest.hasParam(TYPE_PARAM) && GRID_TYPE.equals(restRequest.param(TYPE_PARAM)); + + final VectorTileAggConfig config = resolveConfig(restRequest); + final SearchRequestBuilder builder = searchBuilder(client, index.split(","), field, z, x, y, config); + final int extent = 1 << config.getScaling(); + // TODO: how do we handle cancellations? + return channel -> builder.execute( new RestResponseListener<>(channel) { + + @Override + public RestResponse buildResponse(SearchResponse searchResponse) throws IOException { + // TODO: of there is no hits, should we return an empty tile with no layers or + // a tile with empty layers? + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder(z, x, y, extent); + final InternalGeoTileGrid grid = searchResponse.getAggregations().get(GRID_FIELD); + tileBuilder.addLayers(getPointLayer(grid, geomBuilder)); + final InternalGeoBounds bounds = searchResponse.getAggregations().get(BOUNDS_FIELD); + tileBuilder.addLayers(getMetaLayer(bounds, geomBuilder)); + final BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput()); + tileBuilder.build().writeTo(bytesOut); + return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); + } + + private VectorTile.Tile.Layer.Builder getPointLayer(InternalGeoTileGrid t, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder pointLayerBuilder = VectorTileUtils.createLayerBuilder("AGG", extent); + pointLayerBuilder.addKeys("count"); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); + final HashMap values = new HashMap<>(); + + for (InternalGeoGridBucket bucket : t.getBuckets()) { + long count = bucket.getDocCount(); + if (count > 0) { + featureBuilder.clear(); + // create geometry commands + if (isGrid) { + Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); + } + // Add count as key value pair + featureBuilder.addTags(0); + final int tagValue; + if (values.containsKey(count)) { + tagValue = values.get(count); + } else { + valueBuilder.clear(); + valueBuilder.setIntValue(count); + tagValue = values.size(); + pointLayerBuilder.addValues(valueBuilder); + values.put(count, tagValue); + } + featureBuilder.addTags(tagValue); + pointLayerBuilder.addFeatures(featureBuilder); + } + } + return pointLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getMetaLayer(InternalGeoBounds t, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder("META", extent); + final GeoPoint topLeft = t.topLeft(); + final GeoPoint bottomRight = t.bottomRight(); + if (topLeft != null && bottomRight != null) { + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + metaLayerBuilder.addFeatures(featureBuilder); + } + return metaLayerBuilder; + } + }); + } + + public static SearchRequestBuilder searchBuilder(Client client, String[] index, String field, int z, int x, int y, + VectorTileAggConfig config) throws IOException { + final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); + if (config.getQueryBuilder() != null) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(config.getQueryBuilder()); + boolQueryBuilder.filter(qBuilder); + qBuilder = boolQueryBuilder; + } + int extent = 1 << config.getScaling(); + GeoBoundingBox boundingBox = + new GeoBoundingBox(new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())); + GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD) + .field(field).precision(Math.min(GeoTileUtils.MAX_ZOOM, z + config.getScaling())) + .setGeoBoundingBox(boundingBox).size(extent * extent); + if (config.getAggBuilder() != null) { + aBuilder.subAggregations(config.getAggBuilder()); + } + GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(field).wrapLongitude(false); + SearchRequestBuilder requestBuilder = client.prepareSearch(index).setQuery(qBuilder) + .addAggregation(aBuilder).addAggregation(boundsBuilder).setSize(0); + if (config.getRuntimeMappings() != null) { + requestBuilder.setRuntimeMappings(config.getRuntimeMappings()); + } + return requestBuilder; + } + + private VectorTileAggConfig resolveConfig(RestRequest restRequest) throws IOException { + if (restRequest.hasContent()) { + try (XContentParser parser = restRequest.contentParser()) { + return VectorTileAggConfig.PARSER.apply(parser, null); + } + } else { + return VectorTileAggConfig.getInstance(); + } + } + + @Override + public String getName() { + return "vectortile_aggregation_action"; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java new file mode 100644 index 0000000000000..4222536a365ca --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -0,0 +1,107 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.vectortile; + + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.Client; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; +import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; + +import java.io.IOException; +import java.util.List; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +public class RestVectorTileAction extends BaseRestHandler { + + @Override + public List routes() { + return List.of(new Route(GET, "{index}/_mvt/{field}/{z}/{x}/{y}")); + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + final String index = restRequest.param("index"); + final String field = restRequest.param("field"); + final int z = Integer.parseInt(restRequest.param("z")); + final int x = Integer.parseInt(restRequest.param("x")); + final int y = Integer.parseInt(restRequest.param("y")); + + QueryBuilder queryBuilder = null; + if (restRequest.hasContent()) { + try (XContentParser parser = restRequest.contentParser()) { + XContentParser.Token token = parser.nextToken(); + if (token != XContentParser.Token.START_OBJECT) { + throw new IllegalArgumentException("Invalid content format"); + } + String currentFieldName; + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + if (currentFieldName.equals("query")) { + queryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser); + } else { + throw new IllegalArgumentException("Unsupported field " + currentFieldName); + } + } else { + throw new IllegalArgumentException("Invalid content format"); + } + } + } + } + + final SearchRequestBuilder builder = searchBuilder(client, index.split(","), field, z, x, y, queryBuilder); + // TODO: how do we handle cancellations? + return channel -> builder.execute( new RestResponseListener<>(channel) { + + @Override + public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { + final InternalVectorTile t = searchResponse.getAggregations().get(field); + final BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput()); + t.writeTileToStream(bytesOut); + return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); + } + }); + } + + public static SearchRequestBuilder searchBuilder(Client client, String[] index, String field, int z, int x, int y, + QueryBuilder queryBuilder) throws IOException { + final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); + if (queryBuilder != null) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(queryBuilder); + boolQueryBuilder.filter(qBuilder); + qBuilder = boolQueryBuilder; + } + final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(field).field(field).z(z).x(x).y(y); + return client.prepareSearch(index).setQuery(qBuilder).addAggregation(aBuilder).setSize(0); + } + + @Override + public String getName() { + return "vectortile_action"; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java new file mode 100644 index 0000000000000..fbbb37a250e9d --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java @@ -0,0 +1,97 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.vectortile; + + +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.builder.SearchSourceBuilder; + +import java.io.IOException; +import java.util.Map; + +import static java.util.Collections.emptyMap; + + +public class VectorTileAggConfig { + + private static final ParseField SCALING = new ParseField("scaling"); + + public static final ObjectParser PARSER = + new ObjectParser<>("gridmvt_config", VectorTileAggConfig::new); + + static { + PARSER.declareField( + VectorTileAggConfig::setRuntimeField, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileAggConfig::setQueryBuilder, + (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, + SearchSourceBuilder.QUERY_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileAggConfig::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareInt(VectorTileAggConfig::setScaling, SCALING); + } + + public static VectorTileAggConfig getInstance() { + return new VectorTileAggConfig(); + } + + private QueryBuilder queryBuilder; + private AggregatorFactories.Builder aggBuilder; + private int scaling = 8; + private Map runtimeMappings = emptyMap(); + + private VectorTileAggConfig() { + } + + private void setQueryBuilder(QueryBuilder queryBuilder) { + this.queryBuilder = queryBuilder; + } + + public QueryBuilder getQueryBuilder() { + return queryBuilder; + } + + private void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + this.aggBuilder = aggBuilder; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + private void setScaling(int scaling) { + this.scaling = scaling; + } + + public int getScaling() { + return scaling; + } + + private void setRuntimeField(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public Map getRuntimeMappings() { + return runtimeMappings; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java new file mode 100644 index 0000000000000..38df698ca2f38 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java @@ -0,0 +1,67 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.vectortile; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; +import com.wdtinc.mapbox_vector_tile.encoding.GeomCmdHdr; +import org.apache.lucene.util.BitUtil; +import org.elasticsearch.geometry.Rectangle; + +class VectorTileGeometryBuilder { + + private final int extent; + private final Rectangle rectangle; + private final double pointXScale, pointYScale; + + VectorTileGeometryBuilder(int z, int x, int y, int extent) { + this.extent = extent; + rectangle = VectorTileUtils.getTileBounds(z , x, y); + pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) extent); + pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); + } + + public int lat(double lat) { + return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; + } + + public int lon(double lon) { + return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + } + + public void point(VectorTile.Tile.Feature.Builder featureBuilder, double lon, double lat) { + featureBuilder.setType(VectorTile.Tile.GeomType.POINT); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(lon(lon))); + featureBuilder.addGeometry(BitUtil.zigZagEncode(lat(lat))); + } + + public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, double maxLon, double minLat, double maxLat) { + featureBuilder.setType(VectorTile.Tile.GeomType.POLYGON); + final int minX = lon(minLon); + final int minY = lat(minLat); + final int maxX = lon(maxLon); + final int maxY = lat(maxLat); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(minX)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(minY)); + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.LineTo, 3)); + // 1 + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxY - minY)); + // 2 + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxX - minX)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + // 3 + featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(minY - maxY)); + // close + featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.ClosePath, 1)); + } + +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java new file mode 100644 index 0000000000000..361328b2f48d9 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.vectortile; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.elasticsearch.geometry.Rectangle; +import org.locationtech.jts.geom.Envelope; + +/** + * Utility methods For vector tiles. Transforms WGS84 into spherical mercator. + */ +public class VectorTileUtils { + + public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName, int extent) { + final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); + layerBuilder.setVersion(2); + layerBuilder.setName(layerName); + layerBuilder.setExtent(extent); + return layerBuilder; + } + + /** + * Gets the JTS envelope for z/x/y/ tile in spherical mercator projection. + */ + public static Envelope getJTSTileBounds(int z, int x, int y) { + return new Envelope(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + /** + * Gets the {@link org.elasticsearch.geometry.Geometry} envelope for z/x/y/ tile + * in spherical mercator projection. + */ + public static Rectangle getTileBounds(int z, int x, int y) { + return new Rectangle(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + private static double getLong(int x, int zoom) + { + return lonToSphericalMercator( x / Math.pow(2, zoom) * 360 - 180 ); + } + + private static double getLat(int y, int zoom) { + double r2d = 180 / Math.PI; + double n = Math.PI - 2 * Math.PI * y / Math.pow(2, zoom); + return latToSphericalMercator(r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))); + } + + private static double MERCATOR_FACTOR = 20037508.34 / 180.0; + + /** + * Transforms WGS84 longitude to a Spherical mercator longitude + */ + public static double lonToSphericalMercator(double lon) { + return lon * MERCATOR_FACTOR; + } + + /** + * Transforms WGS84 latitude to a Spherical mercator latitude + */ + public static double latToSphericalMercator(double lat) { + double y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180); + return y * MERCATOR_FACTOR; + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java new file mode 100644 index 0000000000000..18f297402e1dc --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java @@ -0,0 +1,182 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.util.CollectionUtils; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.ParsedAggregation; +import org.elasticsearch.test.InternalAggregationTestCase; +import org.elasticsearch.xpack.spatial.SpatialPlugin; +import org.elasticsearch.xpack.spatial.vectortile.FeatureFactory; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +public class InternalVectorTileTests extends InternalAggregationTestCase { + + @Override + protected SearchPlugin registerPlugin() { + return new SpatialPlugin(); + } + + static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { + final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); + layerBuilder.setVersion(2); + layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); + layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); + final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + for (int i =0; i < shapes; i++) { + int count = layerBuilder.getFeaturesCount(); + while(true) { + Geometry geometry = GeometryTestUtils.randomPolygon(false); + List features = factory.getFeatures(geometry); + for (VectorTile.Tile.Feature feature : features) { + layerBuilder.addFeatures(feature); + } + if (count < layerBuilder.getFeaturesCount()) { + break; + } + } + } + return layerBuilder.build(); + + } + + static VectorTile.Tile.Layer randomLineLayer(int shapes) { + final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); + layerBuilder.setVersion(2); + layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); + layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); + final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + for (int i =0; i < shapes; i++) { + int count = layerBuilder.getFeaturesCount(); + while(true) { + Geometry geometry = GeometryTestUtils.randomLine(false); + List features = factory.getFeatures(geometry); + for (VectorTile.Tile.Feature feature : features) { + layerBuilder.addFeatures(feature); + } + if (count < layerBuilder.getFeaturesCount()) { + break; + } + } + } + return layerBuilder.build(); + + } + + static long[] randomPoints() { + long[] points = new long[AbstractVectorTileAggregator.POINT_EXTENT * AbstractVectorTileAggregator.POINT_EXTENT]; + for( int i = 0; i < points.length; i++) { + points[i] = randomInt(10); + } + return points; + } + + @Override + protected InternalVectorTile createTestInstance(String name, Map metadata) { + int size = randomIntBetween(5, 10); + VectorTile.Tile.Layer polygons = randomBoolean() ? randomPolygonLayer(size) : null; + VectorTile.Tile.Layer lines = randomBoolean() ? randomLineLayer(size) : null; + long[] points = randomBoolean() ? randomPoints() : null; + return new InternalVectorTile(name, polygons, lines, points, metadata); + } + + @Override + protected InternalVectorTile mutateInstance(InternalVectorTile instance) { + String name = instance.getName(); + VectorTile.Tile.Layer polygons = instance.polygons; + VectorTile.Tile.Layer lines = instance.lines; + long[] points = instance.points; + Map metadata = instance.getMetadata(); + switch (randomIntBetween(0, 4)) { + case 0: + name += randomAlphaOfLength(5); + break; + case 1: + if (metadata == null) { + metadata = new HashMap<>(1); + } else { + metadata = new HashMap<>(instance.getMetadata()); + } + metadata.put(randomAlphaOfLength(15), randomInt()); + break; + case 2: + polygons = randomPolygonLayer(randomIntBetween(10, 20)); + break; + case 3: + lines = randomLineLayer(randomIntBetween(10, 20)); + break; + case 4: + points = randomPoints(); + break; + default: + throw new AssertionError("Illegal randomisation branch"); + } + return new InternalVectorTile(name, polygons, lines, points, metadata); + } + + @Override + protected List randomResultsToReduce(String name, int size) { + List instances = new ArrayList<>(size); + for (int i = 0; i < size; i++) { + instances.add(createTestInstance(name, null)); + } + return instances; + } + + @Override + protected void assertReduced(InternalVectorTile reduced, List inputs) { + int numPolygons = 0; + int numLines = 0; + long numPoints = 0; + for (InternalVectorTile input : inputs) { + if (input.polygons != null) { + numPolygons += input.polygons.getFeaturesCount(); + } + if (input.lines != null) { + numLines += input.lines.getFeaturesCount(); + } + if (input.points != null) { + numPoints += Arrays.stream(input.points).sum(); + } + } + int numReducedPolygons = reduced.polygons != null ? reduced.polygons.getFeaturesCount() : 0; + assertThat(numReducedPolygons, Matchers.equalTo(numPolygons)); + int numReducedLines = reduced.lines != null ? reduced.lines.getFeaturesCount() : 0; + assertThat(numReducedLines, Matchers.equalTo(numLines)); + long numReducedPoints = reduced.points != null ? Arrays.stream(reduced.points).sum() : 0; + assertThat(numReducedPoints, Matchers.equalTo(numPoints)); + } + + @Override + protected void assertFromXContent(InternalVectorTile aggregation, ParsedAggregation parsedAggregation) throws IOException { + // The output is binary so nothing to do here + } + + @Override + protected List getNamedXContents() { + return CollectionUtils.appendToCopy(super.getNamedXContents(), new NamedXContentRegistry.Entry(Aggregation.class, + new ParseField(VectorTileAggregationBuilder.NAME), + (p, c) -> { + assumeTrue("There is no ParsedVectorTile yet", false); + return null; + } + )); + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java new file mode 100644 index 0000000000000..b608ef623c8fa --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java @@ -0,0 +1,45 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.search.aggregations; + +import org.elasticsearch.common.io.stream.Writeable; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.test.AbstractSerializingTestCase; + +import java.io.IOException; + +import static org.hamcrest.Matchers.equalTo; + +public class VectorTileAggregationBuilderTests extends AbstractSerializingTestCase { + + @Override + protected VectorTileAggregationBuilder doParseInstance(XContentParser parser) throws IOException { + assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + String name = parser.currentName(); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); + assertThat(parser.currentName(), equalTo(VectorTileAggregationBuilder.NAME)); + VectorTileAggregationBuilder parsed = VectorTileAggregationBuilder.PARSER.apply(parser, name); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); + assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); + return parsed; + } + + @Override + protected Writeable.Reader instanceReader() { + return VectorTileAggregationBuilder::new; + } + + @Override + protected VectorTileAggregationBuilder createTestInstance() { + VectorTileAggregationBuilder vectorTileAggregationBuilder = new VectorTileAggregationBuilder("_name"); + vectorTileAggregationBuilder.z(0).x(0).y(0); + return vectorTileAggregationBuilder; + } +} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java new file mode 100644 index 0000000000000..ca6ab460eac5a --- /dev/null +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java @@ -0,0 +1,239 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.spatial.search.aggregations; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.apache.lucene.document.Document; +import org.apache.lucene.document.LatLonDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.index.IndexReader; +import org.apache.lucene.index.RandomIndexWriter; +import org.apache.lucene.search.IndexSearcher; +import org.apache.lucene.search.MatchAllDocsQuery; +import org.apache.lucene.store.Directory; +import org.apache.lucene.util.BytesRef; +import org.elasticsearch.common.geo.builders.ShapeBuilder; +import org.elasticsearch.geo.GeometryTestUtils; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.utils.WellKnownText; +import org.elasticsearch.index.mapper.GeoPointFieldMapper; +import org.elasticsearch.index.mapper.GeoShapeIndexer; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.SourceFieldMapper; +import org.elasticsearch.plugins.SearchPlugin; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorTestCase; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; +import org.elasticsearch.search.aggregations.support.ValuesSourceType; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; +import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; +import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; +import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; +import org.elasticsearch.xpack.spatial.util.GeoTestUtils; +import org.hamcrest.Matchers; +import org.locationtech.spatial4j.exception.InvalidShapeException; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.function.Function; + +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.greaterThanOrEqualTo; + + +public class VectorTileAggregatorTests extends AggregatorTestCase { + + @Override + protected List getSearchPlugins() { + return List.of(new LocalStateSpatialPlugin()); + } + + public void testGeoPoint() throws Exception { + int numDocs = scaledRandomIntBetween(64, 256); + List points = new ArrayList<>(); + for (int i = 0; i < numDocs; i++) { + points.add(randomValueOtherThanMany((p) -> p.getLat() > 80 || p.getLat() < -80, GeometryTestUtils::randomPoint)); + } + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + for (Point point : points) { + Document document = new Document(); + document.add(new LatLonDocValuesField("field", point.getLat(), point.getLon())); + BytesRef val = new BytesRef("{\"field\" : \"" + WellKnownText.INSTANCE.toWKT(point) + "\"}"); + document.add(new StoredField("_source", val)); + w.addDocument(document); + } + if (randomBoolean()) { + w.forceMerge(1); + } + assertVectorTile(w, 0, 0, numDocs, new GeoPointFieldMapper.GeoPointFieldType("field")); + } + } + + @SuppressWarnings("unchecked") + @AwaitsFix(bugUrl = "That doesn't work yet") + public void testGeoShape() throws Exception { + int numDocs = scaledRandomIntBetween(64, 256); + List geometries = new ArrayList<>(); + GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); + for (int i = 0; i < numDocs; i++) { + Function geometryGenerator = ESTestCase.randomFrom( + GeometryTestUtils::randomLine, + GeometryTestUtils::randomPoint, + GeometryTestUtils::randomPolygon, + GeometryTestUtils::randomMultiLine, + GeometryTestUtils::randomMultiPolygon + ); + Geometry geometry = geometryGenerator.apply(false); + try { + geometries.add(indexer.prepareForIndexing(geometry)); + } catch (InvalidShapeException e) { + // do not include geometry + } + } + int expectedPolygons = 0; + int expectedLines = 0; + int expectedPoints = 0; + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { + for (Geometry geometry : geometries) { + Document document = new Document(); + document.add(GeoTestUtils.binaryGeoShapeDocValuesField("field", geometry)); + BytesRef val = new BytesRef("{\"field\" : \"" + WellKnownText.INSTANCE.toWKT(geometry) + "\"}"); + document.add(new StoredField("_source", val)); + w.addDocument(document); + GeoShapeValues.GeoShapeValue value = GeoTestUtils.geoShapeValue(geometry); + switch (value.dimensionalShapeType()) { + case POINT: + if (value.lat() < GeoTileUtils.LATITUDE_MASK && value.lat() > -GeoTileUtils.LATITUDE_MASK) { + expectedPoints++; + } + break; + case LINE: + expectedLines++; + break; + case POLYGON: + expectedPolygons++; + break; + } + } + if (randomBoolean()) { + w.forceMerge(1); + } + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", + true, true, ShapeBuilder.Orientation.RIGHT, null, Collections.emptyMap()); + assertVectorTile(w, expectedPolygons, expectedLines, expectedPoints, fieldType); + } + } + + private void assertVectorTile(RandomIndexWriter w, int expectedPolygons, int expectedLines, + int expectedPoints, MappedFieldType fieldType) throws IOException { + MappedFieldType sourceFieldType = new SourceFieldMapper.Builder().build().fieldType(); + VectorTileAggregationBuilder aggBuilder = new VectorTileAggregationBuilder("my_agg") + .field("field"); + try (IndexReader reader = w.getReader()) { + IndexSearcher searcher = new IndexSearcher(reader); + InternalVectorTile result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType, sourceFieldType); + assertEquals("my_agg", result.getName()); + VectorTile.Tile tileResult = VectorTile.Tile.newBuilder().mergeFrom(result.getVectorTileBytes()).build(); + assertNotNull(tileResult); + int expectedLayers = (expectedPolygons > 0 ? 1 : 0) + (expectedLines > 0 ? 1 : 0) + (expectedPoints > 0 ? 1 : 0); + assertThat(expectedLayers, Matchers.equalTo(tileResult.getLayersCount())); + if (expectedPolygons > 0) { + VectorTile.Tile.Layer polygons = null; + for (int i = 0; i < tileResult.getLayersCount(); i++) { + VectorTile.Tile.Layer layer = tileResult.getLayers(i); + if (AbstractVectorTileAggregator.POLYGON_LAYER.equals(layer.getName())) { + polygons = layer; + break; + } + } + assertThat(polygons, Matchers.notNullValue()); + assertThat(AbstractVectorTileAggregator.POLYGON_EXTENT, Matchers.equalTo(polygons.getExtent())); + assertThat(2, Matchers.equalTo(polygons.getVersion())); + assertThat(expectedPolygons, Matchers.greaterThanOrEqualTo(polygons.getFeaturesCount())); + } + + if (expectedLines > 0) { + VectorTile.Tile.Layer lines = null; + for (int i = 0; i < tileResult.getLayersCount(); i++) { + VectorTile.Tile.Layer layer = tileResult.getLayers(i); + if (AbstractVectorTileAggregator.LINE_LAYER.equals(layer.getName())) { + lines = layer; + break; + } + } + assertThat(lines, Matchers.notNullValue()); + assertThat(AbstractVectorTileAggregator.LINE_EXTENT, Matchers.equalTo(lines.getExtent())); + assertThat(2, Matchers.equalTo(lines.getVersion())); + assertThat(expectedLines, Matchers.greaterThanOrEqualTo(lines.getFeaturesCount())); + } + + if (expectedPoints > 0) { + VectorTile.Tile.Layer points = null; + for (int i = 0; i < tileResult.getLayersCount(); i++) { + VectorTile.Tile.Layer layer = tileResult.getLayers(i); + if (AbstractVectorTileAggregator.POINT_LAYER.equals(layer.getName())) { + points = layer; + break; + } + } + assertThat(points, Matchers.notNullValue()); + assertThat(AbstractVectorTileAggregator.POINT_EXTENT, Matchers.equalTo(points.getExtent())); + assertThat(2, Matchers.equalTo(points.getVersion())); + int tagIndex = -1; + for (int i = 0; i < points.getKeysCount(); i++) { + if ("count".equals(points.getKeys(i))) { + tagIndex = i; + break; + } + } + assertThat(tagIndex, greaterThanOrEqualTo(0)); + int numberPoints = 0; + for (int i = 0; i < points.getFeaturesCount(); i++) { + VectorTile.Tile.Feature feature = points.getFeatures(i); + numberPoints += points.getValues(feature.getTags(tagIndex + 1)).getIntValue(); + } + assertThat(expectedPoints, Matchers.equalTo(numberPoints)); + } + } + } + + public void testInvalidChildAggregation() throws Exception { + try (Directory dir = newDirectory(); + RandomIndexWriter w = new RandomIndexWriter(random(), dir); + IndexReader reader = w.getReader()) { + VectorTileAggregationBuilder aggBuilder = new VectorTileAggregationBuilder("my_agg") + .field("field"); + GeoGridAggregationBuilder parent = new GeoTileGridAggregationBuilder("my_parent") + .field("field").subAggregation(aggBuilder); + MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", + true, true, ShapeBuilder.Orientation.RIGHT, null, Collections.emptyMap()); + IndexSearcher searcher = new IndexSearcher(reader); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () + -> searchAndReduce(searcher, new MatchAllDocsQuery(), parent, fieldType)); + assertThat(e.getMessage(), equalTo("vector-tile aggregation must be at the top level")); + } + } + + @Override + protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { + return new VectorTileAggregationBuilder("foo").field(fieldName).y(0).x(0).z(0); + } + + @Override + protected List getSupportedValuesSourceTypes() { + return List.of(CoreValuesSourceType.GEOPOINT, GeoShapeValuesSourceType.instance()); + } +} diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java new file mode 100644 index 0000000000000..d096b7e60ae1c --- /dev/null +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java @@ -0,0 +1,279 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; + +public class VectorTileGridRestIT extends ESRestTestCase { + + private static String INDEX_POINTS = "index-points"; + private static String INDEX_SHAPES = "index-shapes"; + + private int x, y, z; + + @Before + public void indexDocuments() throws IOException { + + z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); + x = randomIntBetween(0, (1 << z) - 1); + y = randomIntBetween(0, (1 << z) - 1); + indexPoints(); + indexShapes(); + } + + private void indexPoints() throws IOException { + + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); + mappingRequest.setJsonEntity("{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_point\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}"); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + double x = (r.getMaxX() + r.getMinX()) / 2; + double y = (r.getMaxY() + r.getMinY()) / 2; + for (int i = 0; i < 30; i+=10) { + for (int j = 0; j <= i; j++) { + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + + ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + + "}"); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + } + } + + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + private void indexShapes() throws IOException { + + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); + mappingRequest.setJsonEntity("{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_shape\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}"); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + + ", \"name\": \"rectangle\"" + + ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + + "}"); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + + + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + @After + public void deleteData() throws IOException { + final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); + Response response = client().performRequest(deleteRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + public void testBasicGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + } + + public void testEmpty() throws Exception { + final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + newY); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(0)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + } + + public void testBasicScaling() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"scaling\": 7 }"); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(1)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(128)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(128)); + } + } + + private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { + for (int i = 0; i < tile.getLayersCount(); i++) { + VectorTile.Tile.Layer layer = tile.getLayers(i); + if (layerName.equals(layer.getName())) { + return layer; + } + } + fail("Could not find layer " + layerName); + return null; + } + + @AwaitsFix(bugUrl = "doesn't work yet") + public void testBasicQueryGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\n" + + " \"query\": {\n" + + " \"term\": {\n" + + " \"name\": {\n" + + " \"value\": \"point2\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + Response response = client().performRequest(mvtRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + InputStream inputStream = response.getEntity().getContent(); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(1)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + } + + public void testBasicShape() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + } + + public void testMinAgg() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\n" + + " \"aggs\": {\n" + + " \"minVal\": {\n" + + " \"min\": {\n" + + " \"field\": \"value1\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + { + VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); + assertThat(layer.getValuesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); + } + { + VectorTile.Tile.Layer layer = getLayer(tile, "META"); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); + assertThat(layer.getExtent(), Matchers.equalTo(256)); + } + } +} diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java new file mode 100644 index 0000000000000..6e4af5d93f3e5 --- /dev/null +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -0,0 +1,105 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import org.apache.http.HttpStatus; +import org.apache.http.client.methods.HttpDelete; +import org.apache.http.client.methods.HttpGet; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.client.methods.HttpPut; +import org.elasticsearch.client.Request; +import org.elasticsearch.client.Response; +import org.elasticsearch.test.rest.ESRestTestCase; +import org.hamcrest.Matchers; +import org.junit.After; +import org.junit.Before; + +import java.io.IOException; +import java.io.InputStream; + +public class VectorTileRestIT extends ESRestTestCase { + + private static String INDEX_NAME = "my-index"; + + @Before + public void indexDocuments() throws IOException { + + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME + "/_mapping"); + mappingRequest.setJsonEntity("{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_point\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}"); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + + final Request putRequest1 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); + putRequest1.setJsonEntity("{\n" + + " \"location\": \"POINT(0 0)\", \"name\": \"point1\"\n" + + "}"); + + response = client().performRequest(putRequest1); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + + final Request putRequest2 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); + putRequest2.setJsonEntity("{\n" + + " \"location\": \"POINT(1 1)\", \"name\": \"point2\"\n" + + "}"); + + response = client().performRequest(putRequest2); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + @After + public void deleteData() throws IOException { + final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_NAME); + Response response = client().performRequest(deleteRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } + + public void testBasicGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); + Response response = client().performRequest(mvtRequest); + InputStream inputStream = response.getEntity().getContent(); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); + assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(2)); + } + + public void testBasicQueryGet() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); + mvtRequest.setJsonEntity("{\n" + + " \"query\": {\n" + + " \"term\": {\n" + + " \"name\": {\n" + + " \"value\": \"point2\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + Response response = client().performRequest(mvtRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + InputStream inputStream = response.getEntity().getContent(); + VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); + assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(1)); + } +} From bf2b209e3d77766cb70822c4c8f0ab1da786eb2b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 31 Mar 2021 12:43:44 -0400 Subject: [PATCH 02/15] Vector Tiles: Start refactoring rest layer (#71080) This is the first iteration that just extracts most obvious shared parts of vector tile and aggregated vector tile. Besides removing code repetition this common basis will simplify dealing with async and other aspects of search api later on. --- .../AbstractVectorTileSearchAction.java | 71 +++++++ .../RestAggregatedVectorTileAction.java | 188 +++++++++--------- .../vectortile/RestVectorTileAction.java | 54 ++--- .../vectortile/VectorTileGeometryBuilder.java | 2 +- 4 files changed, 180 insertions(+), 135 deletions(-) create mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java new file mode 100644 index 0000000000000..bba6deb280d72 --- /dev/null +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java @@ -0,0 +1,71 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.spatial.vectortile; + +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestResponseListener; + +import java.io.IOException; +import java.io.OutputStream; + +/** + * Base class for rest actions that performs a search and translates it into + * a protobuf response + */ +public abstract class AbstractVectorTileSearchAction extends BaseRestHandler { + + private static final String INDEX_PARAM = "index"; + private static final String FIELD_PARAM = "field"; + private static final String Z_PARAM = "z"; + private static final String X_PARAM = "x"; + private static final String Y_PARAM = "y"; + + @FunctionalInterface + protected interface ResponseBuilder { + void buildResponse(SearchResponse searchResponse, OutputStream outputStream) throws IOException; + } + + protected abstract ResponseBuilder doParseRequest( + RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException; + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + final String index = restRequest.param(INDEX_PARAM); + final String field = restRequest.param(FIELD_PARAM); + final int z = Integer.parseInt(restRequest.param(Z_PARAM)); + final int x = Integer.parseInt(restRequest.param(X_PARAM)); + final int y = Integer.parseInt(restRequest.param(Y_PARAM)); + + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(index)); + + ResponseBuilder responseBuilder = doParseRequest(restRequest, field, z, x, y, searchRequestBuilder); + + // TODO: how do we handle cancellations? + return channel -> searchRequestBuilder.execute(new RestResponseListener<>(channel) { + + @Override + public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { + try (BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput())) { + responseBuilder.buildResponse(searchResponse, bytesOut); + return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); + } + } + }); + } + +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java index 47f1646f2b5ec..59ec55313ebcf 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java @@ -8,24 +8,14 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.BytesStream; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.RestResponseListener; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -40,13 +30,8 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestAggregatedVectorTileAction extends BaseRestHandler { +public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchAction { - private static final String INDEX_PARAM = "index"; - private static final String FIELD_PARAM = "field"; - private static final String Z_PARAM = "z"; - private static final String X_PARAM = "x"; - private static final String Y_PARAM = "y"; private static final String TYPE_PARAM = "type"; private static final String GRID_TYPE = "grid"; @@ -59,90 +44,91 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - final String index = restRequest.param(INDEX_PARAM); - final String field = restRequest.param(FIELD_PARAM); - final int z = Integer.parseInt(restRequest.param(Z_PARAM)); - final int x = Integer.parseInt(restRequest.param(X_PARAM)); - final int y = Integer.parseInt(restRequest.param(Y_PARAM)); + protected ResponseBuilder doParseRequest( + RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException { final boolean isGrid = restRequest.hasParam(TYPE_PARAM) && GRID_TYPE.equals(restRequest.param(TYPE_PARAM)); final VectorTileAggConfig config = resolveConfig(restRequest); - final SearchRequestBuilder builder = searchBuilder(client, index.split(","), field, z, x, y, config); - final int extent = 1 << config.getScaling(); - // TODO: how do we handle cancellations? - return channel -> builder.execute( new RestResponseListener<>(channel) { - - @Override - public RestResponse buildResponse(SearchResponse searchResponse) throws IOException { - // TODO: of there is no hits, should we return an empty tile with no layers or - // a tile with empty layers? - final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder(z, x, y, extent); - final InternalGeoTileGrid grid = searchResponse.getAggregations().get(GRID_FIELD); - tileBuilder.addLayers(getPointLayer(grid, geomBuilder)); - final InternalGeoBounds bounds = searchResponse.getAggregations().get(BOUNDS_FIELD); - tileBuilder.addLayers(getMetaLayer(bounds, geomBuilder)); - final BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput()); - tileBuilder.build().writeTo(bytesOut); - return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); - } + searchBuilder(searchRequestBuilder, field, z, x, y, config); + final int extent = 1 << config.getScaling(); + + return (s, b) -> { + // TODO: of there is no hits, should we return an empty tile with no layers or + // a tile with empty layers? + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder(z, x, y, extent); + final InternalGeoTileGrid grid = s.getAggregations().get(GRID_FIELD); + tileBuilder.addLayers(getPointLayer(extent, isGrid, grid, geomBuilder)); + final InternalGeoBounds bounds = s.getAggregations().get(BOUNDS_FIELD); + tileBuilder.addLayers(getMetaLayer(extent, bounds, geomBuilder)); + tileBuilder.build().writeTo(b); + }; + } - private VectorTile.Tile.Layer.Builder getPointLayer(InternalGeoTileGrid t, VectorTileGeometryBuilder geomBuilder) { - final VectorTile.Tile.Layer.Builder pointLayerBuilder = VectorTileUtils.createLayerBuilder("AGG", extent); - pointLayerBuilder.addKeys("count"); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); - final HashMap values = new HashMap<>(); - - for (InternalGeoGridBucket bucket : t.getBuckets()) { - long count = bucket.getDocCount(); - if (count > 0) { - featureBuilder.clear(); - // create geometry commands - if (isGrid) { - Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); - geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - } else { - GeoPoint point = (GeoPoint) bucket.getKey(); - geomBuilder.point(featureBuilder, point.lon(), point.lat()); - } - // Add count as key value pair - featureBuilder.addTags(0); - final int tagValue; - if (values.containsKey(count)) { - tagValue = values.get(count); - } else { - valueBuilder.clear(); - valueBuilder.setIntValue(count); - tagValue = values.size(); - pointLayerBuilder.addValues(valueBuilder); - values.put(count, tagValue); - } - featureBuilder.addTags(tagValue); - pointLayerBuilder.addFeatures(featureBuilder); - } + private VectorTile.Tile.Layer.Builder getPointLayer( + int extent, + boolean isGrid, + InternalGeoTileGrid t, + VectorTileGeometryBuilder geomBuilder + ) { + final VectorTile.Tile.Layer.Builder pointLayerBuilder = VectorTileUtils.createLayerBuilder("AGG", extent); + pointLayerBuilder.addKeys("count"); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); + final HashMap values = new HashMap<>(); + + for (InternalGeoGridBucket bucket : t.getBuckets()) { + long count = bucket.getDocCount(); + if (count > 0) { + featureBuilder.clear(); + // create geometry commands + if (isGrid) { + Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); } - return pointLayerBuilder; - } - - private VectorTile.Tile.Layer.Builder getMetaLayer(InternalGeoBounds t, VectorTileGeometryBuilder geomBuilder) { - final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder("META", extent); - final GeoPoint topLeft = t.topLeft(); - final GeoPoint bottomRight = t.bottomRight(); - if (topLeft != null && bottomRight != null) { - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); - metaLayerBuilder.addFeatures(featureBuilder); + // Add count as key value pair + featureBuilder.addTags(0); + final int tagValue; + if (values.containsKey(count)) { + tagValue = values.get(count); + } else { + valueBuilder.clear(); + valueBuilder.setIntValue(count); + tagValue = values.size(); + pointLayerBuilder.addValues(valueBuilder); + values.put(count, tagValue); } - return metaLayerBuilder; + featureBuilder.addTags(tagValue); + pointLayerBuilder.addFeatures(featureBuilder); } - }); + } + return pointLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getMetaLayer(int extent, InternalGeoBounds t, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder("META", extent); + final GeoPoint topLeft = t.topLeft(); + final GeoPoint bottomRight = t.bottomRight(); + if (topLeft != null && bottomRight != null) { + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + metaLayerBuilder.addFeatures(featureBuilder); + } + return metaLayerBuilder; } - public static SearchRequestBuilder searchBuilder(Client client, String[] index, String field, int z, int x, int y, - VectorTileAggConfig config) throws IOException { - final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + private static SearchRequestBuilder searchBuilder( + SearchRequestBuilder searchRequestBuilder, + String field, + int z, + int x, + int y, + VectorTileAggConfig config + ) throws IOException { + Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); if (config.getQueryBuilder() != null) { BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); @@ -151,18 +137,22 @@ public static SearchRequestBuilder searchBuilder(Client client, String[] index, qBuilder = boolQueryBuilder; } int extent = 1 << config.getScaling(); - GeoBoundingBox boundingBox = - new GeoBoundingBox(new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), - new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon())); - GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD) - .field(field).precision(Math.min(GeoTileUtils.MAX_ZOOM, z + config.getScaling())) - .setGeoBoundingBox(boundingBox).size(extent * extent); + GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(field) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, z + config.getScaling())) + .setGeoBoundingBox(boundingBox) + .size(extent * extent); if (config.getAggBuilder() != null) { aBuilder.subAggregations(config.getAggBuilder()); } GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(field).wrapLongitude(false); - SearchRequestBuilder requestBuilder = client.prepareSearch(index).setQuery(qBuilder) - .addAggregation(aBuilder).addAggregation(boundsBuilder).setSize(0); + SearchRequestBuilder requestBuilder = searchRequestBuilder.setQuery(qBuilder) + .addAggregation(aBuilder) + .addAggregation(boundsBuilder) + .setSize(0); if (config.getRuntimeMappings() != null) { requestBuilder.setRuntimeMappings(config.getRuntimeMappings()); } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index 4222536a365ca..e0d883fab5b26 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -6,25 +6,14 @@ */ package org.elasticsearch.xpack.spatial.vectortile; - import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.Client; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.BytesStream; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.BoolQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.RestResponseListener; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; @@ -34,7 +23,7 @@ import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestVectorTileAction extends BaseRestHandler { +public class RestVectorTileAction extends AbstractVectorTileSearchAction { @Override public List routes() { @@ -42,13 +31,8 @@ public List routes() { } @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - final String index = restRequest.param("index"); - final String field = restRequest.param("field"); - final int z = Integer.parseInt(restRequest.param("z")); - final int x = Integer.parseInt(restRequest.param("x")); - final int y = Integer.parseInt(restRequest.param("y")); - + protected ResponseBuilder doParseRequest( + RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException { QueryBuilder queryBuilder = null; if (restRequest.hasContent()) { try (XContentParser parser = restRequest.contentParser()) { @@ -66,28 +50,28 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient throw new IllegalArgumentException("Unsupported field " + currentFieldName); } } else { - throw new IllegalArgumentException("Invalid content format"); + throw new IllegalArgumentException("Invalid content format"); } } } } - final SearchRequestBuilder builder = searchBuilder(client, index.split(","), field, z, x, y, queryBuilder); - // TODO: how do we handle cancellations? - return channel -> builder.execute( new RestResponseListener<>(channel) { - - @Override - public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { - final InternalVectorTile t = searchResponse.getAggregations().get(field); - final BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput()); - t.writeTileToStream(bytesOut); - return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); - } - }); + searchBuilder(searchRequestBuilder, field, z, x, y, queryBuilder); + return (s, b) -> { + InternalVectorTile t = s.getAggregations().get(field); + // TODO: Error processing + t.writeTileToStream(b); + }; } - public static SearchRequestBuilder searchBuilder(Client client, String[] index, String field, int z, int x, int y, - QueryBuilder queryBuilder) throws IOException { + private static void searchBuilder( + SearchRequestBuilder searchRequestBuilder, + String field, + int z, + int x, + int y, + QueryBuilder queryBuilder + ) throws IOException { final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); if (queryBuilder != null) { @@ -97,7 +81,7 @@ public static SearchRequestBuilder searchBuilder(Client client, String[] index, qBuilder = boolQueryBuilder; } final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(field).field(field).z(z).x(x).y(y); - return client.prepareSearch(index).setQuery(qBuilder).addAggregation(aBuilder).setSize(0); + searchRequestBuilder.setQuery(qBuilder).addAggregation(aBuilder).setSize(0); } @Override diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java index 38df698ca2f38..ea1fcc5a7b01f 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java @@ -21,7 +21,7 @@ class VectorTileGeometryBuilder { VectorTileGeometryBuilder(int z, int x, int y, int extent) { this.extent = extent; - rectangle = VectorTileUtils.getTileBounds(z , x, y); + rectangle = VectorTileUtils.getTileBounds(z, x, y); pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) extent); pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); } From e3acfc613f7b1bb9b0e4620d17b3f0798587b134 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 6 Apr 2021 18:24:19 -0400 Subject: [PATCH 03/15] Vector Tiles: continue REST layer refactoring (#71259) This change unifies the request parsing and makes it extensible. --- .../AbstractVectorTileSearchAction.java | 129 ++++++++++++++++-- .../RestAggregatedVectorTileAction.java | 129 +++++++++++------- .../vectortile/RestVectorTileAction.java | 70 ++-------- .../vectortile/VectorTileAggConfig.java | 97 ------------- 4 files changed, 213 insertions(+), 212 deletions(-) delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java index bba6deb280d72..dcc39a00c502e 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java @@ -10,24 +10,35 @@ import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.CheckedFunction; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.BaseRestHandler; import org.elasticsearch.rest.BytesRestResponse; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.io.OutputStream; +import java.util.function.Supplier; /** * Base class for rest actions that performs a search and translates it into * a protobuf response */ -public abstract class AbstractVectorTileSearchAction extends BaseRestHandler { +public abstract class AbstractVectorTileSearchAction extends BaseRestHandler { private static final String INDEX_PARAM = "index"; private static final String FIELD_PARAM = "field"; @@ -35,25 +46,121 @@ public abstract class AbstractVectorTileSearchAction extends BaseRestHandler { private static final String X_PARAM = "x"; private static final String Y_PARAM = "y"; + protected final ObjectParser parser; + @FunctionalInterface protected interface ResponseBuilder { void buildResponse(SearchResponse searchResponse, OutputStream outputStream) throws IOException; } - protected abstract ResponseBuilder doParseRequest( - RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException; + private final Supplier emptyRequestProvider; + + protected static class Request { + QueryBuilder queryBuilder; + String index; + String field; + int x; + int y; + int z; + + public String getIndex() { + return index; + } + + public void setIndex(String index) { + this.index = index; + } + + public String getField() { + return field; + } + + public void setField(String field) { + this.field = field; + } + + public int getX() { + return x; + } + + public void setX(int x) { + this.x = x; + } + + public int getY() { + return y; + } + + public void setY(int y) { + this.y = y; + } + + public int getZ() { + return z; + } + + public void setZ(int z) { + this.z = z; + } + + public QueryBuilder getQueryBuilder() { + return queryBuilder; + } + + public void setQueryBuilder(QueryBuilder queryBuilder) { + this.queryBuilder = queryBuilder; + } + + public Rectangle getBoundingBox() { + return GeoTileUtils.toBoundingBox(x, y, z); + } + + public QueryBuilder getQuery() throws IOException { + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, getBoundingBox()); + if (queryBuilder != null) { + BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(queryBuilder); + boolQueryBuilder.filter(qBuilder); + qBuilder = boolQueryBuilder; + } + return qBuilder; + } + } + + protected AbstractVectorTileSearchAction(Supplier emptyRequestProvider) { + this.emptyRequestProvider = emptyRequestProvider; + parser = new ObjectParser<>(getName(), emptyRequestProvider); + parser.declareField( + Request::setQueryBuilder, + (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, + SearchSourceBuilder.QUERY_FIELD, + ObjectParser.ValueType.OBJECT + ); + } + + protected abstract ResponseBuilder doParseRequest(RestRequest restRequest, R request, SearchRequestBuilder searchRequestBuilder) + throws IOException; @Override protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - final String index = restRequest.param(INDEX_PARAM); - final String field = restRequest.param(FIELD_PARAM); - final int z = Integer.parseInt(restRequest.param(Z_PARAM)); - final int x = Integer.parseInt(restRequest.param(X_PARAM)); - final int y = Integer.parseInt(restRequest.param(Y_PARAM)); - - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(index)); + final R request; + if (restRequest.hasContent()) { + try (XContentParser contentParser = restRequest.contentParser()) { + request = parser.parse(contentParser, restRequest); + } + } else { + request = emptyRequestProvider.get(); + } + request.setIndex(restRequest.param(INDEX_PARAM)); + request.setField(restRequest.param(FIELD_PARAM)); + request.setZ(Integer.parseInt(restRequest.param(Z_PARAM))); + request.setX(Integer.parseInt(restRequest.param(X_PARAM))); + request.setY(Integer.parseInt(restRequest.param(Y_PARAM))); - ResponseBuilder responseBuilder = doParseRequest(restRequest, field, z, x, y, searchRequestBuilder); + SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(request.getIndex())); + searchRequestBuilder.setQuery(request.getQuery()); + searchRequestBuilder.setSize(0); + ResponseBuilder responseBuilder = doParseRequest(restRequest, request, searchRequestBuilder); // TODO: how do we handle cancellations? return channel -> searchRequestBuilder.execute(new RestResponseListener<>(channel) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java index 59ec55313ebcf..7ea8e86d54f58 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java @@ -8,14 +8,14 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.xcontent.ObjectParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -23,14 +23,17 @@ import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.builder.SearchSourceBuilder; import java.io.IOException; import java.util.HashMap; import java.util.List; +import java.util.Map; +import static java.util.Collections.emptyMap; import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchAction { +public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchAction { private static final String TYPE_PARAM = "type"; private static final String GRID_TYPE = "grid"; @@ -38,25 +41,80 @@ public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchActi private static final String GRID_FIELD = "grid"; private static final String BOUNDS_FIELD = "bounds"; + private static final ParseField SCALING = new ParseField("scaling"); + + public RestAggregatedVectorTileAction() { + super(AggregatedRequest::new); + parser.declareField( + AggregatedRequest::setRuntimeMappings, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + parser.declareField( + AggregatedRequest::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + parser.declareInt(AggregatedRequest::setScaling, SCALING); + } + + protected static class AggregatedRequest extends AbstractVectorTileSearchAction.Request { + private Map runtimeMappings = emptyMap(); + private int scaling = 8; + private AggregatorFactories.Builder aggBuilder; + + public AggregatedRequest() {} + + public Map getRuntimeMappings() { + return runtimeMappings; + } + + public void setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public int getScaling() { + return scaling; + } + + public void setScaling(int scaling) { + this.scaling = scaling; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + this.aggBuilder = aggBuilder; + } + } + @Override public List routes() { return List.of(new Route(GET, "{index}/_agg_mvt/{field}/{z}/{x}/{y}")); } @Override - protected ResponseBuilder doParseRequest( - RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException { + protected ResponseBuilder doParseRequest(RestRequest restRequest, AggregatedRequest request, SearchRequestBuilder searchRequestBuilder) + throws IOException { final boolean isGrid = restRequest.hasParam(TYPE_PARAM) && GRID_TYPE.equals(restRequest.param(TYPE_PARAM)); - final VectorTileAggConfig config = resolveConfig(restRequest); - searchBuilder(searchRequestBuilder, field, z, x, y, config); - final int extent = 1 << config.getScaling(); + searchBuilder(searchRequestBuilder, request); + final int extent = 1 << request.getScaling(); return (s, b) -> { // TODO: of there is no hits, should we return an empty tile with no layers or // a tile with empty layers? final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder(z, x, y, extent); + final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( + request.getZ(), + request.getX(), + request.getY(), + extent + ); final InternalGeoTileGrid grid = s.getAggregations().get(GRID_FIELD); tileBuilder.addLayers(getPointLayer(extent, isGrid, grid, geomBuilder)); final InternalGeoBounds bounds = s.getAggregations().get(BOUNDS_FIELD); @@ -120,55 +178,30 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(int extent, InternalGeoBounds return metaLayerBuilder; } - private static SearchRequestBuilder searchBuilder( - SearchRequestBuilder searchRequestBuilder, - String field, - int z, - int x, - int y, - VectorTileAggConfig config - ) throws IOException { - Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); - QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); - if (config.getQueryBuilder() != null) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(config.getQueryBuilder()); - boolQueryBuilder.filter(qBuilder); - qBuilder = boolQueryBuilder; - } - int extent = 1 << config.getScaling(); + private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, AggregatedRequest request) + throws IOException { + Rectangle rectangle = request.getBoundingBox(); + int extent = 1 << request.getScaling(); GeoBoundingBox boundingBox = new GeoBoundingBox( new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) ); - GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(field) - .precision(Math.min(GeoTileUtils.MAX_ZOOM, z + config.getScaling())) + GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getScaling())) .setGeoBoundingBox(boundingBox) .size(extent * extent); - if (config.getAggBuilder() != null) { - aBuilder.subAggregations(config.getAggBuilder()); + if (request.getAggBuilder() != null) { + aBuilder.subAggregations(request.getAggBuilder()); } - GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(field).wrapLongitude(false); - SearchRequestBuilder requestBuilder = searchRequestBuilder.setQuery(qBuilder) - .addAggregation(aBuilder) - .addAggregation(boundsBuilder) - .setSize(0); - if (config.getRuntimeMappings() != null) { - requestBuilder.setRuntimeMappings(config.getRuntimeMappings()); + GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) + .wrapLongitude(false); + SearchRequestBuilder requestBuilder = searchRequestBuilder.addAggregation(aBuilder).addAggregation(boundsBuilder).setSize(0); + if (request.getRuntimeMappings() != null) { + requestBuilder.setRuntimeMappings(request.getRuntimeMappings()); } return requestBuilder; } - private VectorTileAggConfig resolveConfig(RestRequest restRequest) throws IOException { - if (restRequest.hasContent()) { - try (XContentParser parser = restRequest.contentParser()) { - return VectorTileAggConfig.PARSER.apply(parser, null); - } - } else { - return VectorTileAggConfig.getInstance(); - } - } - @Override public String getName() { return "vectortile_aggregation_action"; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index e0d883fab5b26..bd54609386568 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -7,23 +7,20 @@ package org.elasticsearch.xpack.spatial.vectortile; import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; +import org.elasticsearch.xpack.spatial.vectortile.AbstractVectorTileSearchAction.Request; -import java.io.IOException; import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestVectorTileAction extends AbstractVectorTileSearchAction { +public class RestVectorTileAction extends AbstractVectorTileSearchAction { + + public RestVectorTileAction() { + super(Request::new); + } @Override public List routes() { @@ -31,59 +28,20 @@ public List routes() { } @Override - protected ResponseBuilder doParseRequest( - RestRequest restRequest, String field, int z, int x, int y, SearchRequestBuilder searchRequestBuilder) throws IOException { - QueryBuilder queryBuilder = null; - if (restRequest.hasContent()) { - try (XContentParser parser = restRequest.contentParser()) { - XContentParser.Token token = parser.nextToken(); - if (token != XContentParser.Token.START_OBJECT) { - throw new IllegalArgumentException("Invalid content format"); - } - String currentFieldName; - while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { - if (token == XContentParser.Token.FIELD_NAME) { - currentFieldName = parser.currentName(); - if (currentFieldName.equals("query")) { - queryBuilder = AbstractQueryBuilder.parseInnerQueryBuilder(parser); - } else { - throw new IllegalArgumentException("Unsupported field " + currentFieldName); - } - } else { - throw new IllegalArgumentException("Invalid content format"); - } - } - } - } - - searchBuilder(searchRequestBuilder, field, z, x, y, queryBuilder); + protected ResponseBuilder doParseRequest(RestRequest restRequest, Request request, SearchRequestBuilder searchRequestBuilder) { + final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(request.getField()) + .field(request.getField()) + .z(request.getZ()) + .x(request.getX()) + .y(request.getY()); + searchRequestBuilder.addAggregation(aBuilder).setSize(0); return (s, b) -> { - InternalVectorTile t = s.getAggregations().get(field); + InternalVectorTile t = s.getAggregations().get(request.getField()); // TODO: Error processing t.writeTileToStream(b); }; } - private static void searchBuilder( - SearchRequestBuilder searchRequestBuilder, - String field, - int z, - int x, - int y, - QueryBuilder queryBuilder - ) throws IOException { - final Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); - QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, rectangle); - if (queryBuilder != null) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(queryBuilder); - boolQueryBuilder.filter(qBuilder); - qBuilder = boolQueryBuilder; - } - final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(field).field(field).z(z).x(x).y(y); - searchRequestBuilder.setQuery(qBuilder).addAggregation(aBuilder).setSize(0); - } - @Override public String getName() { return "vectortile_action"; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java deleted file mode 100644 index fbbb37a250e9d..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileAggConfig.java +++ /dev/null @@ -1,97 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.vectortile; - - -import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.Map; - -import static java.util.Collections.emptyMap; - - -public class VectorTileAggConfig { - - private static final ParseField SCALING = new ParseField("scaling"); - - public static final ObjectParser PARSER = - new ObjectParser<>("gridmvt_config", VectorTileAggConfig::new); - - static { - PARSER.declareField( - VectorTileAggConfig::setRuntimeField, - XContentParser::map, - SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - PARSER.declareField( - VectorTileAggConfig::setQueryBuilder, - (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, - SearchSourceBuilder.QUERY_FIELD, - ObjectParser.ValueType.OBJECT - ); - PARSER.declareField( - VectorTileAggConfig::setAggBuilder, - AggregatorFactories::parseAggregators, - SearchSourceBuilder.AGGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - PARSER.declareInt(VectorTileAggConfig::setScaling, SCALING); - } - - public static VectorTileAggConfig getInstance() { - return new VectorTileAggConfig(); - } - - private QueryBuilder queryBuilder; - private AggregatorFactories.Builder aggBuilder; - private int scaling = 8; - private Map runtimeMappings = emptyMap(); - - private VectorTileAggConfig() { - } - - private void setQueryBuilder(QueryBuilder queryBuilder) { - this.queryBuilder = queryBuilder; - } - - public QueryBuilder getQueryBuilder() { - return queryBuilder; - } - - private void setAggBuilder(AggregatorFactories.Builder aggBuilder) { - this.aggBuilder = aggBuilder; - } - - public AggregatorFactories.Builder getAggBuilder() { - return aggBuilder; - } - - private void setScaling(int scaling) { - this.scaling = scaling; - } - - public int getScaling() { - return scaling; - } - - private void setRuntimeField(Map runtimeMappings) { - this.runtimeMappings = runtimeMappings; - } - - public Map getRuntimeMappings() { - return runtimeMappings; - } -} From 14826ec876c29aa0f1b3fc3f50222eab55818779 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Tue, 13 Apr 2021 07:43:23 +0200 Subject: [PATCH 04/15] simplify mvt end point (#71548) --- .../xpack/spatial/SpatialPlugin.java | 10 +- .../VectorTileGeoShapeAggregator.java | 8 +- .../AbstractVectorTileSearchAction.java | 149 +++++++++- .../spatial/vectortile/FeatureFactory.java | 10 +- .../RestAggregatedVectorTileAction.java | 209 ------------- .../vectortile/RestVectorTileAction.java | 214 +++++++++++++- .../vectortile/VectorTileGeometryBuilder.java | 15 +- .../aggregations/InternalVectorTileTests.java | 9 +- .../xpack/spatial/VectorTileGridRestIT.java | 279 ------------------ .../xpack/spatial/VectorTileRestIT.java | 244 +++++++++++++-- 10 files changed, 586 insertions(+), 561 deletions(-) delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java delete mode 100644 x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 8b2fe7793fe84..0cde726f93fed 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -44,9 +44,6 @@ import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction; -import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; -import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; -import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; @@ -54,6 +51,8 @@ import org.elasticsearch.xpack.spatial.ingest.CircleProcessor; import org.elasticsearch.xpack.spatial.search.aggregations.GeoLineAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.InternalGeoLine; +import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; +import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; @@ -63,9 +62,9 @@ import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; -import org.elasticsearch.xpack.spatial.vectortile.RestAggregatedVectorTileAction; import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction; import java.util.Arrays; @@ -100,8 +99,7 @@ public List getRestHandlers(Settings settings, RestController restC IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { return Arrays.asList( - new RestVectorTileAction(), - new RestAggregatedVectorTileAction()); + new RestVectorTileAction()); } @Override diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java index 3a976fa885285..78a722861e342 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.spatial.search.aggregations; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import org.apache.lucene.index.LeafReaderContext; import org.elasticsearch.common.geo.GeometryParser; import org.elasticsearch.geometry.Point; @@ -68,6 +70,7 @@ public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBuc final FeatureFactory featureFactory = new FeatureFactory(z, x, y, POLYGON_EXTENT); final PointFactory pointFactory = new PointFactory(); final CustomFieldsVisitor visitor = new CustomFieldsVisitor(Set.of(), true); + IUserDataConverter ignoreData = new UserDataIgnoreConverter(); return new LeafBucketCollectorBase(sub, values) { @Override public void collect(int doc, long bucket) throws IOException { @@ -99,7 +102,7 @@ public void collect(int doc, long bucket) throws IOException { final Object lines = lookup.get(fieldName); if (lines != null) { addLineFeatures(visitor.id(), - featureFactory.getFeatures(parser.parseGeometry(lines))); + featureFactory.getFeatures(parser.parseGeometry(lines), ignoreData)); } } break; @@ -114,7 +117,8 @@ public void collect(int doc, long bucket) throws IOException { lookup.setSource(visitor.source()); final Object polygons = lookup.get(fieldName); if (polygons != null) { - addPolygonFeatures(visitor.id(), featureFactory.getFeatures(parser.parseGeometry(polygons))); + addPolygonFeatures(visitor.id(), + featureFactory.getFeatures(parser.parseGeometry(polygons), ignoreData)); } } break; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java index dcc39a00c502e..6dce260c1f2ae 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java @@ -11,6 +11,7 @@ import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.client.node.NodeClient; import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.io.Streams; import org.elasticsearch.common.io.stream.BytesStream; @@ -27,13 +28,21 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.AggregatorFactories; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; import java.util.function.Supplier; +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + /** * Base class for rest actions that performs a search and translates it into * a protobuf response @@ -45,6 +54,22 @@ public abstract class AbstractVectorTileSearchAction parser; @@ -56,12 +81,20 @@ protected interface ResponseBuilder { private final Supplier emptyRequestProvider; protected static class Request { - QueryBuilder queryBuilder; - String index; - String field; - int x; - int y; - int z; + private QueryBuilder queryBuilder; + private String index; + private String field; + private int x; + private int y; + private int z; + private Map runtimeMappings = emptyMap(); + private int gridPrecision = 8; + private GRID_TYPE gridType = GRID_TYPE.GRID; + private int size = 10000; + private int extent = 4096; + private AggregatorFactories.Builder aggBuilder; + private List fields = emptyList(); + private boolean exact_bounds; public String getIndex() { return index; @@ -103,11 +136,37 @@ public void setZ(int z) { this.z = z; } + public int getExtent() { + return extent; + } + + public void setExtent(int extent) { + // TODO: validation + this.extent = extent; + } + + public boolean getExactBounds() { + return exact_bounds; + } + + public void setExactBounds(boolean exact_bounds) { + this.exact_bounds = exact_bounds; + } + + public List getFields() { + return fields; + } + + public void setFields(List fields) { + this.fields = fields; + } + public QueryBuilder getQueryBuilder() { return queryBuilder; } public void setQueryBuilder(QueryBuilder queryBuilder) { + // TODO: validation this.queryBuilder = queryBuilder; } @@ -125,17 +184,94 @@ public QueryBuilder getQuery() throws IOException { } return qBuilder; } + + public Map getRuntimeMappings() { + return runtimeMappings; + } + + public void setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public int getGridPrecision() { + return gridPrecision; + } + + public void setGridPrecision(int gridPrecision) { + if (gridPrecision < 0 || gridPrecision > 8) { + throw new IllegalArgumentException("Invalid grid precision, value should be between 0 and 8, got [" + gridPrecision + "]"); + } + this.gridPrecision = gridPrecision; + } + + public GRID_TYPE getGridType() { + return gridType; + } + + public void setGridType(String gridType) { + this.gridType = GRID_TYPE.fromString(gridType); + } + + public int getSize() { + return size; + } + + public void setSize(int size) { + // TODO: validation + this.size = size; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + // TODO: validation + this.aggBuilder = aggBuilder; + } } protected AbstractVectorTileSearchAction(Supplier emptyRequestProvider) { this.emptyRequestProvider = emptyRequestProvider; parser = new ObjectParser<>(getName(), emptyRequestProvider); + parser.declareInt(Request::setSize, SearchSourceBuilder.SIZE_FIELD); + parser.declareField( + Request::setFields, + AbstractVectorTileSearchAction::parseFetchFields, + SearchSourceBuilder.FETCH_FIELDS_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY + ); parser.declareField( Request::setQueryBuilder, (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, SearchSourceBuilder.QUERY_FIELD, ObjectParser.ValueType.OBJECT ); + parser.declareField( + Request::setRuntimeMappings, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + parser.declareField( + Request::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + // Specific for vector tiles + parser.declareInt(Request::setGridPrecision, GRID_PRECISION_FIELD); + parser.declareInt(Request::setExtent, EXTENT_FIELD); + parser.declareBoolean(Request::setExactBounds, EXACT_BOUNDS_FIELD); + parser.declareString(Request::setGridType, GRID_TYPE_FIELD); + } + + private static List parseFetchFields(XContentParser parser) throws IOException { + List fetchFields = new ArrayList<>(); + while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { + fetchFields.add(FieldAndFormat.fromXContent(parser)); + } + return fetchFields; } protected abstract ResponseBuilder doParseRequest(RestRequest restRequest, R request, SearchRequestBuilder searchRequestBuilder) @@ -159,7 +295,6 @@ protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(request.getIndex())); searchRequestBuilder.setQuery(request.getQuery()); - searchRequestBuilder.setSize(0); ResponseBuilder responseBuilder = doParseRequest(restRequest, request, searchRequestBuilder); // TODO: how do we handle cancellations? diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java index 29077ca2d3258..c99cc238c2bf3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java @@ -12,7 +12,6 @@ import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.adapt.jts.JtsAdapter; import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; import org.elasticsearch.geometry.Circle; @@ -37,7 +36,6 @@ public class FeatureFactory { private final IGeometryFilter acceptAllGeomFilter = geometry -> true; - private final IUserDataConverter ignoreUserData = new UserDataIgnoreConverter(); private final MvtLayerParams layerParams; private final GeometryFactory geomFactory = new GeometryFactory(); private final MvtLayerProps layerProps = new MvtLayerProps(); @@ -55,12 +53,16 @@ public FeatureFactory(int z, int x, int y, int extent) { this.layerParams = new MvtLayerParams(extent, extent); } - public List getFeatures(Geometry geometry) { + public List getFeatures(Geometry geometry, IUserDataConverter userData) { TileGeomResult tileGeom = JtsAdapter.createTileGeom(JtsAdapter.flatFeatureList(geometry.visit(builder)), tileEnvelope, clipEnvelope, geomFactory, layerParams, acceptAllGeomFilter); // MVT tile geometry to MVT features - return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, ignoreUserData); + return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData); + } + + public MvtLayerProps getLayerProps() { + return layerProps; } private static class JTSGeometryBuilder implements GeometryVisitor { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java deleted file mode 100644 index 7ea8e86d54f58..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestAggregatedVectorTileAction.java +++ /dev/null @@ -1,209 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.vectortile; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; -import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; -import org.elasticsearch.search.builder.SearchSourceBuilder; - -import java.io.IOException; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import static java.util.Collections.emptyMap; -import static org.elasticsearch.rest.RestRequest.Method.GET; - -public class RestAggregatedVectorTileAction extends AbstractVectorTileSearchAction { - - private static final String TYPE_PARAM = "type"; - private static final String GRID_TYPE = "grid"; - - private static final String GRID_FIELD = "grid"; - private static final String BOUNDS_FIELD = "bounds"; - - private static final ParseField SCALING = new ParseField("scaling"); - - public RestAggregatedVectorTileAction() { - super(AggregatedRequest::new); - parser.declareField( - AggregatedRequest::setRuntimeMappings, - XContentParser::map, - SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareField( - AggregatedRequest::setAggBuilder, - AggregatorFactories::parseAggregators, - SearchSourceBuilder.AGGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareInt(AggregatedRequest::setScaling, SCALING); - } - - protected static class AggregatedRequest extends AbstractVectorTileSearchAction.Request { - private Map runtimeMappings = emptyMap(); - private int scaling = 8; - private AggregatorFactories.Builder aggBuilder; - - public AggregatedRequest() {} - - public Map getRuntimeMappings() { - return runtimeMappings; - } - - public void setRuntimeMappings(Map runtimeMappings) { - this.runtimeMappings = runtimeMappings; - } - - public int getScaling() { - return scaling; - } - - public void setScaling(int scaling) { - this.scaling = scaling; - } - - public AggregatorFactories.Builder getAggBuilder() { - return aggBuilder; - } - - public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { - this.aggBuilder = aggBuilder; - } - } - - @Override - public List routes() { - return List.of(new Route(GET, "{index}/_agg_mvt/{field}/{z}/{x}/{y}")); - } - - @Override - protected ResponseBuilder doParseRequest(RestRequest restRequest, AggregatedRequest request, SearchRequestBuilder searchRequestBuilder) - throws IOException { - final boolean isGrid = restRequest.hasParam(TYPE_PARAM) && GRID_TYPE.equals(restRequest.param(TYPE_PARAM)); - - searchBuilder(searchRequestBuilder, request); - final int extent = 1 << request.getScaling(); - - return (s, b) -> { - // TODO: of there is no hits, should we return an empty tile with no layers or - // a tile with empty layers? - final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( - request.getZ(), - request.getX(), - request.getY(), - extent - ); - final InternalGeoTileGrid grid = s.getAggregations().get(GRID_FIELD); - tileBuilder.addLayers(getPointLayer(extent, isGrid, grid, geomBuilder)); - final InternalGeoBounds bounds = s.getAggregations().get(BOUNDS_FIELD); - tileBuilder.addLayers(getMetaLayer(extent, bounds, geomBuilder)); - tileBuilder.build().writeTo(b); - }; - } - - private VectorTile.Tile.Layer.Builder getPointLayer( - int extent, - boolean isGrid, - InternalGeoTileGrid t, - VectorTileGeometryBuilder geomBuilder - ) { - final VectorTile.Tile.Layer.Builder pointLayerBuilder = VectorTileUtils.createLayerBuilder("AGG", extent); - pointLayerBuilder.addKeys("count"); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); - final HashMap values = new HashMap<>(); - - for (InternalGeoGridBucket bucket : t.getBuckets()) { - long count = bucket.getDocCount(); - if (count > 0) { - featureBuilder.clear(); - // create geometry commands - if (isGrid) { - Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); - geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - } else { - GeoPoint point = (GeoPoint) bucket.getKey(); - geomBuilder.point(featureBuilder, point.lon(), point.lat()); - } - // Add count as key value pair - featureBuilder.addTags(0); - final int tagValue; - if (values.containsKey(count)) { - tagValue = values.get(count); - } else { - valueBuilder.clear(); - valueBuilder.setIntValue(count); - tagValue = values.size(); - pointLayerBuilder.addValues(valueBuilder); - values.put(count, tagValue); - } - featureBuilder.addTags(tagValue); - pointLayerBuilder.addFeatures(featureBuilder); - } - } - return pointLayerBuilder; - } - - private VectorTile.Tile.Layer.Builder getMetaLayer(int extent, InternalGeoBounds t, VectorTileGeometryBuilder geomBuilder) { - final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder("META", extent); - final GeoPoint topLeft = t.topLeft(); - final GeoPoint bottomRight = t.bottomRight(); - if (topLeft != null && bottomRight != null) { - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); - metaLayerBuilder.addFeatures(featureBuilder); - } - return metaLayerBuilder; - } - - private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, AggregatedRequest request) - throws IOException { - Rectangle rectangle = request.getBoundingBox(); - int extent = 1 << request.getScaling(); - GeoBoundingBox boundingBox = new GeoBoundingBox( - new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), - new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) - ); - GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) - .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getScaling())) - .setGeoBoundingBox(boundingBox) - .size(extent * extent); - if (request.getAggBuilder() != null) { - aBuilder.subAggregations(request.getAggBuilder()); - } - GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) - .wrapLongitude(false); - SearchRequestBuilder requestBuilder = searchRequestBuilder.addAggregation(aBuilder).addAggregation(boundsBuilder).setSize(0); - if (request.getRuntimeMappings() != null) { - requestBuilder.setRuntimeMappings(request.getRuntimeMappings()); - } - return requestBuilder; - } - - @Override - public String getName() { - return "vectortile_aggregation_action"; - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index bd54609386568..797342df69f78 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -6,17 +6,51 @@ */ package org.elasticsearch.xpack.spatial.vectortile; +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; -import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; -import org.elasticsearch.xpack.spatial.vectortile.AbstractVectorTileSearchAction.Request; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.util.List; import static org.elasticsearch.rest.RestRequest.Method.GET; -public class RestVectorTileAction extends AbstractVectorTileSearchAction { +public class RestVectorTileAction extends AbstractVectorTileSearchAction { + + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private static final String GRID_FIELD = "grid"; + private static final String BOUNDS_FIELD = "bounds"; + + private static final String COUNT_TAG = "count"; + private static final String ID_TAG = "id"; public RestVectorTileAction() { super(Request::new); @@ -29,19 +63,173 @@ public List routes() { @Override protected ResponseBuilder doParseRequest(RestRequest restRequest, Request request, SearchRequestBuilder searchRequestBuilder) { - final VectorTileAggregationBuilder aBuilder = new VectorTileAggregationBuilder(request.getField()) - .field(request.getField()) - .z(request.getZ()) - .x(request.getX()) - .y(request.getY()); - searchRequestBuilder.addAggregation(aBuilder).setSize(0); + final int extent = request.getExtent(); + searchBuilder(searchRequestBuilder, request); return (s, b) -> { - InternalVectorTile t = s.getAggregations().get(request.getField()); - // TODO: Error processing - t.writeTileToStream(b); + // Even if there is no hits, we return a tile with the meta layer + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( + request.getZ(), + request.getX(), + request.getY(), + extent + ); + final SearchHit[] hits = s.getHits().getHits(); + if (hits.length > 0) { + tileBuilder.addLayers(getHitsLayer(s, request)); + } + final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; + // TODO: should be expose the total number of buckets on InternalGeoTileGrid? + if (grid != null && grid.getBuckets().size() > 0) { + tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder)); + } + tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder)); + tileBuilder.build().writeTo(b); }; } + private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, Request request) { + searchRequestBuilder.setSize(request.getSize()); + searchRequestBuilder.setFetchSource(false); + // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. + searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); + for (FieldAndFormat field : request.getFields()) { + searchRequestBuilder.addFetchField(field); + } + searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); + if (request.getGridPrecision() > 0) { + final Rectangle rectangle = request.getBoundingBox(); + final GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + final int extent = 1 << request.getGridPrecision(); + final GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) + .setGeoBoundingBox(boundingBox) + .size(extent * extent); + if (request.getAggBuilder() != null) { + aBuilder.subAggregations(request.getAggBuilder()); + } + searchRequestBuilder.addAggregation(aBuilder); + } + if (request.getExactBounds()) { + final GeoBoundsAggregationBuilder boundsBuilder = + new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false); + searchRequestBuilder.addAggregation(boundsBuilder); + } + return searchRequestBuilder; + } + + private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Request request) { + final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); + final GeometryParser parser = new GeometryParser(true, false, false); + final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); + final List fields = request.getFields(); + for (SearchHit searchHit : response.getHits()) { + final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { + // TODO: It would be great if we can add the centroid information for polygons. That information can be + // used to place labels inside those geometries + addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); + if (fields != null) { + for (FieldAndFormat field : fields) { + DocumentField documentField = searchHit.field(field.field); + if (documentField != null) { + addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); + } + } + } + }; + // TODO: See comment on field formats. + final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); + hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); + } + addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + return hitsLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD); + for (InternalGeoGridBucket bucket : grid.getBuckets()) { + final long count = bucket.getDocCount(); + featureBuilder.clear(); + // Add geometry + if (request.getGridType() == GRID_TYPE.GRID) { + final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + // TODO: it should be the centroid of the data + final GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); + } + // Add count as key value pair + addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, count); + // Add aggregations results as key value pair + for (Aggregation aggregation : bucket.getAggregations()) { + final String type = aggregation.getType(); + switch (type) { + case MinAggregationBuilder.NAME: + case MaxAggregationBuilder.NAME: + case AvgAggregationBuilder.NAME: + case SumAggregationBuilder.NAME: + case CardinalityAggregationBuilder.NAME: + final NumericMetricsAggregation.SingleValue metric = (NumericMetricsAggregation.SingleValue) aggregation; + addPropertyToFeature(featureBuilder, layerProps, "aggs." + aggregation.getName(), metric.value()); + break; + default: + // top term and percentile should be supported + throw new IllegalArgumentException("Unknown feature type [" + type + "]"); + } + } + aggLayerBuilder.addFeatures(featureBuilder); + } + addPropertiesToLayer(aggLayerBuilder, layerProps); + return aggLayerBuilder; + } + + private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null; + if (bounds != null && bounds.topLeft() != null) { + final GeoPoint topLeft = bounds.topLeft(); + final GeoPoint bottomRight = bounds.bottomRight(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + } else { + final Rectangle tile = request.getBoundingBox(); + geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); + } + addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards()); + addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards()); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value); + addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name()); + metaLayerBuilder.addFeatures(featureBuilder); + addPropertiesToLayer(metaLayerBuilder, layerProps); + return metaLayerBuilder; + } + + private void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) { + feature.addTags(layerProps.addKey(key)); + feature.addTags(layerProps.addValue(value)); + } + + private void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { + // Add keys + layer.addAllKeys(layerProps.getKeys()); + // Add values + final Iterable values = layerProps.getVals(); + for (Object value : values) { + layer.addValues(MvtValue.toValue(value)); + } + } + @Override public String getName() { return "vectortile_action"; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java index ea1fcc5a7b01f..5d31a7480da5a 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java @@ -26,14 +26,6 @@ class VectorTileGeometryBuilder { pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); } - public int lat(double lat) { - return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; - } - - public int lon(double lon) { - return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); - } - public void point(VectorTile.Tile.Feature.Builder featureBuilder, double lon, double lat) { featureBuilder.setType(VectorTile.Tile.GeomType.POINT); featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); @@ -64,4 +56,11 @@ public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, d featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.ClosePath, 1)); } + private int lat(double lat) { + return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; + } + + private int lon(double lon) { + return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + } } diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java index 18f297402e1dc..3957dd3b8472f 100644 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java +++ b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java @@ -7,6 +7,8 @@ package org.elasticsearch.xpack.spatial.search.aggregations; import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; import org.elasticsearch.common.ParseField; import org.elasticsearch.common.util.CollectionUtils; import org.elasticsearch.common.xcontent.NamedXContentRegistry; @@ -40,11 +42,12 @@ static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); for (int i =0; i < shapes; i++) { int count = layerBuilder.getFeaturesCount(); while(true) { Geometry geometry = GeometryTestUtils.randomPolygon(false); - List features = factory.getFeatures(geometry); + List features = factory.getFeatures(geometry, ignoreData); for (VectorTile.Tile.Feature feature : features) { layerBuilder.addFeatures(feature); } @@ -54,7 +57,6 @@ static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { } } return layerBuilder.build(); - } static VectorTile.Tile.Layer randomLineLayer(int shapes) { @@ -63,11 +65,12 @@ static VectorTile.Tile.Layer randomLineLayer(int shapes) { layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); + final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); for (int i =0; i < shapes; i++) { int count = layerBuilder.getFeaturesCount(); while(true) { Geometry geometry = GeometryTestUtils.randomLine(false); - List features = factory.getFeatures(geometry); + List features = factory.getFeatures(geometry, ignoreData); for (VectorTile.Tile.Feature feature : features) { layerBuilder.addFeatures(feature); } diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java deleted file mode 100644 index d096b7e60ae1c..0000000000000 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileGridRestIT.java +++ /dev/null @@ -1,279 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.apache.http.HttpStatus; -import org.apache.http.client.methods.HttpDelete; -import org.apache.http.client.methods.HttpGet; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.client.methods.HttpPut; -import org.elasticsearch.client.Request; -import org.elasticsearch.client.Response; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.test.rest.ESRestTestCase; -import org.hamcrest.Matchers; -import org.junit.After; -import org.junit.Before; - -import java.io.IOException; -import java.io.InputStream; - -public class VectorTileGridRestIT extends ESRestTestCase { - - private static String INDEX_POINTS = "index-points"; - private static String INDEX_SHAPES = "index-shapes"; - - private int x, y, z; - - @Before - public void indexDocuments() throws IOException { - - z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); - x = randomIntBetween(0, (1 << z) - 1); - y = randomIntBetween(0, (1 << z) - 1); - indexPoints(); - indexShapes(); - } - - private void indexPoints() throws IOException { - - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); - Response response = client().performRequest(createRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_point\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); - response = client().performRequest(mappingRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); - double x = (r.getMaxX() + r.getMinX()) / 2; - double y = (r.getMaxY() + r.getMinY()) / 2; - for (int i = 0; i < 30; i+=10) { - for (int j = 0; j <= i; j++) { - final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + - ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + - "}"); - response = client().performRequest(putRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - } - } - - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); - response = client().performRequest(flushRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - private void indexShapes() throws IOException { - - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); - Response response = client().performRequest(createRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_shape\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); - response = client().performRequest(mappingRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); - - final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + - ", \"name\": \"rectangle\"" + - ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + - "}"); - response = client().performRequest(putRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - - - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); - response = client().performRequest(flushRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - @After - public void deleteData() throws IOException { - final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); - Response response = client().performRequest(deleteRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - } - - public void testBasicGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testEmpty() throws Exception { - final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + newY); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(0)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(0)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testBasicScaling() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"scaling\": 7 }"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(128)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(128)); - } - } - - private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { - for (int i = 0; i < tile.getLayersCount(); i++) { - VectorTile.Tile.Layer layer = tile.getLayers(i); - if (layerName.equals(layer.getName())) { - return layer; - } - } - fail("Could not find layer " + layerName); - return null; - } - - @AwaitsFix(bugUrl = "doesn't work yet") - public void testBasicQueryGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_agg_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"query\": {\n" + - " \"term\": {\n" + - " \"name\": {\n" + - " \"value\": \"point2\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); - Response response = client().performRequest(mvtRequest); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - InputStream inputStream = response.getEntity().getContent(); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testBasicShape() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } - - public void testMinAgg() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_agg_mvt/location/"+ z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"aggs\": {\n" + - " \"minVal\": {\n" + - " \"min\": {\n" + - " \"field\": \"value1\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile tile = VectorTile.Tile.parseFrom(inputStream); - assertThat(tile.getLayersCount(), Matchers.equalTo(2)); - { - VectorTile.Tile.Layer layer = getLayer(tile, "AGG"); - assertThat(layer.getValuesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(256 * 256)); - } - { - VectorTile.Tile.Layer layer = getLayer(tile, "META"); - assertThat(layer.getFeaturesCount(), Matchers.equalTo(1)); - assertThat(layer.getExtent(), Matchers.equalTo(256)); - } - } -} diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index 6e4af5d93f3e5..d5a776dd6d097 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -15,6 +15,9 @@ import org.apache.http.client.methods.HttpPut; import org.elasticsearch.client.Request; import org.elasticsearch.client.Response; +import org.elasticsearch.client.ResponseException; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.test.rest.ESRestTestCase; import org.hamcrest.Matchers; import org.junit.After; @@ -23,18 +26,34 @@ import java.io.IOException; import java.io.InputStream; +/** + * Rest test for _mvt end point. The test only check that the structure of the vector tiles is sound in + * respect to the number of layers returned and the number of features abd tags in each layer. + */ public class VectorTileRestIT extends ESRestTestCase { - private static String INDEX_NAME = "my-index"; + private static final String INDEX_POINTS = "index-points"; + private static final String INDEX_SHAPES = "index-shapes"; + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private int x, y, z; @Before public void indexDocuments() throws IOException { + z = randomIntBetween(1, GeoTileUtils.MAX_ZOOM - 10); + x = randomIntBetween(0, (1 << z) - 1); + y = randomIntBetween(0, (1 << z) - 1); + indexPoints(); + indexShapes(); + } - final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME); + private void indexPoints() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS); Response response = client().performRequest(createRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - - final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_NAME + "/_mapping"); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); mappingRequest.setJsonEntity("{\n" + " \"properties\": {\n" + " \"location\": {\n" + @@ -47,59 +66,224 @@ public void indexDocuments() throws IOException { "}"); response = client().performRequest(mappingRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + double x = (r.getMaxX() + r.getMinX()) / 2; + double y = (r.getMaxY() + r.getMinY()) / 2; + for (int i = 0; i < 30; i += 10) { + for (int j = 0; j <= i; j++) { + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + + ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + + "}"); + response = client().performRequest(putRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + } + } - final Request putRequest1 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); - putRequest1.setJsonEntity("{\n" + - " \"location\": \"POINT(0 0)\", \"name\": \"point1\"\n" + - "}"); - - response = client().performRequest(putRequest1); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_refresh"); + response = client().performRequest(flushRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + } - final Request putRequest2 = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_doc"); - putRequest2.setJsonEntity("{\n" + - " \"location\": \"POINT(1 1)\", \"name\": \"point2\"\n" + + private void indexShapes() throws IOException { + final Request createRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES); + Response response = client().performRequest(createRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); + final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); + mappingRequest.setJsonEntity("{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_shape\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + "}"); + response = client().performRequest(mappingRequest); + assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - response = client().performRequest(putRequest2); + final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); + putRequest.setJsonEntity("{\n" + + " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + + ", \"name\": \"rectangle\"" + + ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + + "}"); + response = client().performRequest(putRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); - final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_NAME + "/_refresh"); + final Request flushRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_refresh"); response = client().performRequest(flushRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); } @After public void deleteData() throws IOException { - final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_NAME); - Response response = client().performRequest(deleteRequest); + final Request deleteRequest = new Request(HttpDelete.METHOD_NAME, INDEX_POINTS); + final Response response = client().performRequest(deleteRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); } public void testBasicGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); - Response response = client().performRequest(mvtRequest); - InputStream inputStream = response.getEntity().getContent(); - assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); - assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(2)); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + + } + + public void testEmpty() throws Exception { + final int newY = (1 << z) - 1 == y ? y - 1 : y + 1; + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(1)); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testGridPrecision() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 7 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 9 }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testGridType() throws Exception { + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"point\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"grid\" }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_type\": \"invalid_type\" }"); + final ResponseException ex = expectThrows(ResponseException.class, () -> execute(mvtRequest)); + assertThat(ex.getResponse().getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_BAD_REQUEST)); + } + } + + public void testNoAggLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"grid_precision\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 33, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testNoHitsLayer() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\": 0 }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); } public void testBasicQueryGet() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_NAME + "/_mvt/location/0/0/0"); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); mvtRequest.setJsonEntity("{\n" + " \"query\": {\n" + " \"term\": {\n" + " \"name\": {\n" + - " \"value\": \"point2\"\n" + + " \"value\": \"point0\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 1, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testBasicShape() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testWithFields() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"fields\": [\"name\", \"value1\"] }"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 3); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + public void testMinAgg() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\n" + + " \"aggs\": {\n" + + " \"minVal\": {\n" + + " \"min\": {\n" + + " \"field\": \"value1\"\n" + " }\n" + " }\n" + " }\n" + "}"); - Response response = client().performRequest(mvtRequest); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 4096, 1, 1); + assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); + assertLayer(tile, META_LAYER, 4096, 1, 7); + } + + private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { + final VectorTile.Tile.Layer layer = getLayer(tile, name); + assertThat(layer.getExtent(), Matchers.equalTo(extent)); + assertThat(layer.getFeaturesCount(), Matchers.equalTo(numFeatures)); + assertThat(layer.getKeysCount(), Matchers.equalTo(numTags)); + } + + private VectorTile.Tile execute(Request mvtRequest) throws IOException { + final Response response = client().performRequest(mvtRequest); + final InputStream inputStream = response.getEntity().getContent(); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); - InputStream inputStream = response.getEntity().getContent(); - VectorTile.Tile.Builder builder = VectorTile.Tile.newBuilder().mergeFrom(inputStream); - assertThat(builder.getLayers(0).getFeaturesCount(), Matchers.equalTo(1)); + return VectorTile.Tile.parseFrom(inputStream); + } + + private VectorTile.Tile.Layer getLayer(VectorTile.Tile tile, String layerName) { + for (int i = 0; i < tile.getLayersCount(); i++) { + final VectorTile.Tile.Layer layer = tile.getLayers(i); + if (layerName.equals(layer.getName())) { + return layer; + } + } + fail("Could not find layer " + layerName); + return null; } } From 7b6f3daee7959db056be7fae0c7d5416ca7e826d Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Mon, 19 Apr 2021 12:57:41 -1000 Subject: [PATCH 05/15] Vector Tiles: add a more generic way of building meta layer (#71804) Replaces a manual way of building the meta layer with a more generic way of collecting unprocessed elements of the search response. --- .../org/elasticsearch/common/util/Maps.java | 21 +++++ .../vectortile/RestVectorTileAction.java | 77 +++++++++++++++---- .../xpack/spatial/VectorTileRestIT.java | 22 +++--- 3 files changed, 92 insertions(+), 28 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java index 30950e51756ae..167d3bd84297c 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Maps.java +++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java @@ -11,8 +11,10 @@ import org.elasticsearch.Assertions; import java.util.Collection; +import java.util.HashMap; import java.util.Map; import java.util.Objects; +import java.util.TreeMap; import java.util.stream.Collectors; import java.util.stream.Stream; @@ -125,4 +127,23 @@ public static boolean deepEquals(Map left, Map right) { .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey()))); } + public static Map flatten(Map map, boolean ordered) { + return flatten(map, ordered, null); + } + + @SuppressWarnings("unchecked") + private static Map flatten(Map map, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap() : new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() instanceof Map) { + flatMap.putAll(flatten((Map) entry.getValue(), ordered, prefix + entry.getKey())); + } else { + flatMap.put(prefix + entry.getKey(), entry.getValue()); + } + } + + return flatMap; + } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index 797342df69f78..8b0909cd5f5a0 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -12,15 +12,21 @@ import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; import org.elasticsearch.common.document.DocumentField; import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; @@ -35,8 +41,12 @@ import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import java.io.IOException; import java.util.List; +import java.util.Map; +import java.util.stream.Collectors; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -75,15 +85,43 @@ protected ResponseBuilder doParseRequest(RestRequest restRequest, Request reques extent ); final SearchHit[] hits = s.getHits().getHits(); + final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; + final InternalGeoBounds bounds = s.getAggregations() != null ? s.getAggregations().get(BOUNDS_FIELD) : null; + final Aggregations aggsWithoutGridAndBounds = s.getAggregations() == null + ? null + : new Aggregations( + s.getAggregations() + .asList() + .stream() + .filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false) + .collect(Collectors.toList()) + ); + SearchResponse meta = new SearchResponse( + new SearchResponseSections( + new SearchHits(SearchHits.EMPTY, s.getHits().getTotalHits(), s.getHits().getMaxScore()), // remove actual hits + aggsWithoutGridAndBounds, + s.getSuggest(), + s.isTimedOut(), + s.isTerminatedEarly(), + s.getProfileResults() == null ? null : new SearchProfileShardResults(s.getProfileResults()), + s.getNumReducePhases() + ), + s.getScrollId(), + s.getTotalShards(), + s.getSuccessfulShards(), + s.getSkippedShards(), + s.getTook().millis(), + s.getShardFailures(), + s.getClusters() + ); if (hits.length > 0) { tileBuilder.addLayers(getHitsLayer(s, request)); } - final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; // TODO: should be expose the total number of buckets on InternalGeoTileGrid? if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(getAggsLayer(s, request, geomBuilder)); + tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder)); } - tileBuilder.addLayers(getMetaLayer(s, request, geomBuilder)); + tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder)); tileBuilder.build().writeTo(b); }; } @@ -114,8 +152,8 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq searchRequestBuilder.addAggregation(aBuilder); } if (request.getExactBounds()) { - final GeoBoundsAggregationBuilder boundsBuilder = - new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()).wrapLongitude(false); + final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) + .wrapLongitude(false); searchRequestBuilder.addAggregation(boundsBuilder); } return searchRequestBuilder; @@ -148,11 +186,10 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ return hitsLayerBuilder; } - private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) { final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final InternalGeoTileGrid grid = response.getAggregations().get(GRID_FIELD); for (InternalGeoGridBucket bucket : grid.getBuckets()) { final long count = bucket.getDocCount(); featureBuilder.clear(); @@ -170,7 +207,7 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ // Add aggregations results as key value pair for (Aggregation aggregation : bucket.getAggregations()) { final String type = aggregation.getType(); - switch (type) { + switch (type) { case MinAggregationBuilder.NAME: case MaxAggregationBuilder.NAME: case AvgAggregationBuilder.NAME: @@ -190,11 +227,19 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(SearchResponse response, Requ return aggLayerBuilder; } - private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Request request, VectorTileGeometryBuilder geomBuilder) { + private VectorTile.Tile.Layer.Builder getMetaLayer( + SearchResponse response, + InternalGeoBounds bounds, + Request request, + VectorTileGeometryBuilder geomBuilder + ) throws IOException { + Map responseMap = Maps.flatten( + XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(), + true + ); final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final InternalGeoBounds bounds = response.getAggregations() != null ? response.getAggregations().get(BOUNDS_FIELD) : null; if (bounds != null && bounds.topLeft() != null) { final GeoPoint topLeft = bounds.topLeft(); final GeoPoint bottomRight = bounds.bottomRight(); @@ -203,13 +248,11 @@ private VectorTile.Tile.Layer.Builder getMetaLayer(SearchResponse response, Requ final Rectangle tile = request.getBoundingBox(); geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); } - addPropertyToFeature(featureBuilder, layerProps, "timed_out", response.isTimedOut()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.total", response.getTotalShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.successful", response.getSuccessfulShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.skipped", response.getSkippedShards()); - addPropertyToFeature(featureBuilder, layerProps, "_shards.failed", response.getFailedShards()); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.value", response.getHits().getTotalHits().value); - addPropertyToFeature(featureBuilder, layerProps, "hits.total.relation", response.getHits().getTotalHits().relation.name()); + for (Map.Entry entry : responseMap.entrySet()) { + if (entry.getValue() != null) { + addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue()); + } + } metaLayerBuilder.addFeatures(featureBuilder); addPropertiesToLayer(metaLayerBuilder, layerProps); return metaLayerBuilder; diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index d5a776dd6d097..84e81a620cb92 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -132,7 +132,7 @@ public void testBasicGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } @@ -141,7 +141,7 @@ public void testEmpty() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(1)); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 10); } public void testGridPrecision() throws Exception { @@ -152,7 +152,7 @@ public void testGridPrecision() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -170,7 +170,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -179,7 +179,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -195,7 +195,7 @@ public void testNoAggLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testNoHitsLayer() throws Exception { @@ -204,7 +204,7 @@ public void testNoHitsLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 10); } public void testBasicQueryGet() throws Exception { @@ -222,7 +222,7 @@ public void testBasicQueryGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testBasicShape() throws Exception { @@ -231,7 +231,7 @@ public void testBasicShape() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testWithFields() throws Exception { @@ -241,7 +241,7 @@ public void testWithFields() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 3); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } public void testMinAgg() throws Exception { @@ -259,7 +259,7 @@ public void testMinAgg() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); - assertLayer(tile, META_LAYER, 4096, 1, 7); + assertLayer(tile, META_LAYER, 4096, 1, 11); } private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { From d6cfec0445cd35a2629533e2d39efd1603a4e4c5 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Thu, 22 Apr 2021 07:50:49 +0200 Subject: [PATCH 06/15] Add value ranges for GeoTile aggregation metrics metrics in the meta layer (#71611) --- .../AbstractVectorTileSearchAction.java | 32 +++++- .../vectortile/RestVectorTileAction.java | 100 ++++++------------ .../vectortile/VectorTileGeometryBuilder.java | 4 +- .../spatial/vectortile/VectorTileUtils.java | 46 ++++++++ .../xpack/spatial/VectorTileRestIT.java | 21 ++-- 5 files changed, 123 insertions(+), 80 deletions(-) diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java index 6dce260c1f2ae..58c3bab938bac 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java @@ -28,14 +28,22 @@ import org.elasticsearch.rest.RestResponse; import org.elasticsearch.rest.RestStatus; import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; import org.elasticsearch.search.builder.SearchSourceBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import java.io.IOException; import java.io.OutputStream; import java.util.ArrayList; +import java.util.Collection; import java.util.List; import java.util.Map; import java.util.function.Supplier; @@ -226,9 +234,31 @@ public AggregatorFactories.Builder getAggBuilder() { } public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { - // TODO: validation + for (AggregationBuilder aggregation : aggBuilder.getAggregatorFactories()) { + final String type = aggregation.getType(); + switch (type) { + case MinAggregationBuilder.NAME: + case MaxAggregationBuilder.NAME: + case AvgAggregationBuilder.NAME: + case SumAggregationBuilder.NAME: + case CardinalityAggregationBuilder.NAME: + break; + default: + // top term and percentile should be supported + throw new IllegalArgumentException("Unsupported aggregation of type [" + type + "]"); + } + } + for (PipelineAggregationBuilder aggregation : aggBuilder.getPipelineAggregatorFactories()) { + // should not have pipeline aggregations + final String type = aggregation.getType(); + throw new IllegalArgumentException("Unsupported pipeline aggregation of type [" + type + "]"); + } this.aggBuilder = aggBuilder; } + + public Collection getAggregations() { + return aggBuilder == null ? emptyList() : aggBuilder.getAggregatorFactories(); + } } protected AbstractVectorTileSearchAction(Supplier emptyRequestProvider) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java index 8b0909cd5f5a0..2abb2aa04b7d3 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java @@ -9,7 +9,6 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; -import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponseSections; @@ -17,35 +16,29 @@ import org.elasticsearch.common.geo.GeoBoundingBox; import org.elasticsearch.common.geo.GeoPoint; import org.elasticsearch.common.geo.GeometryParser; -import org.elasticsearch.common.util.Maps; -import org.elasticsearch.common.xcontent.XContentHelper; -import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.rest.RestRequest; import org.elasticsearch.search.SearchHit; import org.elasticsearch.search.SearchHits; import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilder; import org.elasticsearch.search.aggregations.Aggregations; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; -import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; -import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.NumericMetricsAggregation; -import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder; import org.elasticsearch.search.fetch.subphase.FieldAndFormat; import org.elasticsearch.search.profile.SearchProfileShardResults; import java.io.IOException; +import java.util.Collection; import java.util.List; -import java.util.Map; import java.util.stream.Collectors; import static org.elasticsearch.rest.RestRequest.Method.GET; @@ -59,8 +52,11 @@ public class RestVectorTileAction extends AbstractVectorTileSearchAction 0) { - tileBuilder.addLayers(getHitsLayer(s, request)); + tileBuilder.addLayers(buildHitsLayer(s, request)); } // TODO: should be expose the total number of buckets on InternalGeoTileGrid? if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(getAggsLayer(grid, request, geomBuilder)); + tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder)); } - tileBuilder.addLayers(getMetaLayer(meta, bounds, request, geomBuilder)); + tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder)); tileBuilder.build().writeTo(b); }; } @@ -150,6 +146,15 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq aBuilder.subAggregations(request.getAggBuilder()); } searchRequestBuilder.addAggregation(aBuilder); + searchRequestBuilder.addAggregation(new MaxBucketPipelineAggregationBuilder(COUNT_MAX, GRID_FIELD + "._count")); + searchRequestBuilder.addAggregation(new MinBucketPipelineAggregationBuilder(COUNT_MIN, GRID_FIELD + "._count")); + final Collection aggregations = request.getAggregations(); + for (AggregationBuilder aggregation : aggregations) { + searchRequestBuilder.addAggregation( + new MaxBucketPipelineAggregationBuilder(aggregation.getName() + ".max", GRID_FIELD + ">" + aggregation.getName())); + searchRequestBuilder.addAggregation( + new MinBucketPipelineAggregationBuilder(aggregation.getName() + ".min", GRID_FIELD + ">" + aggregation.getName())); + } } if (request.getExactBounds()) { final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) @@ -159,7 +164,7 @@ private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchReq return searchRequestBuilder; } - private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Request request) { + private VectorTile.Tile.Layer.Builder buildHitsLayer(SearchResponse response, Request request) { final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); final GeometryParser parser = new GeometryParser(true, false, false); final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); @@ -168,12 +173,12 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { // TODO: It would be great if we can add the centroid information for polygons. That information can be // used to place labels inside those geometries - addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); if (fields != null) { for (FieldAndFormat field : fields) { - DocumentField documentField = searchHit.field(field.field); + final DocumentField documentField = searchHit.field(field.field); if (documentField != null) { - addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); } } } @@ -182,16 +187,16 @@ private VectorTile.Tile.Layer.Builder getHitsLayer(SearchResponse response, Requ final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); } - addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); return hitsLayerBuilder; } - private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) { + private VectorTile.Tile.Layer.Builder buildAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) + throws IOException{ final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); for (InternalGeoGridBucket bucket : grid.getBuckets()) { - final long count = bucket.getDocCount(); featureBuilder.clear(); // Add geometry if (request.getGridType() == GRID_TYPE.GRID) { @@ -203,40 +208,22 @@ private VectorTile.Tile.Layer.Builder getAggsLayer(InternalGeoTileGrid grid, Req geomBuilder.point(featureBuilder, point.lon(), point.lat()); } // Add count as key value pair - addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, count); - // Add aggregations results as key value pair + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, bucket.getDocCount()); for (Aggregation aggregation : bucket.getAggregations()) { - final String type = aggregation.getType(); - switch (type) { - case MinAggregationBuilder.NAME: - case MaxAggregationBuilder.NAME: - case AvgAggregationBuilder.NAME: - case SumAggregationBuilder.NAME: - case CardinalityAggregationBuilder.NAME: - final NumericMetricsAggregation.SingleValue metric = (NumericMetricsAggregation.SingleValue) aggregation; - addPropertyToFeature(featureBuilder, layerProps, "aggs." + aggregation.getName(), metric.value()); - break; - default: - // top term and percentile should be supported - throw new IllegalArgumentException("Unknown feature type [" + type + "]"); - } + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, aggregation); } aggLayerBuilder.addFeatures(featureBuilder); } - addPropertiesToLayer(aggLayerBuilder, layerProps); + VectorTileUtils.addPropertiesToLayer(aggLayerBuilder, layerProps); return aggLayerBuilder; } - private VectorTile.Tile.Layer.Builder getMetaLayer( + private VectorTile.Tile.Layer.Builder buildMetaLayer( SearchResponse response, InternalGeoBounds bounds, Request request, VectorTileGeometryBuilder geomBuilder ) throws IOException { - Map responseMap = Maps.flatten( - XContentHelper.convertToMap(XContentHelper.toXContent(response, XContentType.CBOR, false), true, XContentType.CBOR).v2(), - true - ); final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); final MvtLayerProps layerProps = new MvtLayerProps(); final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); @@ -248,31 +235,12 @@ private VectorTile.Tile.Layer.Builder getMetaLayer( final Rectangle tile = request.getBoundingBox(); geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); } - for (Map.Entry entry : responseMap.entrySet()) { - if (entry.getValue() != null) { - addPropertyToFeature(featureBuilder, layerProps, entry.getKey(), entry.getValue()); - } - } + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response); metaLayerBuilder.addFeatures(featureBuilder); - addPropertiesToLayer(metaLayerBuilder, layerProps); + VectorTileUtils.addPropertiesToLayer(metaLayerBuilder, layerProps); return metaLayerBuilder; } - private void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) { - feature.addTags(layerProps.addKey(key)); - feature.addTags(layerProps.addValue(value)); - } - - private void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { - // Add keys - layer.addAllKeys(layerProps.getKeys()); - // Add values - final Iterable values = layerProps.getVals(); - for (Object value : values) { - layer.addValues(MvtValue.toValue(value)); - } - } - @Override public String getName() { return "vectortile_action"; diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java index 5d31a7480da5a..62b356b276178 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java @@ -57,10 +57,10 @@ public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, d } private int lat(double lat) { - return (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; + return (int) Math.round(pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; } private int lon(double lon) { - return (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + return (int) Math.round(pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); } } diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java index 361328b2f48d9..7cad6cf043a9c 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java @@ -8,14 +8,26 @@ package org.elasticsearch.xpack.spatial.vectortile; import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; +import org.elasticsearch.common.util.Maps; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentHelper; +import org.elasticsearch.common.xcontent.XContentType; import org.elasticsearch.geometry.Rectangle; import org.locationtech.jts.geom.Envelope; +import java.io.IOException; +import java.util.Map; + /** * Utility methods For vector tiles. Transforms WGS84 into spherical mercator. */ public class VectorTileUtils { + /** + * Creates a vector layer builder with the provided name and extent. + */ public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName, int extent) { final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); layerBuilder.setVersion(2); @@ -24,6 +36,40 @@ public static VectorTile.Tile.Layer.Builder createLayerBuilder(String layerName, return layerBuilder; } + /** + * Adds the flatten elements of toXContent into the feature as tags. + */ + public static void addToXContentToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, ToXContent toXContent) + throws IOException { + final Map map = Maps.flatten( + XContentHelper.convertToMap(XContentHelper.toXContent(toXContent, XContentType.CBOR, false), true, XContentType.CBOR).v2(), + true + ); + for (Map.Entry entry : map.entrySet()) { + if (entry.getValue() != null) { + addPropertyToFeature(feature, layerProps, entry.getKey(), entry.getValue()); + } + } + } + + /** + * Adds the provided key / value pair into the feature as tags. + */ + public static void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, MvtLayerProps layerProps, String key, Object value) { + feature.addTags(layerProps.addKey(key)); + feature.addTags(layerProps.addValue(value)); + } + + public static void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { + // Add keys + layer.addAllKeys(layerProps.getKeys()); + // Add values + final Iterable values = layerProps.getVals(); + for (Object value : values) { + layer.addValues(MvtValue.toValue(value)); + } + } + /** * Gets the JTS envelope for z/x/y/ tile in spherical mercator projection. */ diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index 84e81a620cb92..48fd9887409f0 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -132,8 +132,7 @@ public void testBasicGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); - + assertLayer(tile, META_LAYER, 4096, 1, 15); } public void testEmpty() throws Exception { @@ -141,7 +140,7 @@ public void testEmpty() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(1)); - assertLayer(tile, META_LAYER, 4096, 1, 10); + assertLayer(tile, META_LAYER, 4096, 1, 12); } public void testGridPrecision() throws Exception { @@ -152,7 +151,7 @@ public void testGridPrecision() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -170,7 +169,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -179,7 +178,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -204,7 +203,7 @@ public void testNoHitsLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 10); + assertLayer(tile, META_LAYER, 4096, 1, 14); } public void testBasicQueryGet() throws Exception { @@ -222,7 +221,7 @@ public void testBasicQueryGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } public void testBasicShape() throws Exception { @@ -231,7 +230,7 @@ public void testBasicShape() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } public void testWithFields() throws Exception { @@ -241,7 +240,7 @@ public void testWithFields() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 3); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 15); } public void testMinAgg() throws Exception { @@ -259,7 +258,7 @@ public void testMinAgg() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 19); } private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { From c4f919d1c001b3b42652e12e37aa3330baddc2ad Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Fri, 23 Apr 2021 14:22:06 -1000 Subject: [PATCH 07/15] Vector Tiles: add support for array serialization in meta layer (#72136) Adds support for array serialization in meta layer --- .../org/elasticsearch/common/util/Maps.java | 31 +++++++-- .../elasticsearch/common/util/MapsTests.java | 65 +++++++++++++++++++ .../spatial/vectortile/VectorTileUtils.java | 1 + .../xpack/spatial/VectorTileRestIT.java | 24 +++---- 4 files changed, 104 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java index 167d3bd84297c..5c1058a4665be 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Maps.java +++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java @@ -12,6 +12,7 @@ import java.util.Collection; import java.util.HashMap; +import java.util.List; import java.util.Map; import java.util.Objects; import java.util.TreeMap; @@ -127,23 +128,41 @@ public static boolean deepEquals(Map left, Map right) { .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey()))); } - public static Map flatten(Map map, boolean ordered) { - return flatten(map, ordered, null); + public static Map flatten(Map map, boolean flattenArrays, boolean ordered) { + return flatten(map, flattenArrays, ordered, null); } @SuppressWarnings("unchecked") - private static Map flatten(Map map, boolean ordered, String parentPath) { - Map flatMap = ordered ? new TreeMap() : new HashMap<>(); + private static Map flatten(Map map, boolean flattenArrays, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap<>() : new HashMap<>(); String prefix = parentPath != null ? parentPath + "." : ""; - for (Map.Entry entry : map.entrySet()) { if (entry.getValue() instanceof Map) { - flatMap.putAll(flatten((Map) entry.getValue(), ordered, prefix + entry.getKey())); + flatMap.putAll(flatten((Map) entry.getValue(), flattenArrays, ordered, prefix + entry.getKey())); + } else if (flattenArrays && entry.getValue() instanceof List) { + flatMap.putAll(flatten((List) entry.getValue(), ordered, prefix + entry.getKey())); } else { flatMap.put(prefix + entry.getKey(), entry.getValue()); } } + return flatMap; + } + @SuppressWarnings("unchecked") + private static Map flatten(List list, boolean ordered, String parentPath) { + Map flatMap = ordered ? new TreeMap<>() : new HashMap<>(); + String prefix = parentPath != null ? parentPath + "." : ""; + for (int i = 0; i < list.size(); i++) { + Object cur = list.get(i); + if (cur instanceof Map) { + flatMap.putAll(flatten((Map) cur, true, ordered, prefix + i)); + } + if (cur instanceof List) { + flatMap.putAll(flatten((List) cur, ordered, prefix + i)); + } else { + flatMap.put(prefix + i, cur); + } + } return flatMap; } } diff --git a/server/src/test/java/org/elasticsearch/common/util/MapsTests.java b/server/src/test/java/org/elasticsearch/common/util/MapsTests.java index 262d9839b0ad8..23cc91357301d 100644 --- a/server/src/test/java/org/elasticsearch/common/util/MapsTests.java +++ b/server/src/test/java/org/elasticsearch/common/util/MapsTests.java @@ -15,6 +15,7 @@ import java.util.Collection; import java.util.HashMap; import java.util.HashSet; +import java.util.List; import java.util.Map; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -122,6 +123,70 @@ public void testDeepEquals() { assertFalse(Maps.deepEquals(map, mapModified)); } + public void testFlatten() { + Map map = randomNestedMap(10); + Map flatten = Maps.flatten(map, true, true); + assertThat(flatten.size(), equalTo(deepCount(map.values()))); + for (Map.Entry entry : flatten.entrySet()) { + assertThat(entry.getKey(), entry.getValue(), equalTo(deepGet(entry.getKey(), map))); + } + } + + @SuppressWarnings("unchecked") + private static Object deepGet(String path, Object obj) { + Object cur = obj; + String[] keys = path.split("\\."); + for (String key : keys) { + if (Character.isDigit(key.charAt(0))) { + List list = (List) cur; + cur = list.get(Integer.parseInt(key)); + } else { + Map map = (Map) cur; + cur = map.get(key); + } + } + return cur; + } + + @SuppressWarnings("unchecked") + private int deepCount(Collection map) { + int sum = 0; + for (Object val : map) { + if (val instanceof Map) { + sum += deepCount(((Map) val).values()); + } else if (val instanceof List) { + sum += deepCount((List) val); + } else { + sum ++; + } + } + return sum; + } + + private Map randomNestedMap(int level) { + final Supplier keyGenerator = () -> randomAlphaOfLengthBetween(1, 5); + final Supplier arrayValueGenerator = () -> random().ints(randomInt(5)) + .boxed() + .map(s -> (Object) s) + .collect(Collectors.toList()); + + final Supplier mapSupplier; + if (level > 0) { + mapSupplier = () -> randomNestedMap(level - 1); + } else { + mapSupplier = ESTestCase::randomLong; + } + final Supplier> valueSupplier = () -> randomFrom( + ESTestCase::randomBoolean, + ESTestCase::randomDouble, + ESTestCase::randomLong, + arrayValueGenerator, + mapSupplier + ); + final Supplier valueGenerator = () -> valueSupplier.get().get(); + return randomMap(randomInt(5), keyGenerator, valueGenerator); + } + private void assertMapEntries(final Map map, final Collection> entries) { for (var entry : entries) { assertThat("map [" + map + "] does not contain key [" + entry.getKey() + "]", map.keySet(), hasItem(entry.getKey())); diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java index 7cad6cf043a9c..2ac20b9bba1ad 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java @@ -43,6 +43,7 @@ public static void addToXContentToFeature(VectorTile.Tile.Feature.Builder featur throws IOException { final Map map = Maps.flatten( XContentHelper.convertToMap(XContentHelper.toXContent(toXContent, XContentType.CBOR, false), true, XContentType.CBOR).v2(), + true, true ); for (Map.Entry entry : map.entrySet()) { diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java index 48fd9887409f0..b312ad15d8d47 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java @@ -132,7 +132,7 @@ public void testBasicGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 13); } public void testEmpty() throws Exception { @@ -140,7 +140,7 @@ public void testEmpty() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(1)); - assertLayer(tile, META_LAYER, 4096, 1, 12); + assertLayer(tile, META_LAYER, 4096, 1, 8); } public void testGridPrecision() throws Exception { @@ -151,7 +151,7 @@ public void testGridPrecision() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 13); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -169,7 +169,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 13); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -178,7 +178,7 @@ public void testGridType() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 13); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -194,7 +194,7 @@ public void testNoAggLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); - assertLayer(tile, META_LAYER, 4096, 1, 11); + assertLayer(tile, META_LAYER, 4096, 1, 9); } public void testNoHitsLayer() throws Exception { @@ -203,7 +203,7 @@ public void testNoHitsLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 14); + assertLayer(tile, META_LAYER, 4096, 1, 12); } public void testBasicQueryGet() throws Exception { @@ -221,7 +221,7 @@ public void testBasicQueryGet() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 13); } public void testBasicShape() throws Exception { @@ -230,7 +230,9 @@ public void testBasicShape() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + // The same max and min values are present in all buckets so min and max aggs return all buckets as max and mins + // as an array at the moment hence 2 additional values and 256 * 256 * 2 additional keys + assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2); } public void testWithFields() throws Exception { @@ -240,7 +242,7 @@ public void testWithFields() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 3); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 15); + assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2); } public void testMinAgg() throws Exception { @@ -258,7 +260,7 @@ public void testMinAgg() throws Exception { assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); - assertLayer(tile, META_LAYER, 4096, 1, 19); + assertLayer(tile, META_LAYER, 4096, 1, 9 + 4 + 256 * 256 * 4); } private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { From 2fe2ee73405165c521006bd566bfebd32cc93150 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Tue, 4 May 2021 05:44:19 -1000 Subject: [PATCH 08/15] Vector Tiles: Add support for feature flag (#72657) Adds support for future development under a feature flag --- x-pack/plugin/spatial/build.gradle | 11 ++++ .../xpack/spatial/SpatialPlugin.java | 53 +++++++++++++++---- 2 files changed, 54 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index ddd076c4f9983..7d24ffeb54142 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -1,3 +1,5 @@ +import org.elasticsearch.gradle.internal.info.BuildParams; + apply plugin: 'elasticsearch.esplugin' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.yaml-rest-test' @@ -35,6 +37,15 @@ testClusters.all { setting 'xpack.license.self_generated.type', 'trial' testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } +} + +tasks.named("test").configure { + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } } tasks.named("thirdPartyAudit").configure { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index f9316ef7d6f6a..eac7ba91d621b 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.spatial; +import org.elasticsearch.Build; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; @@ -44,8 +45,6 @@ import org.elasticsearch.xpack.spatial.action.SpatialInfoTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialStatsTransportAction; import org.elasticsearch.xpack.spatial.action.SpatialUsageTransportAction; -import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnBoundedGeoTileGridTiler; -import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.PointFieldMapper; import org.elasticsearch.xpack.spatial.index.mapper.ShapeFieldMapper; @@ -61,12 +60,15 @@ import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeCellIdSource; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeHashGridAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoShapeTileGridAggregator; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnBoundedGeoTileGridTiler; +import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.UnboundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeCentroidAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction; +import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; @@ -85,6 +87,30 @@ protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } + private static final Boolean VECTOR_TILE_FEATURE_FLAG_REGISTERED; + + static { + final String property = System.getProperty("es.vector_tile_feature_flag_registered"); + if (Build.CURRENT.isSnapshot() && property != null) { + throw new IllegalArgumentException("es.vector_tile_feature_flag_registered is only supported in non-snapshot builds"); + } + if ("true".equals(property)) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = true; + } else if ("false".equals(property)) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = false; + } else if (property == null) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = null; + } else { + throw new IllegalArgumentException( + "expected es.vector_tile_feature_flag_registered to be unset or [true|false] but was [" + property + "]" + ); + } + } + + public boolean isVectorTileEnabled() { + return Build.CURRENT.isSnapshot() || (VECTOR_TILE_FEATURE_FLAG_REGISTERED != null && VECTOR_TILE_FEATURE_FLAG_REGISTERED); + } + @Override public List> getActions() { return Arrays.asList( @@ -98,8 +124,11 @@ public List getRestHandlers(Settings settings, RestController restC IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, IndexNameExpressionResolver indexNameExpressionResolver, Supplier nodesInCluster) { - return Arrays.asList( - new RestVectorTileAction()); + if (isVectorTileEnabled()) { + return List.of(new RestVectorTileAction()); + } else { + return List.of(); + } } @Override @@ -136,22 +165,26 @@ public List> getAggregationExtentions() { @Override public List getAggregations() { - return List.of( + List aggSpecs = new ArrayList<>(); + aggSpecs.add( new AggregationSpec( GeoLineAggregationBuilder.NAME, GeoLineAggregationBuilder::new, usage.track(SpatialStatsAction.Item.GEOLINE, checkLicense(GeoLineAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) .addResultReader(InternalGeoLine::new) - .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage), - new AggregationSpec( - VectorTileAggregationBuilder.NAME, + .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage) + + ); + if (isVectorTileEnabled()) { + aggSpecs.add(new AggregationSpec(VectorTileAggregationBuilder.NAME, VectorTileAggregationBuilder::new, VectorTileAggregationBuilder.PARSER) // usage.track(SpatialStatsAction.Item.GEOLINE, // checkLicense(VectorTileAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) - .addResultReader(InternalVectorTile::new) - .setAggregatorRegistrar(VectorTileAggregationBuilder::registerAggregators)); + .addResultReader(InternalVectorTile::new).setAggregatorRegistrar(VectorTileAggregationBuilder::registerAggregators)); + } + return aggSpecs; } @Override From 2b275613d5770697aafb5ce106bb508090d110d8 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Wed, 2 Jun 2021 06:50:33 +0200 Subject: [PATCH 09/15] Move vector tile implementation to its own module (#73562) In addition, a few things have been added: * The API now handles cancellation which it turned to be a big performance booster. * The API supports defining a sort field. * RestVectorTileAction has been refactor to one class again. * VectorTileRequest class has been extracted so it can be unit tested. --- x-pack/plugin/spatial/build.gradle | 101 ------ .../xpack/spatial/SpatialPlugin.java | 97 +---- .../AbstractVectorTileAggregator.java | 161 -------- .../aggregations/InternalVectorTile.java | 228 ------------ .../VectorTileAggregationBuilder.java | 189 ---------- .../VectorTileAggregatorFactory.java | 68 ---- .../VectorTileAggregatorSupplier.java | 30 -- .../VectorTileGeoPointAggregator.java | 58 --- .../VectorTileGeoShapeAggregator.java | 137 ------- .../AbstractVectorTileSearchAction.java | 343 ------------------ .../spatial/vectortile/PointFactory.java | 105 ------ .../vectortile/RestVectorTileAction.java | 248 ------------- .../aggregations/InternalVectorTileTests.java | 185 ---------- .../VectorTileAggregationBuilderTests.java | 45 --- .../VectorTileAggregatorTests.java | 239 ------------ x-pack/plugin/vector-tile/build.gradle | 69 ++++ .../mapbox-vector-tile-3.1.0.jar.sha1 | 0 .../licenses/mapbox-vector-tile-LICENSE.txt | 0 .../licenses/mapbox-vector-tile-NOTICE.txt | 0 .../licenses/protobuf-java-3.14.0.jar.sha1 | 0 .../licenses/protobuf-java-LICENSE.txt | 0 .../licenses/protobuf-java-NOTICE.txt | 0 .../xpack/vectortile}/VectorTileRestIT.java | 240 ++++++++---- .../xpack/vectortile/VectorTilePlugin.java | 90 +++++ .../vectortile/feature}/FeatureFactory.java | 42 ++- .../feature/FeatureFactoryUtils.java | 63 ++++ .../feature/SimpleFeatureFactory.java} | 22 +- .../vectortile/rest/RestVectorTileAction.java | 300 +++++++++++++++ .../vectortile/rest/VectorTileRequest.java | 309 ++++++++++++++++ .../vectortile/rest}/VectorTileUtils.java | 58 +-- .../feature/FeatureFactoryTests.java | 65 ++++ .../rest/VectorTileRequestTests.java | 213 +++++++++++ 32 files changed, 1343 insertions(+), 2362 deletions(-) delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java delete mode 100644 x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java delete mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java delete mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java delete mode 100644 x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java create mode 100644 x-pack/plugin/vector-tile/build.gradle rename x-pack/plugin/{spatial => vector-tile}/licenses/mapbox-vector-tile-3.1.0.jar.sha1 (100%) rename x-pack/plugin/{spatial => vector-tile}/licenses/mapbox-vector-tile-LICENSE.txt (100%) rename x-pack/plugin/{spatial => vector-tile}/licenses/mapbox-vector-tile-NOTICE.txt (100%) rename x-pack/plugin/{spatial => vector-tile}/licenses/protobuf-java-3.14.0.jar.sha1 (100%) rename x-pack/plugin/{spatial => vector-tile}/licenses/protobuf-java-LICENSE.txt (100%) rename x-pack/plugin/{spatial => vector-tile}/licenses/protobuf-java-NOTICE.txt (100%) rename x-pack/plugin/{spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial => vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile}/VectorTileRestIT.java (61%) create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java rename x-pack/plugin/{spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile => vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature}/FeatureFactory.java (82%) create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java rename x-pack/plugin/{spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java => vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java} (78%) create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java create mode 100644 x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java rename x-pack/plugin/{spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile => vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest}/VectorTileUtils.java (59%) create mode 100644 x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java create mode 100644 x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index 1ad39a36adbe0..acd8d27d5a7e5 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -1,5 +1,3 @@ -import org.elasticsearch.gradle.internal.info.BuildParams; - apply plugin: 'elasticsearch.internal-es-plugin' apply plugin: 'elasticsearch.internal-cluster-test' apply plugin: 'elasticsearch.yaml-rest-test' @@ -16,80 +14,9 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) - api project(path: ':modules:geo') restTestConfig project(path: ':modules:geo', configuration: 'restTests') - api "com.wdtinc:mapbox-vector-tile:3.1.0" - api "com.google.protobuf:protobuf-java:3.14.0" - yamlRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") - yamlRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0") -} - -restResources { - restApi { - include '_common', 'bulk', 'indices', 'index', 'search', 'xpack' - } - restTests { - includeCore 'geo_shape' - } -} - -testClusters.all { - setting 'xpack.license.self_generated.type', 'trial' - testDistribution = 'DEFAULT' - setting 'xpack.security.enabled', 'false' - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.vector_tile_feature_flag_registered', 'true' - } -} - -tasks.named("test").configure { - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.vector_tile_feature_flag_registered', 'true' - } -} - -tasks.named("thirdPartyAudit").configure { - ignoreViolations( - // uses internal java api: sun.misc.Unsafe - 'com.google.protobuf.UnsafeUtil', - 'com.google.protobuf.MessageSchema', - 'com.google.protobuf.UnsafeUtil$1', - 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', - 'com.google.protobuf.UnsafeUtil$MemoryAccessor' - ) - - ignoreMissingClasses( - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory' - ) -} - -import org.elasticsearch.gradle.internal.info.BuildParams; - -apply plugin: 'elasticsearch.esplugin' -apply plugin: 'elasticsearch.internal-cluster-test' -apply plugin: 'elasticsearch.yaml-rest-test' -apply plugin: 'elasticsearch.yaml-rest-compat-test' - -esplugin { - name 'spatial' - description 'A plugin for Basic Spatial features' - classname 'org.elasticsearch.xpack.spatial.SpatialPlugin' - extendedPlugins = ['x-pack-core'] -} - -dependencies { - compileOnly project(path: xpackModule('core')) - testImplementation(testArtifact(project(xpackModule('core')))) - yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) api project(path: ':modules:geo') restTestConfig project(path: ':modules:geo', configuration: 'restTests') - api "com.wdtinc:mapbox-vector-tile:3.1.0" - api "com.google.protobuf:protobuf-java:3.14.0" - yamlRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") - yamlRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0") } restResources { @@ -105,32 +32,4 @@ testClusters.all { setting 'xpack.license.self_generated.type', 'trial' testDistribution = 'DEFAULT' setting 'xpack.security.enabled', 'false' - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.vector_tile_feature_flag_registered', 'true' - } -} - -tasks.named("test").configure { - if (BuildParams.isSnapshotBuild() == false) { - systemProperty 'es.vector_tile_feature_flag_registered', 'true' - } -} - -tasks.named("thirdPartyAudit").configure { - ignoreViolations( - // uses internal java api: sun.misc.Unsafe - 'com.google.protobuf.UnsafeUtil', - 'com.google.protobuf.MessageSchema', - 'com.google.protobuf.UnsafeUtil$1', - 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', - 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', - 'com.google.protobuf.UnsafeUtil$MemoryAccessor' - ) - - ignoreMissingClasses( - 'org.slf4j.Logger', - 'org.slf4j.LoggerFactory' - ) } - diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java index 2ed0ff074f816..144b3a874f940 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java +++ b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/SpatialPlugin.java @@ -6,15 +6,8 @@ */ package org.elasticsearch.xpack.spatial; -import org.elasticsearch.Build; import org.elasticsearch.action.ActionRequest; import org.elasticsearch.action.ActionResponse; -import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; -import org.elasticsearch.cluster.node.DiscoveryNodes; -import org.elasticsearch.common.settings.ClusterSettings; -import org.elasticsearch.common.settings.IndexScopedSettings; -import org.elasticsearch.common.settings.Settings; -import org.elasticsearch.common.settings.SettingsFilter; import org.elasticsearch.common.xcontent.ContextParser; import org.elasticsearch.geo.GeoPlugin; import org.elasticsearch.index.mapper.Mapper; @@ -25,8 +18,6 @@ import org.elasticsearch.plugins.IngestPlugin; import org.elasticsearch.plugins.MapperPlugin; import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.rest.RestController; -import org.elasticsearch.rest.RestHandler; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoHashGridAggregationBuilder; import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; @@ -36,8 +27,6 @@ import org.elasticsearch.search.aggregations.metrics.ValueCountAggregationBuilder; import org.elasticsearch.search.aggregations.metrics.ValueCountAggregator; import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; -import org.elasticsearch.threadpool.ExecutorBuilder; -import org.elasticsearch.threadpool.FixedExecutorBuilder; import org.elasticsearch.xpack.core.XPackPlugin; import org.elasticsearch.xpack.core.action.XPackInfoFeatureAction; import org.elasticsearch.xpack.core.action.XPackUsageFeatureAction; @@ -55,8 +44,6 @@ import org.elasticsearch.xpack.spatial.ingest.CircleProcessor; import org.elasticsearch.xpack.spatial.search.aggregations.GeoLineAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.InternalGeoLine; -import org.elasticsearch.xpack.spatial.search.aggregations.InternalVectorTile; -import org.elasticsearch.xpack.spatial.search.aggregations.VectorTileAggregationBuilder; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoHashGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.BoundedGeoTileGridTiler; import org.elasticsearch.xpack.spatial.search.aggregations.bucket.geogrid.GeoGridTiler; @@ -66,76 +53,30 @@ import org.elasticsearch.xpack.spatial.search.aggregations.metrics.GeoShapeBoundsAggregator; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; -import org.elasticsearch.xpack.spatial.vectortile.RestVectorTileAction; -import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.function.Consumer; -import java.util.function.Supplier; import static java.util.Collections.singletonList; public class SpatialPlugin extends GeoPlugin implements ActionPlugin, MapperPlugin, SearchPlugin, IngestPlugin { - private final SpatialUsage usage = new SpatialUsage(); + private final SpatialUsage usage = new SpatialUsage(); // to be overriden by tests protected XPackLicenseState getLicenseState() { return XPackPlugin.getSharedLicenseState(); } - private static final Boolean VECTOR_TILE_FEATURE_FLAG_REGISTERED; - - static { - final String property = System.getProperty("es.vector_tile_feature_flag_registered"); - if (Build.CURRENT.isSnapshot() && property != null) { - throw new IllegalArgumentException("es.vector_tile_feature_flag_registered is only supported in non-snapshot builds"); - } - if ("true".equals(property)) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = true; - } else if ("false".equals(property)) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = false; - } else if (property == null) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = null; - } else { - throw new IllegalArgumentException( - "expected es.vector_tile_feature_flag_registered to be unset or [true|false] but was [" + property + "]" - ); - } - } - - public boolean isVectorTileEnabled() { - return Build.CURRENT.isSnapshot() || (VECTOR_TILE_FEATURE_FLAG_REGISTERED != null && VECTOR_TILE_FEATURE_FLAG_REGISTERED); - } - @Override - public List> getActions() { + public List> getActions() { return Arrays.asList( - new ActionHandler<>(XPackUsageFeatureAction.SPATIAL, SpatialUsageTransportAction.class), - new ActionHandler<>(XPackInfoFeatureAction.SPATIAL, SpatialInfoTransportAction.class), - new ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class)); - } - - @Override - public List getRestHandlers(Settings settings, RestController restController, ClusterSettings clusterSettings, - IndexScopedSettings indexScopedSettings, SettingsFilter settingsFilter, - IndexNameExpressionResolver indexNameExpressionResolver, - Supplier nodesInCluster) { - if (isVectorTileEnabled()) { - return List.of(new RestVectorTileAction()); - } else { - return List.of(); - } - } - - @Override - public List> getExecutorBuilders(Settings settings) { - FixedExecutorBuilder indexing = - new FixedExecutorBuilder(settings, "vector_tile_generation", 1, -1, "thread_pool.vectortile", false); - return Collections.singletonList(indexing); + new ActionPlugin.ActionHandler<>(XPackUsageFeatureAction.SPATIAL, SpatialUsageTransportAction.class), + new ActionPlugin.ActionHandler<>(XPackInfoFeatureAction.SPATIAL, SpatialInfoTransportAction.class), + new ActionPlugin.ActionHandler<>(SpatialStatsAction.INSTANCE, SpatialStatsTransportAction.class)); } @Override @@ -165,26 +106,14 @@ public List> getAggregationExtentions() { @Override public List getAggregations() { - List aggSpecs = new ArrayList<>(); - aggSpecs.add( + return List.of( new AggregationSpec( - GeoLineAggregationBuilder.NAME, - GeoLineAggregationBuilder::new, - usage.track(SpatialStatsAction.Item.GEOLINE, - checkLicense(GeoLineAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) + GeoLineAggregationBuilder.NAME, + GeoLineAggregationBuilder::new, + usage.track(SpatialStatsAction.Item.GEOLINE, + checkLicense(GeoLineAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) .addResultReader(InternalGeoLine::new) - .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage) - - ); - if (isVectorTileEnabled()) { - aggSpecs.add(new AggregationSpec(VectorTileAggregationBuilder.NAME, - VectorTileAggregationBuilder::new, - VectorTileAggregationBuilder.PARSER) - // usage.track(SpatialStatsAction.Item.GEOLINE, - // checkLicense(VectorTileAggregationBuilder.PARSER, XPackLicenseState.Feature.SPATIAL_GEO_LINE))) - .addResultReader(InternalVectorTile::new).setAggregatorRegistrar(VectorTileAggregationBuilder::registerAggregators)); - } - return aggSpecs; + .setAggregatorRegistrar(GeoLineAggregationBuilder::registerUsage)); } @Override @@ -217,7 +146,7 @@ private void registerGeoShapeCentroidAggregator(ValuesSourceRegistry.Builder bui private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builder) { builder.register(GeoHashGridAggregationBuilder.REGISTRY_KEY, GeoShapeValuesSourceType.instance(), (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, - aggregationContext, parent, collectsFromSingleBucket, metadata) -> { + aggregationContext, parent, collectsFromSingleBucket, metadata) -> { if (getLicenseState().checkFeature(XPackLicenseState.Feature.SPATIAL_GEO_GRID)) { final GeoGridTiler tiler; if (geoBoundingBox.isUnbounded()) { @@ -239,7 +168,7 @@ private void registerGeoShapeGridAggregators(ValuesSourceRegistry.Builder builde builder.register(GeoTileGridAggregationBuilder.REGISTRY_KEY, GeoShapeValuesSourceType.instance(), (name, factories, valuesSource, precision, geoBoundingBox, requiredSize, shardSize, - context, parent, collectsFromSingleBucket, metadata) -> { + context, parent, collectsFromSingleBucket, metadata) -> { if (getLicenseState().checkFeature(XPackLicenseState.Feature.SPATIAL_GEO_GRID)) { final GeoGridTiler tiler; if (geoBoundingBox.isUnbounded()) { diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java deleted file mode 100644 index 13cf03f91f7e8..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/AbstractVectorTileAggregator.java +++ /dev/null @@ -1,161 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.search.aggregations; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.apache.lucene.search.ScoreMode; -import org.elasticsearch.common.lease.Releasables; -import org.elasticsearch.common.util.LongArray; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.search.aggregations.metrics.MetricsAggregator; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.xpack.spatial.vectortile.VectorTileUtils; - -import java.io.IOException; -import java.util.List; -import java.util.Map; - -/** - * Base Vector tile aggregator class. It might generate up to three layers: If there is any point it generates a POINT layer - * that cluster points in a cluster them in a 256 X 256 grid. If there is any line, it generated a LINE layer with a 4096 extent - * contains ing all lines. If there is any polygon, it generated a POLYGON layer with a 4096 extent contains ing all lines. - */ -public abstract class AbstractVectorTileAggregator extends MetricsAggregator { - - protected static final int POINT_EXTENT = 256; - protected static final String POINT_LAYER = "POINT"; - protected static final int LINE_EXTENT = 4096; - protected static final String LINE_LAYER = "LINE"; - protected static final int POLYGON_EXTENT = 4096; - protected static final String POLYGON_LAYER = "POLYGON"; - - protected static final String ID_TAG = "id"; - - protected final ValuesSource valuesSource; - protected final int x; - protected final int y; - protected final int z; - private VectorTile.Tile.Layer.Builder polygonLayerBuilder; - private int numPolygons; - private VectorTile.Tile.Layer.Builder lineLayerBuilder; - private int numLines; - private LongArray clusters; - private final double pointXScale; - private final double pointYScale; - protected final Rectangle rectangle; - final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); - - public AbstractVectorTileAggregator( - String name, - ValuesSourceConfig valuesSourceConfig, - int z, - int x, - int y, - AggregationContext context, - Aggregator parent, - Map metadata - ) throws IOException { - super(name, context, parent, metadata); - this.valuesSource = valuesSourceConfig.getValuesSource(); - this.z = z; - this.x = x; - this.y = y; - this.rectangle = VectorTileUtils.getTileBounds(z, x, y); - this.pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) POINT_EXTENT); - this.pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) POINT_EXTENT); - } - - @Override - public ScoreMode scoreMode() { - return valuesSource != null && valuesSource.needsScores() ? ScoreMode.COMPLETE : ScoreMode.COMPLETE_NO_SCORES; - } - - protected void addPoint(double lat, double lon) { - final int x = (int) (pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); - final int y = (int) (pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + POINT_EXTENT; - if (x >= 0 && x < POINT_EXTENT && y >= 0 && y < POINT_EXTENT) { - if (clusters == null) { - clusters = bigArrays().newLongArray(POINT_EXTENT * POINT_EXTENT); - } - final int pos = POINT_EXTENT * x + y; - clusters.increment(pos, 1); - } - } - - protected void addLineFeatures(String id, List features) { - for (VectorTile.Tile.Feature feature : features) { - if (lineLayerBuilder == null) { - lineLayerBuilder = VectorTile.Tile.Layer.newBuilder(); - lineLayerBuilder.setVersion(2); - lineLayerBuilder.setName(LINE_LAYER); - lineLayerBuilder.setExtent(LINE_EXTENT); - lineLayerBuilder.addKeys(ID_TAG); - } - valueBuilder.clear(); - valueBuilder.setStringValue(id); - lineLayerBuilder.addValues(valueBuilder); - VectorTile.Tile.Feature.Builder builder = feature.toBuilder(); - builder.addTags(0); - builder.addTags(numLines++); - lineLayerBuilder.addFeatures(builder); - } - } - - protected void addPolygonFeatures(String id, List features) { - for (VectorTile.Tile.Feature feature : features) { - if (polygonLayerBuilder == null) { - polygonLayerBuilder = VectorTile.Tile.Layer.newBuilder(); - polygonLayerBuilder.setVersion(2); - polygonLayerBuilder.setName(POLYGON_LAYER); - polygonLayerBuilder.setExtent(POLYGON_EXTENT); - polygonLayerBuilder.addKeys(ID_TAG); - } - valueBuilder.clear(); - valueBuilder.setStringValue(id); - polygonLayerBuilder.addValues(valueBuilder); - VectorTile.Tile.Feature.Builder builder = feature.toBuilder(); - builder.addTags(0); - builder.addTags(numPolygons++); - polygonLayerBuilder.addFeatures(builder); - } - } - - - - @Override - public InternalAggregation buildAggregation(long bucket) { - if (valuesSource == null) { - return buildEmptyAggregation(); - } - final VectorTile.Tile.Layer polygons = polygonLayerBuilder != null ? polygonLayerBuilder.build() : null; - final VectorTile.Tile.Layer lines = lineLayerBuilder != null ? lineLayerBuilder.build() : null; - final long[] points; - if (clusters != null) { - points = new long[POINT_EXTENT * POINT_EXTENT]; - for (int i = 0; i < POINT_EXTENT * POINT_EXTENT; i++) { - points[i] = clusters.get(i); - } - } else { - points = null; - } - return new InternalVectorTile(name, polygons, lines, points, metadata()); - } - - @Override - public InternalAggregation buildEmptyAggregation() { - return new InternalVectorTile(name, null, null, null, metadata()); - } - - @Override - public void doClose() { - Releasables.close(clusters); - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java deleted file mode 100644 index 591f900c4433a..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTile.java +++ /dev/null @@ -1,228 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.search.aggregations; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; -import com.wdtinc.mapbox_vector_tile.encoding.GeomCmdHdr; -import org.apache.lucene.util.BitUtil; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.aggregations.InternalAggregation; -import org.elasticsearch.xpack.spatial.vectortile.VectorTileUtils; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Objects; - -public class InternalVectorTile extends InternalAggregation { - private static String COUNT_KEY = "count"; - protected final VectorTile.Tile.Layer polygons; - protected final VectorTile.Tile.Layer lines; - protected final long[] points; - - public InternalVectorTile(String name, VectorTile.Tile.Layer polygons, - VectorTile.Tile.Layer lines, long[] points, Map metadata) { - super(name, metadata); - this.polygons = polygons; - this.lines = lines; - this.points = points; - } - - /** - * Read from a stream. - */ - public InternalVectorTile(StreamInput in) throws IOException { - super(in); - this.polygons = in.readBoolean() ? VectorTile.Tile.Layer.parseDelimitedFrom(in) : null; - this.lines = in.readBoolean() ? VectorTile.Tile.Layer.parseDelimitedFrom(in) : null; - this.points = in.readBoolean() ? in.readLongArray() : null; - } - - @Override - protected void doWriteTo(StreamOutput out) throws IOException { - if (polygons != null) { - out.writeBoolean(true); - polygons.writeDelimitedTo(out); - } else { - out.writeBoolean(false); - } - if (lines != null) { - out.writeBoolean(true); - lines.writeDelimitedTo(out); - } else { - out.writeBoolean(false); - } - if (points != null) { - out.writeBoolean(true); - out.writeLongArray(points); - } else { - out.writeBoolean(false); - } - } - - @Override - public String getWriteableName() { - return VectorTileAggregationBuilder.NAME; - } - - public byte[] getVectorTileBytes() { - return getVectorTile().toByteArray(); - } - - public void writeTileToStream(OutputStream stream) throws IOException { - getVectorTile().writeTo(stream); - } - - private VectorTile.Tile getVectorTile() { - final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - if (polygons != null) { - tileBuilder.addLayers(polygons); - } - if (lines != null) { - tileBuilder.addLayers(lines); - } - if (points != null) { - tileBuilder.addLayers(buildPointLayer()); - } - return tileBuilder.build(); - } - - private VectorTile.Tile.Layer buildPointLayer() { - final VectorTile.Tile.Layer.Builder pointLayerBuilder = - VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.POINT_LAYER, AbstractVectorTileAggregator.POINT_EXTENT); - pointLayerBuilder.addKeys(COUNT_KEY); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - final VectorTile.Tile.Value.Builder valueBuilder = VectorTile.Tile.Value.newBuilder(); - final HashMap values = new HashMap<>(); - for (int i = 0; i < AbstractVectorTileAggregator.POINT_EXTENT; i++) { - final int xVal = i * AbstractVectorTileAggregator.POINT_EXTENT; - for (int j = 0; j < AbstractVectorTileAggregator.POINT_EXTENT; j++) { - final long count = points[xVal + j]; - if (count > 0) { - featureBuilder.clear(); - featureBuilder.setType(VectorTile.Tile.GeomType.POINT); - // create geometry commands - featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.MoveTo, 1)); - featureBuilder.addGeometry(BitUtil.zigZagEncode(i)); - featureBuilder.addGeometry(BitUtil.zigZagEncode(j)); - // Add count as key value pair - featureBuilder.addTags(0); - final int tagValue; - if (values.containsKey(count)) { - tagValue = values.get(count); - } else { - valueBuilder.clear(); - valueBuilder.setIntValue(count); - tagValue = values.size(); - pointLayerBuilder.addValues(valueBuilder); - values.put(count, tagValue); - } - featureBuilder.addTags(tagValue); - pointLayerBuilder.addFeatures(featureBuilder); - } - } - } - return pointLayerBuilder.build(); - } - - @Override - public InternalAggregation reduce(List aggregations, ReduceContext reduceContext) { - VectorTile.Tile.Layer.Builder polygons = null; - VectorTile.Tile.Layer.Builder lines = null; - long[] points = null; - for (InternalAggregation aggregation : aggregations) { - InternalVectorTile internalVectorTile = (InternalVectorTile) aggregation; - if (internalVectorTile.polygons != null) { - if (polygons == null) { - polygons = - VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.POLYGON_LAYER, - AbstractVectorTileAggregator.POLYGON_EXTENT - ); - polygons.addKeys(AbstractVectorTileAggregator.ID_TAG); - } - mergeLayer(polygons, internalVectorTile.polygons); - } - if (internalVectorTile.lines != null) { - if (lines == null) { - lines = VectorTileUtils.createLayerBuilder(AbstractVectorTileAggregator.LINE_LAYER, - AbstractVectorTileAggregator.LINE_EXTENT - ); - lines.addKeys(AbstractVectorTileAggregator.ID_TAG); - } - mergeLayer(lines, internalVectorTile.lines); - } - if (internalVectorTile.points != null) { - if (points == null) { - points = new long[AbstractVectorTileAggregator.POINT_EXTENT * AbstractVectorTileAggregator.POINT_EXTENT]; - } - mergePoints(points, internalVectorTile.points); - } - } - final VectorTile.Tile.Layer polygonLayer = polygons != null ? polygons.build() : null; - final VectorTile.Tile.Layer lineLayer = lines != null ? lines.build() : null; - return new InternalVectorTile(name, polygonLayer, lineLayer, points, getMetadata()); - } - - private void mergeLayer(VectorTile.Tile.Layer.Builder layerBuilder, VectorTile.Tile.Layer layer) { - layerBuilder.addAllKeys(layer.getKeysList()); - layerBuilder.addAllValues(layer.getValuesList()); - for(int i = 0; i < layer.getFeaturesCount(); i++) { - layerBuilder.addFeatures(layer.getFeatures(i)); - } - } - - private void mergePoints(long[] pointsBuilder, long[] layer) { - for(int i = 0; i < pointsBuilder.length; i++) { - pointsBuilder[i] += layer[i]; - } - } - - @Override - protected boolean mustReduceOnSingleInternalAgg() { - return false; - } - - @Override - public Object getProperty(List path) { - if (path.isEmpty()) { - return this; - } else if (path.size() == 1 && "value".equals(path.get(0))) { - return getVectorTileBytes(); - } else { - throw new IllegalArgumentException("path not supported for [" + getName() + "]: " + path); - } - } - - @Override - public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - builder.field(CommonFields.VALUE.getPreferredName(), getVectorTileBytes()); - return builder; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), polygons, lines, Arrays.hashCode(points)); - } - - @Override - public boolean equals(Object obj) { - if (this == obj) return true; - if (obj == null || getClass() != obj.getClass()) return false; - if (super.equals(obj) == false) return false; - - InternalVectorTile that = (InternalVectorTile) obj; - return Arrays.equals(points, that.points) && - Objects.equals(polygons, that.polygons) && - Objects.equals(lines, that.lines); - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java deleted file mode 100644 index 09bab2a7741b7..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilder.java +++ /dev/null @@ -1,189 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.search.aggregations; - -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.io.stream.StreamInput; -import org.elasticsearch.common.io.stream.StreamOutput; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentBuilder; -import org.elasticsearch.search.DocValueFormat; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSourceAggregationBuilder; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.aggregations.support.ValuesSourceRegistry; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; - -import java.io.IOException; -import java.util.Collections; -import java.util.Map; -import java.util.Objects; - -/** - * Aggregates geo data in a vector tile.It currently only supports to be o the top-level (e.g cannot have a parent) - * but it can change. - */ -public class VectorTileAggregationBuilder extends ValuesSourceAggregationBuilder.LeafOnly { - public static final String NAME = "vector-tile"; - public static final ParseField ZOOM_FIELD = new ParseField("z"); - public static final ParseField X_FIELD = new ParseField("x"); - public static final ParseField Y_FIELD = new ParseField("y"); - public static final ValuesSourceRegistry.RegistryKey REGISTRY_KEY = - new ValuesSourceRegistry.RegistryKey<>( - NAME, - VectorTileAggregatorSupplier.class - ); - public static final ObjectParser PARSER = - ObjectParser.fromBuilder(NAME, VectorTileAggregationBuilder::new); - - static { - ValuesSourceAggregationBuilder.declareFields(PARSER, false, false, false, false); - PARSER.declareInt(VectorTileAggregationBuilder::z, ZOOM_FIELD); - PARSER.declareInt(VectorTileAggregationBuilder::x, X_FIELD); - PARSER.declareInt(VectorTileAggregationBuilder::x, Y_FIELD); - } - - private int z; - private int x; - private int y; - - public static void registerAggregators(ValuesSourceRegistry.Builder builder) { - builder.register( - VectorTileAggregationBuilder.REGISTRY_KEY, - Collections.singletonList(CoreValuesSourceType.GEOPOINT), - VectorTileGeoPointAggregator::new, - true - ); - builder.register( - VectorTileAggregationBuilder.REGISTRY_KEY, - Collections.singletonList(GeoShapeValuesSourceType.instance()), - VectorTileGeoShapeAggregator::new, - true - ); - } - - public VectorTileAggregationBuilder(String name) { - super(name); - } - - protected VectorTileAggregationBuilder( - VectorTileAggregationBuilder clone, - AggregatorFactories.Builder factoriesBuilder, - Map metadata - ) { - super(clone, factoriesBuilder, metadata); - this.x = clone.x; - this.y = clone.y; - this.z = clone.z; - } - - @Override - protected AggregationBuilder shallowCopy(AggregatorFactories.Builder factoriesBuilder, Map metadata) { - return new VectorTileAggregationBuilder(this, factoriesBuilder, metadata); - } - - /** - * Read from a stream. - */ - public VectorTileAggregationBuilder(StreamInput in) throws IOException { - super(in); - z = in.readVInt(); - x = in.readVInt(); - y = in.readVInt(); - } - - @Override - protected ValuesSourceType defaultValueSourceType() { - return CoreValuesSourceType.GEOPOINT; - } - - @Override - protected void innerWriteTo(StreamOutput out) throws IOException { - out.writeVInt(z); - out.writeVInt(x); - out.writeVInt(y); - } - - @Override - protected ValuesSourceRegistry.RegistryKey getRegistryKey() { - return REGISTRY_KEY; - } - - @Override - protected VectorTileAggregatorFactory innerBuild( - AggregationContext context, - ValuesSourceConfig config, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder) throws IOException { - if (parent != null) { - // we don't allow vector-tile aggregations to be the child of a bucketing aggregation - throw new IllegalArgumentException(NAME + " aggregation must be at the top level"); - } - VectorTileAggregatorSupplier aggregatorSupplier = - context.getValuesSourceRegistry().getAggregator(REGISTRY_KEY, config); - return new VectorTileAggregatorFactory(name, config, z, x, y, context, parent, - subFactoriesBuilder, metadata, aggregatorSupplier); - } - - @Override - public XContentBuilder doXContentBody(XContentBuilder builder, Params params) throws IOException { - builder.field(ZOOM_FIELD.getPreferredName(), z); - builder.field(X_FIELD.getPreferredName(), x); - builder.field(Y_FIELD.getPreferredName(), y); - return builder; - } - - @Override - public String getType() { - return NAME; - } - - public VectorTileAggregationBuilder z(int z) { - this.z = z; - return this; - } - - public VectorTileAggregationBuilder x(int x) { - this.x = x; - return this; - } - - public VectorTileAggregationBuilder y(int y) { - this.y = y; - return this; - } - - @Override - protected ValuesSourceConfig resolveConfig(AggregationContext context) { - // TODO: make this behavior right - if (field() == null && script() == null) { - return new ValuesSourceConfig(CoreValuesSourceType.GEOPOINT, null, true, null, null, 1.0, null, DocValueFormat.RAW, context); - } else { - return super.resolveConfig(context); - } - } - - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null || getClass() != o.getClass()) return false; - if (super.equals(o) == false) return false; - VectorTileAggregationBuilder that = (VectorTileAggregationBuilder) o; - return z == that.z && x == that.x && y == that.y; - } - - @Override - public int hashCode() { - return Objects.hash(super.hashCode(), z, x, y); - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java deleted file mode 100644 index 5b4a6c45bb043..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorFactory.java +++ /dev/null @@ -1,68 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.search.aggregations; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.AggregatorFactory; -import org.elasticsearch.search.aggregations.CardinalityUpperBound; -import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSourceAggregatorFactory; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; - -import java.io.IOException; -import java.util.Map; - -class VectorTileAggregatorFactory extends ValuesSourceAggregatorFactory { - - private final VectorTileAggregatorSupplier aggregatorSupplier; - private final int x; - private final int y; - private final int z; - - VectorTileAggregatorFactory( - String name, - ValuesSourceConfig config, - int z, - int x, - int y, - AggregationContext context, - AggregatorFactory parent, - AggregatorFactories.Builder subFactoriesBuilder, - Map metadata, - VectorTileAggregatorSupplier aggregatorSupplier - ) throws IOException { - super(name, config, context, parent, subFactoriesBuilder, metadata); - this.aggregatorSupplier = aggregatorSupplier; - this.z = z; - this.x = x; - this.y = y; - } - - @Override - protected Aggregator createUnmapped(Aggregator parent, Map metadata) throws IOException { - return new AbstractVectorTileAggregator(name, config, z, x, y, context, parent, metadata) { - @Override - protected LeafBucketCollector getLeafCollector(LeafReaderContext ctx, LeafBucketCollector sub) throws IOException { - return LeafBucketCollector.NO_OP_COLLECTOR; - } - }; - } - - @Override - protected Aggregator doCreateInternal( - Aggregator parent, - CardinalityUpperBound bucketCardinality, - Map metadata - ) throws IOException { - return aggregatorSupplier - .build(name, config, z, x, y, context, parent, metadata); - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java deleted file mode 100644 index e2e02bcef51d1..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorSupplier.java +++ /dev/null @@ -1,30 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.search.aggregations; - -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; - -import java.io.IOException; -import java.util.Map; - -@FunctionalInterface -public interface VectorTileAggregatorSupplier { - - AbstractVectorTileAggregator build( - String name, - ValuesSourceConfig valuesSourceConfig, - int z, - int x, - int y, - AggregationContext context, - Aggregator parent, - Map metadata - ) throws IOException; -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java deleted file mode 100644 index 4a961c3dddd30..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoPointAggregator.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.search.aggregations; - -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.index.fielddata.MultiGeoPointValues; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSource; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; - -import java.io.IOException; -import java.util.Map; - -/** - * Aggregator over a geo point field. - */ -public class VectorTileGeoPointAggregator extends AbstractVectorTileAggregator { - - public VectorTileGeoPointAggregator( - String name, - ValuesSourceConfig valuesSourceConfig, - int z, - int x, - int y, - AggregationContext context, - Aggregator parent, - Map metadata - ) throws IOException { - super(name, valuesSourceConfig, z, x, y, context, parent, metadata); - } - - - @Override - public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) { - final MultiGeoPointValues values = ((ValuesSource.GeoPoint) valuesSource).geoPointValues(ctx); - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - if (values.advanceExact(doc)) { - final int numPoints = values.docValueCount(); - for (int i = 0; i < numPoints; i++) { - final GeoPoint point = values.nextValue(); - addPoint(point.lat(), point.lon()); - } - } - } - }; - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java deleted file mode 100644 index 78a722861e342..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileGeoShapeAggregator.java +++ /dev/null @@ -1,137 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.search.aggregations; - -import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; -import org.apache.lucene.index.LeafReaderContext; -import org.elasticsearch.common.geo.GeometryParser; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.fieldvisitor.CustomFieldsVisitor; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.search.aggregations.Aggregator; -import org.elasticsearch.search.aggregations.LeafBucketCollector; -import org.elasticsearch.search.aggregations.LeafBucketCollectorBase; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.support.AggregationContext; -import org.elasticsearch.search.aggregations.support.ValuesSourceConfig; -import org.elasticsearch.search.lookup.SourceLookup; -import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; -import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSource; -import org.elasticsearch.xpack.spatial.vectortile.FeatureFactory; -import org.elasticsearch.xpack.spatial.vectortile.PointFactory; - -import java.io.IOException; -import java.util.List; -import java.util.Map; -import java.util.Set; - -/** - * Aggregator over a geo shape field. It skips shapes where bounding box is smaller - * than the distance between two pixels. It reads Geometry from source which is slow. - */ -public class VectorTileGeoShapeAggregator extends AbstractVectorTileAggregator { - - private final String fieldName; - private final double latPointPrecision; - private final double lonPointPrecision; - private final MappedFieldType sourceField; - private final GeometryParser parser = new GeometryParser(true, false, false); - - public VectorTileGeoShapeAggregator( - String name, - ValuesSourceConfig valuesSourceConfig, - int z, - int x, - int y, - AggregationContext context, - Aggregator parent, - Map metadata - ) throws IOException { - super(name, valuesSourceConfig, z, x, y, context, parent, metadata); - this.sourceField = context.getFieldType(SourceFieldMapper.NAME); - this.fieldName = valuesSourceConfig.fieldType().name(); - Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); - // TODO: Reason the logic for when we should skip a complex geometry - this.latPointPrecision = 2 * (rectangle.getMaxLat() - rectangle.getMinLat()) / POINT_EXTENT; - this.lonPointPrecision = 2 * (rectangle.getMaxLon() - rectangle.getMinLon()) / POINT_EXTENT; - } - - @Override - public LeafBucketCollector getLeafCollector(LeafReaderContext ctx, final LeafBucketCollector sub) { - final GeoShapeValues values = ((GeoShapeValuesSource) valuesSource).geoShapeValues(ctx); - final FeatureFactory featureFactory = new FeatureFactory(z, x, y, POLYGON_EXTENT); - final PointFactory pointFactory = new PointFactory(); - final CustomFieldsVisitor visitor = new CustomFieldsVisitor(Set.of(), true); - IUserDataConverter ignoreData = new UserDataIgnoreConverter(); - return new LeafBucketCollectorBase(sub, values) { - @Override - public void collect(int doc, long bucket) throws IOException { - if (values.advanceExact(doc)) { - GeoShapeValues.GeoShapeValue shapeValue = values.value(); - switch (shapeValue.dimensionalShapeType()) { - case POINT: { - visitor.reset(); - ctx.reader().document(doc, visitor); - final SourceLookup lookup = new SourceLookup(); - lookup.setSource(visitor.source()); - final Object pointObject = lookup.get(fieldName); - if (pointObject != null) { - final List points = pointFactory.getPoints(parser.parseGeometry(pointObject)); - for (Point point : points) { - addPoint(point.getLat(), point.getLon()); - } - } - break; - } - case LINE: { - if (skip(shapeValue)) { - addPoint(shapeValue.lat(), shapeValue.lon()); - } else { - visitor.reset(); - ctx.reader().document(doc, visitor); - final SourceLookup lookup = new SourceLookup(); - lookup.setSource(visitor.source()); - final Object lines = lookup.get(fieldName); - if (lines != null) { - addLineFeatures(visitor.id(), - featureFactory.getFeatures(parser.parseGeometry(lines), ignoreData)); - } - } - break; - } - case POLYGON: { - if (skip(shapeValue)) { - addPoint(shapeValue.lat(), shapeValue.lon()); - } else { - visitor.reset(); - ctx.reader().document(doc, visitor); - final SourceLookup lookup = new SourceLookup(); - lookup.setSource(visitor.source()); - final Object polygons = lookup.get(fieldName); - if (polygons != null) { - addPolygonFeatures(visitor.id(), - featureFactory.getFeatures(parser.parseGeometry(polygons), ignoreData)); - } - } - break; - } - } - } - } - }; - } - - private boolean skip(GeoShapeValues.GeoShapeValue shapeValue) { - final double width = shapeValue.boundingBox().maxX() - shapeValue.boundingBox().minX(); - final double height = shapeValue.boundingBox().maxY() - shapeValue.boundingBox().minY(); - return width <= this.lonPointPrecision && height <= this.latPointPrecision; - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java deleted file mode 100644 index 58c3bab938bac..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/AbstractVectorTileSearchAction.java +++ /dev/null @@ -1,343 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.vectortile; - -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.client.node.NodeClient; -import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.Strings; -import org.elasticsearch.common.io.Streams; -import org.elasticsearch.common.io.stream.BytesStream; -import org.elasticsearch.common.xcontent.ObjectParser; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.index.query.AbstractQueryBuilder; -import org.elasticsearch.index.query.BoolQueryBuilder; -import org.elasticsearch.index.query.QueryBuilder; -import org.elasticsearch.index.query.QueryBuilders; -import org.elasticsearch.rest.BaseRestHandler; -import org.elasticsearch.rest.BytesRestResponse; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.rest.RestResponse; -import org.elasticsearch.rest.RestStatus; -import org.elasticsearch.rest.action.RestResponseListener; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregatorFactories; -import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; -import org.elasticsearch.search.builder.SearchSourceBuilder; -import org.elasticsearch.search.fetch.subphase.FieldAndFormat; - -import java.io.IOException; -import java.io.OutputStream; -import java.util.ArrayList; -import java.util.Collection; -import java.util.List; -import java.util.Map; -import java.util.function.Supplier; - -import static java.util.Collections.emptyList; -import static java.util.Collections.emptyMap; - -/** - * Base class for rest actions that performs a search and translates it into - * a protobuf response - */ -public abstract class AbstractVectorTileSearchAction extends BaseRestHandler { - - private static final String INDEX_PARAM = "index"; - private static final String FIELD_PARAM = "field"; - private static final String Z_PARAM = "z"; - private static final String X_PARAM = "x"; - private static final String Y_PARAM = "y"; - private static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision"); - private static final ParseField GRID_TYPE_FIELD = new ParseField("grid_type"); - private static final ParseField EXTENT_FIELD = new ParseField("extent"); - private static final ParseField EXACT_BOUNDS_FIELD = new ParseField("exact_bounds"); - - protected enum GRID_TYPE { - GRID, POINT; - - static GRID_TYPE fromString(String type) { - switch (type) { - case "grid" : return GRID; - case "point" : return POINT; - default: throw new IllegalArgumentException("Invalid grid type [" + type + "]"); - } - } - } - - protected final ObjectParser parser; - - @FunctionalInterface - protected interface ResponseBuilder { - void buildResponse(SearchResponse searchResponse, OutputStream outputStream) throws IOException; - } - - private final Supplier emptyRequestProvider; - - protected static class Request { - private QueryBuilder queryBuilder; - private String index; - private String field; - private int x; - private int y; - private int z; - private Map runtimeMappings = emptyMap(); - private int gridPrecision = 8; - private GRID_TYPE gridType = GRID_TYPE.GRID; - private int size = 10000; - private int extent = 4096; - private AggregatorFactories.Builder aggBuilder; - private List fields = emptyList(); - private boolean exact_bounds; - - public String getIndex() { - return index; - } - - public void setIndex(String index) { - this.index = index; - } - - public String getField() { - return field; - } - - public void setField(String field) { - this.field = field; - } - - public int getX() { - return x; - } - - public void setX(int x) { - this.x = x; - } - - public int getY() { - return y; - } - - public void setY(int y) { - this.y = y; - } - - public int getZ() { - return z; - } - - public void setZ(int z) { - this.z = z; - } - - public int getExtent() { - return extent; - } - - public void setExtent(int extent) { - // TODO: validation - this.extent = extent; - } - - public boolean getExactBounds() { - return exact_bounds; - } - - public void setExactBounds(boolean exact_bounds) { - this.exact_bounds = exact_bounds; - } - - public List getFields() { - return fields; - } - - public void setFields(List fields) { - this.fields = fields; - } - - public QueryBuilder getQueryBuilder() { - return queryBuilder; - } - - public void setQueryBuilder(QueryBuilder queryBuilder) { - // TODO: validation - this.queryBuilder = queryBuilder; - } - - public Rectangle getBoundingBox() { - return GeoTileUtils.toBoundingBox(x, y, z); - } - - public QueryBuilder getQuery() throws IOException { - QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(field, getBoundingBox()); - if (queryBuilder != null) { - BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); - boolQueryBuilder.filter(queryBuilder); - boolQueryBuilder.filter(qBuilder); - qBuilder = boolQueryBuilder; - } - return qBuilder; - } - - public Map getRuntimeMappings() { - return runtimeMappings; - } - - public void setRuntimeMappings(Map runtimeMappings) { - this.runtimeMappings = runtimeMappings; - } - - public int getGridPrecision() { - return gridPrecision; - } - - public void setGridPrecision(int gridPrecision) { - if (gridPrecision < 0 || gridPrecision > 8) { - throw new IllegalArgumentException("Invalid grid precision, value should be between 0 and 8, got [" + gridPrecision + "]"); - } - this.gridPrecision = gridPrecision; - } - - public GRID_TYPE getGridType() { - return gridType; - } - - public void setGridType(String gridType) { - this.gridType = GRID_TYPE.fromString(gridType); - } - - public int getSize() { - return size; - } - - public void setSize(int size) { - // TODO: validation - this.size = size; - } - - public AggregatorFactories.Builder getAggBuilder() { - return aggBuilder; - } - - public void setAggBuilder(AggregatorFactories.Builder aggBuilder) { - for (AggregationBuilder aggregation : aggBuilder.getAggregatorFactories()) { - final String type = aggregation.getType(); - switch (type) { - case MinAggregationBuilder.NAME: - case MaxAggregationBuilder.NAME: - case AvgAggregationBuilder.NAME: - case SumAggregationBuilder.NAME: - case CardinalityAggregationBuilder.NAME: - break; - default: - // top term and percentile should be supported - throw new IllegalArgumentException("Unsupported aggregation of type [" + type + "]"); - } - } - for (PipelineAggregationBuilder aggregation : aggBuilder.getPipelineAggregatorFactories()) { - // should not have pipeline aggregations - final String type = aggregation.getType(); - throw new IllegalArgumentException("Unsupported pipeline aggregation of type [" + type + "]"); - } - this.aggBuilder = aggBuilder; - } - - public Collection getAggregations() { - return aggBuilder == null ? emptyList() : aggBuilder.getAggregatorFactories(); - } - } - - protected AbstractVectorTileSearchAction(Supplier emptyRequestProvider) { - this.emptyRequestProvider = emptyRequestProvider; - parser = new ObjectParser<>(getName(), emptyRequestProvider); - parser.declareInt(Request::setSize, SearchSourceBuilder.SIZE_FIELD); - parser.declareField( - Request::setFields, - AbstractVectorTileSearchAction::parseFetchFields, - SearchSourceBuilder.FETCH_FIELDS_FIELD, - ObjectParser.ValueType.OBJECT_ARRAY - ); - parser.declareField( - Request::setQueryBuilder, - (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, - SearchSourceBuilder.QUERY_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareField( - Request::setRuntimeMappings, - XContentParser::map, - SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - parser.declareField( - Request::setAggBuilder, - AggregatorFactories::parseAggregators, - SearchSourceBuilder.AGGS_FIELD, - ObjectParser.ValueType.OBJECT - ); - // Specific for vector tiles - parser.declareInt(Request::setGridPrecision, GRID_PRECISION_FIELD); - parser.declareInt(Request::setExtent, EXTENT_FIELD); - parser.declareBoolean(Request::setExactBounds, EXACT_BOUNDS_FIELD); - parser.declareString(Request::setGridType, GRID_TYPE_FIELD); - } - - private static List parseFetchFields(XContentParser parser) throws IOException { - List fetchFields = new ArrayList<>(); - while ((parser.nextToken()) != XContentParser.Token.END_ARRAY) { - fetchFields.add(FieldAndFormat.fromXContent(parser)); - } - return fetchFields; - } - - protected abstract ResponseBuilder doParseRequest(RestRequest restRequest, R request, SearchRequestBuilder searchRequestBuilder) - throws IOException; - - @Override - protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { - final R request; - if (restRequest.hasContent()) { - try (XContentParser contentParser = restRequest.contentParser()) { - request = parser.parse(contentParser, restRequest); - } - } else { - request = emptyRequestProvider.get(); - } - request.setIndex(restRequest.param(INDEX_PARAM)); - request.setField(restRequest.param(FIELD_PARAM)); - request.setZ(Integer.parseInt(restRequest.param(Z_PARAM))); - request.setX(Integer.parseInt(restRequest.param(X_PARAM))); - request.setY(Integer.parseInt(restRequest.param(Y_PARAM))); - - SearchRequestBuilder searchRequestBuilder = client.prepareSearch(Strings.splitStringByCommaToArray(request.getIndex())); - searchRequestBuilder.setQuery(request.getQuery()); - ResponseBuilder responseBuilder = doParseRequest(restRequest, request, searchRequestBuilder); - - // TODO: how do we handle cancellations? - return channel -> searchRequestBuilder.execute(new RestResponseListener<>(channel) { - - @Override - public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { - try (BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput())) { - responseBuilder.buildResponse(searchResponse, bytesOut); - return new BytesRestResponse(RestStatus.OK, "application/x-protobuf", bytesOut.bytes()); - } - } - }); - } - -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java deleted file mode 100644 index 7824342ed16a4..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/PointFactory.java +++ /dev/null @@ -1,105 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.vectortile; - -import org.elasticsearch.geometry.Circle; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.GeometryCollection; -import org.elasticsearch.geometry.GeometryVisitor; -import org.elasticsearch.geometry.Line; -import org.elasticsearch.geometry.LinearRing; -import org.elasticsearch.geometry.MultiLine; -import org.elasticsearch.geometry.MultiPoint; -import org.elasticsearch.geometry.MultiPolygon; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.Polygon; -import org.elasticsearch.geometry.Rectangle; - -import java.util.ArrayList; -import java.util.List; - -public class PointFactory { - - private PointExtractor pointExtractor = new PointExtractor(); - - public PointFactory() { - } - - public List getPoints(Geometry geometry) { - pointExtractor.points.clear(); - geometry.visit(pointExtractor); - return pointExtractor.points; - } - - private static class PointExtractor implements GeometryVisitor { - - List points = new ArrayList<>(); - - PointExtractor() { - } - - @Override - public Void visit(Point point) throws IllegalArgumentException { - points.add(point); - return null; - } - - @Override - public Void visit(MultiPoint multiPoint) throws IllegalArgumentException { - for (Point p : multiPoint) { - visit(p); - } - return null; - } - - - @Override - public Void visit(GeometryCollection collection) throws IllegalArgumentException { - for (Geometry geometry : collection) { - geometry.visit(this); - } - return null; - } - - @Override - public Void visit(Circle circle) { - return null; - } - - @Override - public Void visit(Line line) throws IllegalArgumentException { - return null; - } - - @Override - public Void visit(LinearRing ring) throws IllegalArgumentException { - return null; - } - - @Override - public Void visit(MultiLine multiLine) throws IllegalArgumentException { - return null; - } - - - @Override - public Void visit(MultiPolygon multiPolygon) throws IllegalArgumentException { - return null; - } - - @Override - public Void visit(Polygon polygon) throws IllegalArgumentException { - return null; - } - - @Override - public Void visit(Rectangle rectangle) throws IllegalArgumentException { - return null; - } - } -} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java b/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java deleted file mode 100644 index 2abb2aa04b7d3..0000000000000 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/RestVectorTileAction.java +++ /dev/null @@ -1,248 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.vectortile; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; -import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; -import org.elasticsearch.action.search.SearchRequestBuilder; -import org.elasticsearch.action.search.SearchResponse; -import org.elasticsearch.action.search.SearchResponseSections; -import org.elasticsearch.common.document.DocumentField; -import org.elasticsearch.common.geo.GeoBoundingBox; -import org.elasticsearch.common.geo.GeoPoint; -import org.elasticsearch.common.geo.GeometryParser; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.Rectangle; -import org.elasticsearch.rest.RestRequest; -import org.elasticsearch.search.SearchHit; -import org.elasticsearch.search.SearchHits; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.Aggregations; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; -import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; -import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; -import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; -import org.elasticsearch.search.aggregations.pipeline.MaxBucketPipelineAggregationBuilder; -import org.elasticsearch.search.aggregations.pipeline.MinBucketPipelineAggregationBuilder; -import org.elasticsearch.search.fetch.subphase.FieldAndFormat; -import org.elasticsearch.search.profile.SearchProfileShardResults; - -import java.io.IOException; -import java.util.Collection; -import java.util.List; -import java.util.stream.Collectors; - -import static org.elasticsearch.rest.RestRequest.Method.GET; - -public class RestVectorTileAction extends AbstractVectorTileSearchAction { - - private static final String META_LAYER = "meta"; - private static final String HITS_LAYER = "hits"; - private static final String AGGS_LAYER = "aggs"; - - private static final String GRID_FIELD = "grid"; - private static final String BOUNDS_FIELD = "bounds"; - - private static final String COUNT_TAG = "_count"; - private static final String ID_TAG = "_id"; - - private static final String COUNT_MIN = "_count.min"; - private static final String COUNT_MAX = "_count.max"; - - public RestVectorTileAction() { - super(Request::new); - } - - @Override - public List routes() { - return List.of(new Route(GET, "{index}/_mvt/{field}/{z}/{x}/{y}")); - } - - @Override - protected ResponseBuilder doParseRequest(RestRequest restRequest, Request request, SearchRequestBuilder searchRequestBuilder) { - final int extent = request.getExtent(); - searchBuilder(searchRequestBuilder, request); - return (s, b) -> { - // Even if there is no hits, we return a tile with the meta layer - final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); - final VectorTileGeometryBuilder geomBuilder = new VectorTileGeometryBuilder( - request.getZ(), - request.getX(), - request.getY(), - extent - ); - final SearchHit[] hits = s.getHits().getHits(); - final InternalGeoTileGrid grid = s.getAggregations() != null ? s.getAggregations().get(GRID_FIELD) : null; - final InternalGeoBounds bounds = s.getAggregations() != null ? s.getAggregations().get(BOUNDS_FIELD) : null; - final Aggregations aggsWithoutGridAndBounds = s.getAggregations() == null - ? null - : new Aggregations( - s.getAggregations() - .asList() - .stream() - .filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false) - .collect(Collectors.toList()) - ); - SearchResponse meta = new SearchResponse( - new SearchResponseSections( - new SearchHits(SearchHits.EMPTY, s.getHits().getTotalHits(), s.getHits().getMaxScore()), // remove actual hits - aggsWithoutGridAndBounds, - s.getSuggest(), - s.isTimedOut(), - s.isTerminatedEarly(), - s.getProfileResults() == null ? null : new SearchProfileShardResults(s.getProfileResults()), - s.getNumReducePhases() - ), - s.getScrollId(), - s.getTotalShards(), - s.getSuccessfulShards(), - s.getSkippedShards(), - s.getTook().millis(), - s.getShardFailures(), - s.getClusters() - ); - if (hits.length > 0) { - tileBuilder.addLayers(buildHitsLayer(s, request)); - } - // TODO: should be expose the total number of buckets on InternalGeoTileGrid? - if (grid != null && grid.getBuckets().size() > 0) { - tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder)); - } - tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder)); - tileBuilder.build().writeTo(b); - }; - } - - private static SearchRequestBuilder searchBuilder(SearchRequestBuilder searchRequestBuilder, Request request) { - searchRequestBuilder.setSize(request.getSize()); - searchRequestBuilder.setFetchSource(false); - // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. - searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); - for (FieldAndFormat field : request.getFields()) { - searchRequestBuilder.addFetchField(field); - } - searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); - if (request.getGridPrecision() > 0) { - final Rectangle rectangle = request.getBoundingBox(); - final GeoBoundingBox boundingBox = new GeoBoundingBox( - new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), - new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) - ); - final int extent = 1 << request.getGridPrecision(); - final GeoGridAggregationBuilder aBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) - .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) - .setGeoBoundingBox(boundingBox) - .size(extent * extent); - if (request.getAggBuilder() != null) { - aBuilder.subAggregations(request.getAggBuilder()); - } - searchRequestBuilder.addAggregation(aBuilder); - searchRequestBuilder.addAggregation(new MaxBucketPipelineAggregationBuilder(COUNT_MAX, GRID_FIELD + "._count")); - searchRequestBuilder.addAggregation(new MinBucketPipelineAggregationBuilder(COUNT_MIN, GRID_FIELD + "._count")); - final Collection aggregations = request.getAggregations(); - for (AggregationBuilder aggregation : aggregations) { - searchRequestBuilder.addAggregation( - new MaxBucketPipelineAggregationBuilder(aggregation.getName() + ".max", GRID_FIELD + ">" + aggregation.getName())); - searchRequestBuilder.addAggregation( - new MinBucketPipelineAggregationBuilder(aggregation.getName() + ".min", GRID_FIELD + ">" + aggregation.getName())); - } - } - if (request.getExactBounds()) { - final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) - .wrapLongitude(false); - searchRequestBuilder.addAggregation(boundsBuilder); - } - return searchRequestBuilder; - } - - private VectorTile.Tile.Layer.Builder buildHitsLayer(SearchResponse response, Request request) { - final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); - final GeometryParser parser = new GeometryParser(true, false, false); - final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); - final List fields = request.getFields(); - for (SearchHit searchHit : response.getHits()) { - final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { - // TODO: It would be great if we can add the centroid information for polygons. That information can be - // used to place labels inside those geometries - VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); - if (fields != null) { - for (FieldAndFormat field : fields) { - final DocumentField documentField = searchHit.field(field.field); - if (documentField != null) { - VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); - } - } - } - }; - // TODO: See comment on field formats. - final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); - hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); - } - VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); - return hitsLayerBuilder; - } - - private VectorTile.Tile.Layer.Builder buildAggsLayer(InternalGeoTileGrid grid, Request request, VectorTileGeometryBuilder geomBuilder) - throws IOException{ - final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); - final MvtLayerProps layerProps = new MvtLayerProps(); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - for (InternalGeoGridBucket bucket : grid.getBuckets()) { - featureBuilder.clear(); - // Add geometry - if (request.getGridType() == GRID_TYPE.GRID) { - final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); - geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); - } else { - // TODO: it should be the centroid of the data - final GeoPoint point = (GeoPoint) bucket.getKey(); - geomBuilder.point(featureBuilder, point.lon(), point.lat()); - } - // Add count as key value pair - VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, bucket.getDocCount()); - for (Aggregation aggregation : bucket.getAggregations()) { - VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, aggregation); - } - aggLayerBuilder.addFeatures(featureBuilder); - } - VectorTileUtils.addPropertiesToLayer(aggLayerBuilder, layerProps); - return aggLayerBuilder; - } - - private VectorTile.Tile.Layer.Builder buildMetaLayer( - SearchResponse response, - InternalGeoBounds bounds, - Request request, - VectorTileGeometryBuilder geomBuilder - ) throws IOException { - final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); - final MvtLayerProps layerProps = new MvtLayerProps(); - final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); - if (bounds != null && bounds.topLeft() != null) { - final GeoPoint topLeft = bounds.topLeft(); - final GeoPoint bottomRight = bounds.bottomRight(); - geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); - } else { - final Rectangle tile = request.getBoundingBox(); - geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); - } - VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response); - metaLayerBuilder.addFeatures(featureBuilder); - VectorTileUtils.addPropertiesToLayer(metaLayerBuilder, layerProps); - return metaLayerBuilder; - } - - @Override - public String getName() { - return "vectortile_action"; - } -} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java deleted file mode 100644 index 3957dd3b8472f..0000000000000 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/InternalVectorTileTests.java +++ /dev/null @@ -1,185 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.search.aggregations; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; -import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; -import org.elasticsearch.common.ParseField; -import org.elasticsearch.common.util.CollectionUtils; -import org.elasticsearch.common.xcontent.NamedXContentRegistry; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.search.aggregations.Aggregation; -import org.elasticsearch.search.aggregations.ParsedAggregation; -import org.elasticsearch.test.InternalAggregationTestCase; -import org.elasticsearch.xpack.spatial.SpatialPlugin; -import org.elasticsearch.xpack.spatial.vectortile.FeatureFactory; -import org.hamcrest.Matchers; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -public class InternalVectorTileTests extends InternalAggregationTestCase { - - @Override - protected SearchPlugin registerPlugin() { - return new SpatialPlugin(); - } - - static VectorTile.Tile.Layer randomPolygonLayer(int shapes) { - final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); - layerBuilder.setVersion(2); - layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); - layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); - final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); - final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); - for (int i =0; i < shapes; i++) { - int count = layerBuilder.getFeaturesCount(); - while(true) { - Geometry geometry = GeometryTestUtils.randomPolygon(false); - List features = factory.getFeatures(geometry, ignoreData); - for (VectorTile.Tile.Feature feature : features) { - layerBuilder.addFeatures(feature); - } - if (count < layerBuilder.getFeaturesCount()) { - break; - } - } - } - return layerBuilder.build(); - } - - static VectorTile.Tile.Layer randomLineLayer(int shapes) { - final VectorTile.Tile.Layer.Builder layerBuilder = VectorTile.Tile.Layer.newBuilder(); - layerBuilder.setVersion(2); - layerBuilder.setName(AbstractVectorTileAggregator.POLYGON_LAYER); - layerBuilder.setExtent(AbstractVectorTileAggregator.POLYGON_EXTENT); - final FeatureFactory factory = new FeatureFactory(0, 0, 0, AbstractVectorTileAggregator.POLYGON_EXTENT); - final IUserDataConverter ignoreData = new UserDataIgnoreConverter(); - for (int i =0; i < shapes; i++) { - int count = layerBuilder.getFeaturesCount(); - while(true) { - Geometry geometry = GeometryTestUtils.randomLine(false); - List features = factory.getFeatures(geometry, ignoreData); - for (VectorTile.Tile.Feature feature : features) { - layerBuilder.addFeatures(feature); - } - if (count < layerBuilder.getFeaturesCount()) { - break; - } - } - } - return layerBuilder.build(); - - } - - static long[] randomPoints() { - long[] points = new long[AbstractVectorTileAggregator.POINT_EXTENT * AbstractVectorTileAggregator.POINT_EXTENT]; - for( int i = 0; i < points.length; i++) { - points[i] = randomInt(10); - } - return points; - } - - @Override - protected InternalVectorTile createTestInstance(String name, Map metadata) { - int size = randomIntBetween(5, 10); - VectorTile.Tile.Layer polygons = randomBoolean() ? randomPolygonLayer(size) : null; - VectorTile.Tile.Layer lines = randomBoolean() ? randomLineLayer(size) : null; - long[] points = randomBoolean() ? randomPoints() : null; - return new InternalVectorTile(name, polygons, lines, points, metadata); - } - - @Override - protected InternalVectorTile mutateInstance(InternalVectorTile instance) { - String name = instance.getName(); - VectorTile.Tile.Layer polygons = instance.polygons; - VectorTile.Tile.Layer lines = instance.lines; - long[] points = instance.points; - Map metadata = instance.getMetadata(); - switch (randomIntBetween(0, 4)) { - case 0: - name += randomAlphaOfLength(5); - break; - case 1: - if (metadata == null) { - metadata = new HashMap<>(1); - } else { - metadata = new HashMap<>(instance.getMetadata()); - } - metadata.put(randomAlphaOfLength(15), randomInt()); - break; - case 2: - polygons = randomPolygonLayer(randomIntBetween(10, 20)); - break; - case 3: - lines = randomLineLayer(randomIntBetween(10, 20)); - break; - case 4: - points = randomPoints(); - break; - default: - throw new AssertionError("Illegal randomisation branch"); - } - return new InternalVectorTile(name, polygons, lines, points, metadata); - } - - @Override - protected List randomResultsToReduce(String name, int size) { - List instances = new ArrayList<>(size); - for (int i = 0; i < size; i++) { - instances.add(createTestInstance(name, null)); - } - return instances; - } - - @Override - protected void assertReduced(InternalVectorTile reduced, List inputs) { - int numPolygons = 0; - int numLines = 0; - long numPoints = 0; - for (InternalVectorTile input : inputs) { - if (input.polygons != null) { - numPolygons += input.polygons.getFeaturesCount(); - } - if (input.lines != null) { - numLines += input.lines.getFeaturesCount(); - } - if (input.points != null) { - numPoints += Arrays.stream(input.points).sum(); - } - } - int numReducedPolygons = reduced.polygons != null ? reduced.polygons.getFeaturesCount() : 0; - assertThat(numReducedPolygons, Matchers.equalTo(numPolygons)); - int numReducedLines = reduced.lines != null ? reduced.lines.getFeaturesCount() : 0; - assertThat(numReducedLines, Matchers.equalTo(numLines)); - long numReducedPoints = reduced.points != null ? Arrays.stream(reduced.points).sum() : 0; - assertThat(numReducedPoints, Matchers.equalTo(numPoints)); - } - - @Override - protected void assertFromXContent(InternalVectorTile aggregation, ParsedAggregation parsedAggregation) throws IOException { - // The output is binary so nothing to do here - } - - @Override - protected List getNamedXContents() { - return CollectionUtils.appendToCopy(super.getNamedXContents(), new NamedXContentRegistry.Entry(Aggregation.class, - new ParseField(VectorTileAggregationBuilder.NAME), - (p, c) -> { - assumeTrue("There is no ParsedVectorTile yet", false); - return null; - } - )); - } -} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java deleted file mode 100644 index b608ef623c8fa..0000000000000 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregationBuilderTests.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ - -package org.elasticsearch.xpack.spatial.search.aggregations; - -import org.elasticsearch.common.io.stream.Writeable; -import org.elasticsearch.common.xcontent.XContentParser; -import org.elasticsearch.test.AbstractSerializingTestCase; - -import java.io.IOException; - -import static org.hamcrest.Matchers.equalTo; - -public class VectorTileAggregationBuilderTests extends AbstractSerializingTestCase { - - @Override - protected VectorTileAggregationBuilder doParseInstance(XContentParser parser) throws IOException { - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - String name = parser.currentName(); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.START_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.FIELD_NAME)); - assertThat(parser.currentName(), equalTo(VectorTileAggregationBuilder.NAME)); - VectorTileAggregationBuilder parsed = VectorTileAggregationBuilder.PARSER.apply(parser, name); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - assertThat(parser.nextToken(), equalTo(XContentParser.Token.END_OBJECT)); - return parsed; - } - - @Override - protected Writeable.Reader instanceReader() { - return VectorTileAggregationBuilder::new; - } - - @Override - protected VectorTileAggregationBuilder createTestInstance() { - VectorTileAggregationBuilder vectorTileAggregationBuilder = new VectorTileAggregationBuilder("_name"); - vectorTileAggregationBuilder.z(0).x(0).y(0); - return vectorTileAggregationBuilder; - } -} diff --git a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java b/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java deleted file mode 100644 index ca6ab460eac5a..0000000000000 --- a/x-pack/plugin/spatial/src/test/java/org/elasticsearch/xpack/spatial/search/aggregations/VectorTileAggregatorTests.java +++ /dev/null @@ -1,239 +0,0 @@ -/* - * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one - * or more contributor license agreements. Licensed under the Elastic License - * 2.0; you may not use this file except in compliance with the Elastic License - * 2.0. - */ -package org.elasticsearch.xpack.spatial.search.aggregations; - -import com.wdtinc.mapbox_vector_tile.VectorTile; -import org.apache.lucene.document.Document; -import org.apache.lucene.document.LatLonDocValuesField; -import org.apache.lucene.document.StoredField; -import org.apache.lucene.index.IndexReader; -import org.apache.lucene.index.RandomIndexWriter; -import org.apache.lucene.search.IndexSearcher; -import org.apache.lucene.search.MatchAllDocsQuery; -import org.apache.lucene.store.Directory; -import org.apache.lucene.util.BytesRef; -import org.elasticsearch.common.geo.builders.ShapeBuilder; -import org.elasticsearch.geo.GeometryTestUtils; -import org.elasticsearch.geometry.Geometry; -import org.elasticsearch.geometry.Point; -import org.elasticsearch.geometry.utils.WellKnownText; -import org.elasticsearch.index.mapper.GeoPointFieldMapper; -import org.elasticsearch.index.mapper.GeoShapeIndexer; -import org.elasticsearch.index.mapper.MappedFieldType; -import org.elasticsearch.index.mapper.SourceFieldMapper; -import org.elasticsearch.plugins.SearchPlugin; -import org.elasticsearch.search.aggregations.AggregationBuilder; -import org.elasticsearch.search.aggregations.AggregatorTestCase; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; -import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; -import org.elasticsearch.search.aggregations.support.CoreValuesSourceType; -import org.elasticsearch.search.aggregations.support.ValuesSourceType; -import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.spatial.LocalStateSpatialPlugin; -import org.elasticsearch.xpack.spatial.index.fielddata.GeoShapeValues; -import org.elasticsearch.xpack.spatial.index.mapper.GeoShapeWithDocValuesFieldMapper; -import org.elasticsearch.xpack.spatial.search.aggregations.support.GeoShapeValuesSourceType; -import org.elasticsearch.xpack.spatial.util.GeoTestUtils; -import org.hamcrest.Matchers; -import org.locationtech.spatial4j.exception.InvalidShapeException; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.Collections; -import java.util.List; -import java.util.function.Function; - -import static org.hamcrest.Matchers.equalTo; -import static org.hamcrest.Matchers.greaterThanOrEqualTo; - - -public class VectorTileAggregatorTests extends AggregatorTestCase { - - @Override - protected List getSearchPlugins() { - return List.of(new LocalStateSpatialPlugin()); - } - - public void testGeoPoint() throws Exception { - int numDocs = scaledRandomIntBetween(64, 256); - List points = new ArrayList<>(); - for (int i = 0; i < numDocs; i++) { - points.add(randomValueOtherThanMany((p) -> p.getLat() > 80 || p.getLat() < -80, GeometryTestUtils::randomPoint)); - } - try (Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (Point point : points) { - Document document = new Document(); - document.add(new LatLonDocValuesField("field", point.getLat(), point.getLon())); - BytesRef val = new BytesRef("{\"field\" : \"" + WellKnownText.INSTANCE.toWKT(point) + "\"}"); - document.add(new StoredField("_source", val)); - w.addDocument(document); - } - if (randomBoolean()) { - w.forceMerge(1); - } - assertVectorTile(w, 0, 0, numDocs, new GeoPointFieldMapper.GeoPointFieldType("field")); - } - } - - @SuppressWarnings("unchecked") - @AwaitsFix(bugUrl = "That doesn't work yet") - public void testGeoShape() throws Exception { - int numDocs = scaledRandomIntBetween(64, 256); - List geometries = new ArrayList<>(); - GeoShapeIndexer indexer = new GeoShapeIndexer(true, "test"); - for (int i = 0; i < numDocs; i++) { - Function geometryGenerator = ESTestCase.randomFrom( - GeometryTestUtils::randomLine, - GeometryTestUtils::randomPoint, - GeometryTestUtils::randomPolygon, - GeometryTestUtils::randomMultiLine, - GeometryTestUtils::randomMultiPolygon - ); - Geometry geometry = geometryGenerator.apply(false); - try { - geometries.add(indexer.prepareForIndexing(geometry)); - } catch (InvalidShapeException e) { - // do not include geometry - } - } - int expectedPolygons = 0; - int expectedLines = 0; - int expectedPoints = 0; - try (Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), dir)) { - for (Geometry geometry : geometries) { - Document document = new Document(); - document.add(GeoTestUtils.binaryGeoShapeDocValuesField("field", geometry)); - BytesRef val = new BytesRef("{\"field\" : \"" + WellKnownText.INSTANCE.toWKT(geometry) + "\"}"); - document.add(new StoredField("_source", val)); - w.addDocument(document); - GeoShapeValues.GeoShapeValue value = GeoTestUtils.geoShapeValue(geometry); - switch (value.dimensionalShapeType()) { - case POINT: - if (value.lat() < GeoTileUtils.LATITUDE_MASK && value.lat() > -GeoTileUtils.LATITUDE_MASK) { - expectedPoints++; - } - break; - case LINE: - expectedLines++; - break; - case POLYGON: - expectedPolygons++; - break; - } - } - if (randomBoolean()) { - w.forceMerge(1); - } - MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", - true, true, ShapeBuilder.Orientation.RIGHT, null, Collections.emptyMap()); - assertVectorTile(w, expectedPolygons, expectedLines, expectedPoints, fieldType); - } - } - - private void assertVectorTile(RandomIndexWriter w, int expectedPolygons, int expectedLines, - int expectedPoints, MappedFieldType fieldType) throws IOException { - MappedFieldType sourceFieldType = new SourceFieldMapper.Builder().build().fieldType(); - VectorTileAggregationBuilder aggBuilder = new VectorTileAggregationBuilder("my_agg") - .field("field"); - try (IndexReader reader = w.getReader()) { - IndexSearcher searcher = new IndexSearcher(reader); - InternalVectorTile result = searchAndReduce(searcher, new MatchAllDocsQuery(), aggBuilder, fieldType, sourceFieldType); - assertEquals("my_agg", result.getName()); - VectorTile.Tile tileResult = VectorTile.Tile.newBuilder().mergeFrom(result.getVectorTileBytes()).build(); - assertNotNull(tileResult); - int expectedLayers = (expectedPolygons > 0 ? 1 : 0) + (expectedLines > 0 ? 1 : 0) + (expectedPoints > 0 ? 1 : 0); - assertThat(expectedLayers, Matchers.equalTo(tileResult.getLayersCount())); - if (expectedPolygons > 0) { - VectorTile.Tile.Layer polygons = null; - for (int i = 0; i < tileResult.getLayersCount(); i++) { - VectorTile.Tile.Layer layer = tileResult.getLayers(i); - if (AbstractVectorTileAggregator.POLYGON_LAYER.equals(layer.getName())) { - polygons = layer; - break; - } - } - assertThat(polygons, Matchers.notNullValue()); - assertThat(AbstractVectorTileAggregator.POLYGON_EXTENT, Matchers.equalTo(polygons.getExtent())); - assertThat(2, Matchers.equalTo(polygons.getVersion())); - assertThat(expectedPolygons, Matchers.greaterThanOrEqualTo(polygons.getFeaturesCount())); - } - - if (expectedLines > 0) { - VectorTile.Tile.Layer lines = null; - for (int i = 0; i < tileResult.getLayersCount(); i++) { - VectorTile.Tile.Layer layer = tileResult.getLayers(i); - if (AbstractVectorTileAggregator.LINE_LAYER.equals(layer.getName())) { - lines = layer; - break; - } - } - assertThat(lines, Matchers.notNullValue()); - assertThat(AbstractVectorTileAggregator.LINE_EXTENT, Matchers.equalTo(lines.getExtent())); - assertThat(2, Matchers.equalTo(lines.getVersion())); - assertThat(expectedLines, Matchers.greaterThanOrEqualTo(lines.getFeaturesCount())); - } - - if (expectedPoints > 0) { - VectorTile.Tile.Layer points = null; - for (int i = 0; i < tileResult.getLayersCount(); i++) { - VectorTile.Tile.Layer layer = tileResult.getLayers(i); - if (AbstractVectorTileAggregator.POINT_LAYER.equals(layer.getName())) { - points = layer; - break; - } - } - assertThat(points, Matchers.notNullValue()); - assertThat(AbstractVectorTileAggregator.POINT_EXTENT, Matchers.equalTo(points.getExtent())); - assertThat(2, Matchers.equalTo(points.getVersion())); - int tagIndex = -1; - for (int i = 0; i < points.getKeysCount(); i++) { - if ("count".equals(points.getKeys(i))) { - tagIndex = i; - break; - } - } - assertThat(tagIndex, greaterThanOrEqualTo(0)); - int numberPoints = 0; - for (int i = 0; i < points.getFeaturesCount(); i++) { - VectorTile.Tile.Feature feature = points.getFeatures(i); - numberPoints += points.getValues(feature.getTags(tagIndex + 1)).getIntValue(); - } - assertThat(expectedPoints, Matchers.equalTo(numberPoints)); - } - } - } - - public void testInvalidChildAggregation() throws Exception { - try (Directory dir = newDirectory(); - RandomIndexWriter w = new RandomIndexWriter(random(), dir); - IndexReader reader = w.getReader()) { - VectorTileAggregationBuilder aggBuilder = new VectorTileAggregationBuilder("my_agg") - .field("field"); - GeoGridAggregationBuilder parent = new GeoTileGridAggregationBuilder("my_parent") - .field("field").subAggregation(aggBuilder); - MappedFieldType fieldType = new GeoShapeWithDocValuesFieldMapper.GeoShapeWithDocValuesFieldType("field", - true, true, ShapeBuilder.Orientation.RIGHT, null, Collections.emptyMap()); - IndexSearcher searcher = new IndexSearcher(reader); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () - -> searchAndReduce(searcher, new MatchAllDocsQuery(), parent, fieldType)); - assertThat(e.getMessage(), equalTo("vector-tile aggregation must be at the top level")); - } - } - - @Override - protected AggregationBuilder createAggBuilderForTypeTest(MappedFieldType fieldType, String fieldName) { - return new VectorTileAggregationBuilder("foo").field(fieldName).y(0).x(0).z(0); - } - - @Override - protected List getSupportedValuesSourceTypes() { - return List.of(CoreValuesSourceType.GEOPOINT, GeoShapeValuesSourceType.instance()); - } -} diff --git a/x-pack/plugin/vector-tile/build.gradle b/x-pack/plugin/vector-tile/build.gradle new file mode 100644 index 0000000000000..337d4986cbf1d --- /dev/null +++ b/x-pack/plugin/vector-tile/build.gradle @@ -0,0 +1,69 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0 and the Server Side Public License, v 1; you may not use this file except + * in compliance with, at your election, the Elastic License 2.0 or the Server + * Side Public License, v 1. + */ + +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +import org.elasticsearch.gradle.internal.info.BuildParams + +apply plugin: 'elasticsearch.internal-es-plugin' +apply plugin: 'elasticsearch.java-rest-test' + +esplugin { + name 'vector-tile' + description 'A plugin for mapbox vector tile features' + classname 'org.elasticsearch.xpack.vectortile.VectorTilePlugin' + extendedPlugins = ['x-pack-core'] +} + +dependencies { + compileOnly project(path: xpackModule('core')) + testImplementation(testArtifact(project(xpackModule('core')))) + api "com.wdtinc:mapbox-vector-tile:3.1.0" + api "com.google.protobuf:protobuf-java:3.14.0" + javaRestTestImplementation("com.wdtinc:mapbox-vector-tile:3.1.0") + javaRestTestImplementation("com.google.protobuf:protobuf-java:3.14.0") +} + +testClusters.all { + setting 'xpack.license.self_generated.type', 'trial' + testDistribution = 'DEFAULT' + setting 'xpack.security.enabled', 'false' + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } +} + +tasks.named("test").configure { + if (BuildParams.isSnapshotBuild() == false) { + systemProperty 'es.vector_tile_feature_flag_registered', 'true' + } +} + +tasks.named("thirdPartyAudit").configure { + ignoreViolations( + // uses internal java api: sun.misc.Unsafe + 'com.google.protobuf.UnsafeUtil', + 'com.google.protobuf.MessageSchema', + 'com.google.protobuf.UnsafeUtil$1', + 'com.google.protobuf.UnsafeUtil$Android32MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$Android64MemoryAccessor', + 'com.google.protobuf.UnsafeUtil$JvmMemoryAccessor', + 'com.google.protobuf.UnsafeUtil$MemoryAccessor' + ) + + ignoreMissingClasses( + 'org.slf4j.Logger', + 'org.slf4j.LoggerFactory' + ) +} + diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1 similarity index 100% rename from x-pack/plugin/spatial/licenses/mapbox-vector-tile-3.1.0.jar.sha1 rename to x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-3.1.0.jar.sha1 diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-LICENSE.txt similarity index 100% rename from x-pack/plugin/spatial/licenses/mapbox-vector-tile-LICENSE.txt rename to x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-LICENSE.txt diff --git a/x-pack/plugin/spatial/licenses/mapbox-vector-tile-NOTICE.txt b/x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-NOTICE.txt similarity index 100% rename from x-pack/plugin/spatial/licenses/mapbox-vector-tile-NOTICE.txt rename to x-pack/plugin/vector-tile/licenses/mapbox-vector-tile-NOTICE.txt diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 b/x-pack/plugin/vector-tile/licenses/protobuf-java-3.14.0.jar.sha1 similarity index 100% rename from x-pack/plugin/spatial/licenses/protobuf-java-3.14.0.jar.sha1 rename to x-pack/plugin/vector-tile/licenses/protobuf-java-3.14.0.jar.sha1 diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt b/x-pack/plugin/vector-tile/licenses/protobuf-java-LICENSE.txt similarity index 100% rename from x-pack/plugin/spatial/licenses/protobuf-java-LICENSE.txt rename to x-pack/plugin/vector-tile/licenses/protobuf-java-LICENSE.txt diff --git a/x-pack/plugin/spatial/licenses/protobuf-java-NOTICE.txt b/x-pack/plugin/vector-tile/licenses/protobuf-java-NOTICE.txt similarity index 100% rename from x-pack/plugin/spatial/licenses/protobuf-java-NOTICE.txt rename to x-pack/plugin/vector-tile/licenses/protobuf-java-NOTICE.txt diff --git a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java similarity index 61% rename from x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java rename to x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java index b312ad15d8d47..debb50c5045df 100644 --- a/x-pack/plugin/spatial/src/yamlRestTest/java/org/elasticsearch/xpack/spatial/VectorTileRestIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.spatial; +package org.elasticsearch.xpack.vectortile; import com.wdtinc.mapbox_vector_tile.VectorTile; import org.apache.http.HttpStatus; @@ -34,6 +34,7 @@ public class VectorTileRestIT extends ESRestTestCase { private static final String INDEX_POINTS = "index-points"; private static final String INDEX_SHAPES = "index-shapes"; + private static final String INDEX_ALL = "index*"; private static final String META_LAYER = "meta"; private static final String HITS_LAYER = "hits"; private static final String AGGS_LAYER = "aggs"; @@ -54,16 +55,18 @@ private void indexPoints() throws IOException { Response response = client().performRequest(createRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_POINTS + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_point\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); + mappingRequest.setJsonEntity( + "{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_point\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}" + ); response = client().performRequest(mappingRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); @@ -72,10 +75,22 @@ private void indexPoints() throws IOException { for (int i = 0; i < 30; i += 10) { for (int j = 0; j <= i; j++) { final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_POINTS + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"POINT(" + x + " " + y + ")\", \"name\": \"point" + i + "\"" + - ", \"value1\": " + i + ", \"value2\": " + (i + 1) + "\n" + - "}"); + putRequest.setJsonEntity( + "{\n" + + " \"location\": \"POINT(" + + x + + " " + + y + + ")\", \"name\": \"point" + + i + + "\"" + + ", \"value1\": " + + i + + ", \"value2\": " + + (i + 1) + + "\n" + + "}" + ); response = client().performRequest(putRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); } @@ -91,26 +106,42 @@ private void indexShapes() throws IOException { Response response = client().performRequest(createRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); final Request mappingRequest = new Request(HttpPut.METHOD_NAME, INDEX_SHAPES + "/_mapping"); - mappingRequest.setJsonEntity("{\n" + - " \"properties\": {\n" + - " \"location\": {\n" + - " \"type\": \"geo_shape\"\n" + - " },\n" + - " \"name\": {\n" + - " \"type\": \"keyword\"\n" + - " }\n" + - " }\n" + - "}"); + mappingRequest.setJsonEntity( + "{\n" + + " \"properties\": {\n" + + " \"location\": {\n" + + " \"type\": \"geo_shape\"\n" + + " },\n" + + " \"name\": {\n" + + " \"type\": \"keyword\"\n" + + " }\n" + + " }\n" + + "}" + ); response = client().performRequest(mappingRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_OK)); final Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); final Request putRequest = new Request(HttpPost.METHOD_NAME, INDEX_SHAPES + "/_doc"); - putRequest.setJsonEntity("{\n" + - " \"location\": \"BBOX (" + r.getMinLon() + ", " + r.getMaxLon() + "," + r.getMaxLat() + "," + r.getMinLat() + ")\"" + - ", \"name\": \"rectangle\"" + - ", \"value1\": " + 1 + ", \"value2\": " + 2 + "\n" + - "}"); + putRequest.setJsonEntity( + "{\n" + + " \"location\": \"BBOX (" + + r.getMinLon() + + ", " + + r.getMaxLon() + + "," + + r.getMaxLat() + + "," + + r.getMinLat() + + ")\"" + + ", \"name\": \"rectangle\"" + + ", \"value1\": " + + 1 + + ", \"value2\": " + + 2 + + "\n" + + "}" + ); response = client().performRequest(putRequest); assertThat(response.getStatusLine().getStatusCode(), Matchers.equalTo(HttpStatus.SC_CREATED)); @@ -128,11 +159,22 @@ public void deleteData() throws IOException { public void testBasicGet() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100}"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 13); + assertLayer(tile, META_LAYER, 4096, 1, 14); + } + + public void testExtent() throws Exception { + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity("{\"size\" : 100, \"extent\" : 256}"); + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(3)); + assertLayer(tile, HITS_LAYER, 256, 33, 1); + assertLayer(tile, AGGS_LAYER, 256, 1, 1); + assertLayer(tile, META_LAYER, 256, 1, 14); } public void testEmpty() throws Exception { @@ -140,18 +182,18 @@ public void testEmpty() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + newY); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(1)); - assertLayer(tile, META_LAYER, 4096, 1, 8); + assertLayer(tile, META_LAYER, 4096, 1, 10); } public void testGridPrecision() throws Exception { { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"grid_precision\": 7 }"); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_precision\": 7 }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 13); + assertLayer(tile, META_LAYER, 4096, 1, 14); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -164,21 +206,21 @@ public void testGridPrecision() throws Exception { public void testGridType() throws Exception { { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"grid_type\": \"point\" }"); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"point\" }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 13); + assertLayer(tile, META_LAYER, 4096, 1, 14); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"grid_type\": \"grid\" }"); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_type\": \"grid\" }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 13); + assertLayer(tile, META_LAYER, 4096, 1, 14); } { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); @@ -190,7 +232,7 @@ public void testGridType() throws Exception { public void testNoAggLayer() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\"grid_precision\": 0 }"); + mvtRequest.setJsonEntity("{\"size\" : 100, \"grid_precision\": 0 }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, HITS_LAYER, 4096, 33, 1); @@ -203,64 +245,128 @@ public void testNoHitsLayer() throws Exception { final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(2)); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 12); + assertLayer(tile, META_LAYER, 4096, 1, 13); + } + + public void testRuntimeFieldWithSort() throws Exception { + String runtimeMapping = "\"runtime_mappings\": {\n" + + " \"width\": {\n" + + " \"script\": " + + "\"emit(doc['location'].getBoundingBox().bottomRight().getLon() - doc['location'].getBoundingBox().topLeft().getLon())\",\n" + + " \"type\": \"double\"\n" + + " }\n" + + "}\n"; + { + // desc order, polygon should be the first hit + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_ALL + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"size\" : 100,\n" + + " \"grid_precision\" : 0,\n" + + runtimeMapping + + "," + + " \"sort\" : [\n" + + " {\n" + + " \"width\": {\n" + + " \"order\": \"desc\"\n" + + " }\n" + + " }\n" + + " ]" + + "}" + ); + + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 34, 1); + final VectorTile.Tile.Layer layer = getLayer(tile, HITS_LAYER); + assertThat(layer.getFeatures(0).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); + assertLayer(tile, META_LAYER, 4096, 1, 8); + } + { + // asc order, polygon should be the last hit + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_ALL + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"size\" : 100,\n" + + " \"grid_precision\" : 0,\n" + + runtimeMapping + + "," + + " \"sort\" : [\n" + + " {\n" + + " \"width\": {\n" + + " \"order\": \"asc\"\n" + + " }\n" + + " }\n" + + " ]" + + "}" + ); + + final VectorTile.Tile tile = execute(mvtRequest); + assertThat(tile.getLayersCount(), Matchers.equalTo(2)); + assertLayer(tile, HITS_LAYER, 4096, 34, 1); + final VectorTile.Tile.Layer layer = getLayer(tile, HITS_LAYER); + assertThat(layer.getFeatures(33).getType(), Matchers.equalTo(VectorTile.Tile.GeomType.POLYGON)); + assertLayer(tile, META_LAYER, 4096, 1, 8); + } } public void testBasicQueryGet() throws Exception { final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_POINTS + "/_mvt/location/" + z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"query\": {\n" + - " \"term\": {\n" + - " \"name\": {\n" + - " \"value\": \"point0\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); + mvtRequest.setJsonEntity( + "{\n" + + " \"query\": {\n" + + " \"term\": {\n" + + " \"name\": {\n" + + " \"value\": \"point0\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 1, 1); - assertLayer(tile, META_LAYER, 4096, 1, 13); + assertLayer(tile, META_LAYER, 4096, 1, 14); } public void testBasicShape() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - // The same max and min values are present in all buckets so min and max aggs return all buckets as max and mins - // as an array at the moment hence 2 additional values and 256 * 256 * 2 additional keys - assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2); + assertLayer(tile, META_LAYER, 4096, 1, 14); } public void testWithFields() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); mvtRequest.setJsonEntity("{\"fields\": [\"name\", \"value1\"] }"); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 3); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 1); - assertLayer(tile, META_LAYER, 4096, 1, 9 + 2 + 256 * 256 * 2); + assertLayer(tile, META_LAYER, 4096, 1, 14); } public void testMinAgg() throws Exception { - final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/"+ z + "/" + x + "/" + y); - mvtRequest.setJsonEntity("{\n" + - " \"aggs\": {\n" + - " \"minVal\": {\n" + - " \"min\": {\n" + - " \"field\": \"value1\"\n" + - " }\n" + - " }\n" + - " }\n" + - "}"); + final Request mvtRequest = new Request(HttpGet.METHOD_NAME, INDEX_SHAPES + "/_mvt/location/" + z + "/" + x + "/" + y); + mvtRequest.setJsonEntity( + "{\n" + + " \"aggs\": {\n" + + " \"minVal\": {\n" + + " \"min\": {\n" + + " \"field\": \"value1\"\n" + + " }\n" + + " }\n" + + " }\n" + + "}" + ); final VectorTile.Tile tile = execute(mvtRequest); assertThat(tile.getLayersCount(), Matchers.equalTo(3)); assertLayer(tile, HITS_LAYER, 4096, 1, 1); assertLayer(tile, AGGS_LAYER, 4096, 256 * 256, 2); - assertLayer(tile, META_LAYER, 4096, 1, 9 + 4 + 256 * 256 * 4); + assertLayer(tile, META_LAYER, 4096, 1, 19); } private void assertLayer(VectorTile.Tile tile, String name, int extent, int numFeatures, int numTags) { diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java new file mode 100644 index 0000000000000..3d659e826fca2 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java @@ -0,0 +1,90 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.vectortile; + +import org.elasticsearch.Build; +import org.elasticsearch.cluster.metadata.IndexNameExpressionResolver; +import org.elasticsearch.cluster.node.DiscoveryNodes; +import org.elasticsearch.common.settings.ClusterSettings; +import org.elasticsearch.common.settings.IndexScopedSettings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.license.XPackLicenseState; +import org.elasticsearch.plugins.ActionPlugin; +import org.elasticsearch.plugins.Plugin; +import org.elasticsearch.rest.RestController; +import org.elasticsearch.rest.RestHandler; +import org.elasticsearch.threadpool.ExecutorBuilder; +import org.elasticsearch.threadpool.FixedExecutorBuilder; +import org.elasticsearch.xpack.core.XPackPlugin; +import org.elasticsearch.xpack.vectortile.rest.RestVectorTileAction; + +import java.util.Collections; +import java.util.List; +import java.util.function.Supplier; + +public class VectorTilePlugin extends Plugin implements ActionPlugin { + + // to be overriden by tests + protected XPackLicenseState getLicenseState() { + return XPackPlugin.getSharedLicenseState(); + } + + private static final Boolean VECTOR_TILE_FEATURE_FLAG_REGISTERED; + + static { + final String property = System.getProperty("es.vector_tile_feature_flag_registered"); + if (Build.CURRENT.isSnapshot() && property != null) { + throw new IllegalArgumentException("es.vector_tile_feature_flag_registered is only supported in non-snapshot builds"); + } + if ("true".equals(property)) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = true; + } else if ("false".equals(property)) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = false; + } else if (property == null) { + VECTOR_TILE_FEATURE_FLAG_REGISTERED = null; + } else { + throw new IllegalArgumentException( + "expected es.vector_tile_feature_flag_registered to be unset or [true|false] but was [" + property + "]" + ); + } + } + + public boolean isVectorTileEnabled() { + return Build.CURRENT.isSnapshot() || (VECTOR_TILE_FEATURE_FLAG_REGISTERED != null && VECTOR_TILE_FEATURE_FLAG_REGISTERED); + } + + @Override + public List getRestHandlers( + Settings settings, + RestController restController, + ClusterSettings clusterSettings, + IndexScopedSettings indexScopedSettings, + SettingsFilter settingsFilter, + IndexNameExpressionResolver indexNameExpressionResolver, + Supplier nodesInCluster + ) { + if (isVectorTileEnabled()) { + return List.of(new RestVectorTileAction()); + } else { + return List.of(); + } + } + + @Override + public List> getExecutorBuilders(Settings settings) { + FixedExecutorBuilder indexing = new FixedExecutorBuilder( + settings, + "vector_tile_generation", + 1, + -1, + "thread_pool.vectortile", + false + ); + return Collections.singletonList(indexing); + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java similarity index 82% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java rename to x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java index c99cc238c2bf3..03771650922c7 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/FeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.spatial.vectortile; +package org.elasticsearch.xpack.vectortile.feature; import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.adapt.jts.IGeometryFilter; @@ -33,6 +33,9 @@ import java.util.List; +/** + * Transforms {@link Geometry} object in WGS84 into mvt features. + */ public class FeatureFactory { private final IGeometryFilter acceptAllGeomFilter = geometry -> true; @@ -45,18 +48,23 @@ public class FeatureFactory { private final Envelope clipEnvelope; public FeatureFactory(int z, int x, int y, int extent) { - this.tileEnvelope = VectorTileUtils.getJTSTileBounds(z, x, y); - this.clipEnvelope = VectorTileUtils.getJTSTileBounds(z, x, y); + this.tileEnvelope = FeatureFactoryUtils.getJTSTileBounds(z, x, y); + this.clipEnvelope = FeatureFactoryUtils.getJTSTileBounds(z, x, y); this.clipEnvelope.expandBy(tileEnvelope.getWidth() * 0.1d, tileEnvelope.getHeight() * 0.1d); this.builder = new JTSGeometryBuilder(geomFactory); // TODO: Not sure what is the difference between extent and tile size? - this.layerParams = new MvtLayerParams(extent, extent); + this.layerParams = new MvtLayerParams(extent, extent); } public List getFeatures(Geometry geometry, IUserDataConverter userData) { - TileGeomResult tileGeom = - JtsAdapter.createTileGeom(JtsAdapter.flatFeatureList(geometry.visit(builder)), - tileEnvelope, clipEnvelope, geomFactory, layerParams, acceptAllGeomFilter); + TileGeomResult tileGeom = JtsAdapter.createTileGeom( + JtsAdapter.flatFeatureList(geometry.visit(builder)), + tileEnvelope, + clipEnvelope, + geomFactory, + layerParams, + acceptAllGeomFilter + ); // MVT tile geometry to MVT features return JtsAdapter.toFeatures(tileGeom.mvtGeoms, layerProps, userData); } @@ -103,8 +111,8 @@ public org.locationtech.jts.geom.Geometry visit(MultiPoint multiPoint) throws Ru } private org.locationtech.jts.geom.Point buildPoint(Point point) { - final double x = VectorTileUtils.lonToSphericalMercator(point.getX()); - final double y = VectorTileUtils.latToSphericalMercator(point.getY()); + final double x = FeatureFactoryUtils.lonToSphericalMercator(point.getX()); + final double y = FeatureFactoryUtils.latToSphericalMercator(point.getY()); return geomFactory.createPoint(new Coordinate(x, y)); } @@ -125,8 +133,8 @@ public org.locationtech.jts.geom.Geometry visit(MultiLine multiLine) throws Runt private LineString buildLine(Line line) { final Coordinate[] coordinates = new Coordinate[line.length()]; for (int i = 0; i < line.length(); i++) { - final double x = VectorTileUtils.lonToSphericalMercator(line.getX(i)); - final double y = VectorTileUtils.latToSphericalMercator(line.getY(i)); + final double x = FeatureFactoryUtils.lonToSphericalMercator(line.getX(i)); + final double y = FeatureFactoryUtils.latToSphericalMercator(line.getY(i)); coordinates[i] = new Coordinate(x, y); } return geomFactory.createLineString(coordinates); @@ -161,8 +169,8 @@ private org.locationtech.jts.geom.Polygon buildPolygon(Polygon polygon) { private org.locationtech.jts.geom.LinearRing buildLinearRing(LinearRing ring) throws RuntimeException { final Coordinate[] coordinates = new Coordinate[ring.length()]; for (int i = 0; i < ring.length(); i++) { - final double x = VectorTileUtils.lonToSphericalMercator(ring.getX(i)); - final double y = VectorTileUtils.latToSphericalMercator(ring.getY(i)); + final double x = FeatureFactoryUtils.lonToSphericalMercator(ring.getX(i)); + final double y = FeatureFactoryUtils.latToSphericalMercator(ring.getY(i)); coordinates[i] = new Coordinate(x, y); } return geomFactory.createLinearRing(coordinates); @@ -171,10 +179,10 @@ private org.locationtech.jts.geom.LinearRing buildLinearRing(LinearRing ring) th @Override public org.locationtech.jts.geom.Geometry visit(Rectangle rectangle) throws RuntimeException { // TODO: handle degenerated rectangles? - final double xMin = VectorTileUtils.lonToSphericalMercator(rectangle.getMinX()); - final double yMin = VectorTileUtils.latToSphericalMercator(rectangle.getMinY()); - final double xMax = VectorTileUtils.lonToSphericalMercator(rectangle.getMaxX()); - final double yMax = VectorTileUtils.latToSphericalMercator(rectangle.getMaxY()); + final double xMin = FeatureFactoryUtils.lonToSphericalMercator(rectangle.getMinX()); + final double yMin = FeatureFactoryUtils.latToSphericalMercator(rectangle.getMinY()); + final double xMax = FeatureFactoryUtils.lonToSphericalMercator(rectangle.getMaxX()); + final double yMax = FeatureFactoryUtils.latToSphericalMercator(rectangle.getMaxY()); final Coordinate[] coordinates = new Coordinate[5]; coordinates[0] = new Coordinate(xMin, yMin); coordinates[1] = new Coordinate(xMax, yMin); diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java new file mode 100644 index 0000000000000..be588e395bdb4 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java @@ -0,0 +1,63 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import org.elasticsearch.geometry.Rectangle; +import org.locationtech.jts.geom.Envelope; + +/** + * Utility functions to transforms WGS84 coordinates into spherical mercator. + */ +class FeatureFactoryUtils { + + private FeatureFactoryUtils() { + // no instances + } + + /** + * Gets the JTS envelope for z/x/y/ tile in spherical mercator projection. + */ + public static Envelope getJTSTileBounds(int z, int x, int y) { + return new Envelope(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + /** + * Gets the {@link org.elasticsearch.geometry.Geometry} envelope for z/x/y/ tile + * in spherical mercator projection. + */ + public static Rectangle getTileBounds(int z, int x, int y) { + return new Rectangle(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); + } + + private static double getLong(int x, int zoom) { + return lonToSphericalMercator(x / Math.pow(2, zoom) * 360 - 180); + } + + private static double getLat(int y, int zoom) { + double r2d = 180 / Math.PI; + double n = Math.PI - 2 * Math.PI * y / Math.pow(2, zoom); + return latToSphericalMercator(r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))); + } + + private static double MERCATOR_FACTOR = 20037508.34 / 180.0; + + /** + * Transforms WGS84 longitude to a Spherical mercator longitude + */ + public static double lonToSphericalMercator(double lon) { + return lon * MERCATOR_FACTOR; + } + + /** + * Transforms WGS84 latitude to a Spherical mercator latitude + */ + public static double latToSphericalMercator(double lat) { + double y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180); + return y * MERCATOR_FACTOR; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java similarity index 78% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java rename to x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java index 62b356b276178..5033d28443b06 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileGeometryBuilder.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.spatial.vectortile; +package org.elasticsearch.xpack.vectortile.feature; import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; @@ -13,15 +13,19 @@ import org.apache.lucene.util.BitUtil; import org.elasticsearch.geometry.Rectangle; -class VectorTileGeometryBuilder { +/** + * Similar to {@link FeatureFactory} but only supports points and rectangles. It is just + * much more efficient for those shapes. + */ +public class SimpleFeatureFactory { private final int extent; private final Rectangle rectangle; private final double pointXScale, pointYScale; - VectorTileGeometryBuilder(int z, int x, int y, int extent) { + public SimpleFeatureFactory(int z, int x, int y, int extent) { this.extent = extent; - rectangle = VectorTileUtils.getTileBounds(z, x, y); + rectangle = FeatureFactoryUtils.getTileBounds(z, x, y); pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) extent); pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); } @@ -44,23 +48,23 @@ public void box(VectorTile.Tile.Feature.Builder featureBuilder, double minLon, d featureBuilder.addGeometry(BitUtil.zigZagEncode(minY)); featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.LineTo, 3)); // 1 + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxX - minX)); featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); - featureBuilder.addGeometry(BitUtil.zigZagEncode(maxY - minY)); // 2 - featureBuilder.addGeometry(BitUtil.zigZagEncode(maxX - minX)); featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); + featureBuilder.addGeometry(BitUtil.zigZagEncode(maxY - minY)); // 3 + featureBuilder.addGeometry(BitUtil.zigZagEncode(minX - maxX)); featureBuilder.addGeometry(BitUtil.zigZagEncode(0)); - featureBuilder.addGeometry(BitUtil.zigZagEncode(minY - maxY)); // close featureBuilder.addGeometry(GeomCmdHdr.cmdHdr(GeomCmd.ClosePath, 1)); } private int lat(double lat) { - return (int) Math.round(pointYScale * (VectorTileUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; + return (int) Math.round(pointYScale * (FeatureFactoryUtils.latToSphericalMercator(lat) - rectangle.getMinY())) + extent; } private int lon(double lon) { - return (int) Math.round(pointXScale * (VectorTileUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); + return (int) Math.round(pointXScale * (FeatureFactoryUtils.lonToSphericalMercator(lon) - rectangle.getMinX())); } } diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java new file mode 100644 index 0000000000000..d968a7f5133c9 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java @@ -0,0 +1,300 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.vectortile.rest; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; +import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; +import org.elasticsearch.action.search.SearchRequestBuilder; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.action.search.SearchResponseSections; +import org.elasticsearch.client.node.NodeClient; +import org.elasticsearch.common.document.DocumentField; +import org.elasticsearch.common.geo.GeoBoundingBox; +import org.elasticsearch.common.geo.GeoPoint; +import org.elasticsearch.common.geo.GeometryParser; +import org.elasticsearch.common.io.Streams; +import org.elasticsearch.common.io.stream.BytesStream; +import org.elasticsearch.geometry.Geometry; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.BoolQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.QueryBuilders; +import org.elasticsearch.rest.BaseRestHandler; +import org.elasticsearch.rest.BytesRestResponse; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.rest.RestResponse; +import org.elasticsearch.rest.RestStatus; +import org.elasticsearch.rest.action.RestCancellableNodeClient; +import org.elasticsearch.rest.action.RestResponseListener; +import org.elasticsearch.search.SearchHit; +import org.elasticsearch.search.SearchHits; +import org.elasticsearch.search.aggregations.Aggregation; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.Aggregations; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileGridAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoGridBucket; +import org.elasticsearch.search.aggregations.bucket.geogrid.InternalGeoTileGrid; +import org.elasticsearch.search.aggregations.metrics.GeoBoundsAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.InternalGeoBounds; +import org.elasticsearch.search.aggregations.pipeline.StatsBucketPipelineAggregationBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.profile.SearchProfileShardResults; +import org.elasticsearch.search.sort.SortBuilder; +import org.elasticsearch.xpack.vectortile.feature.FeatureFactory; +import org.elasticsearch.xpack.vectortile.feature.SimpleFeatureFactory; + +import java.io.IOException; +import java.util.Collection; +import java.util.List; +import java.util.stream.Collectors; + +import static org.elasticsearch.rest.RestRequest.Method.GET; + +/** + * Main class handling a call to the _mvt API. + */ +public class RestVectorTileAction extends BaseRestHandler { + + private static final String META_LAYER = "meta"; + private static final String HITS_LAYER = "hits"; + private static final String AGGS_LAYER = "aggs"; + + private static final String GRID_FIELD = "grid"; + private static final String BOUNDS_FIELD = "bounds"; + + private static final String COUNT_TAG = "_count"; + private static final String ID_TAG = "_id"; + + // mime type as defined by the mapbox vector tile specification + private static final String MIME_TYPE = "application/vnd.mapbox-vector-tile"; + + public RestVectorTileAction() {} + + @Override + public List routes() { + return List.of(new Route(GET, "{index}/_mvt/{field}/{z}/{x}/{y}")); + } + + @Override + public String getName() { + return "vector_tile_action"; + } + + @Override + protected RestChannelConsumer prepareRequest(RestRequest restRequest, NodeClient client) throws IOException { + // This will allow to cancel the search request if the http channel is closed + final RestCancellableNodeClient cancellableNodeClient = new RestCancellableNodeClient(client, restRequest.getHttpChannel()); + final VectorTileRequest request = VectorTileRequest.parseRestRequest(restRequest); + final SearchRequestBuilder searchRequestBuilder = searchRequestBuilder(cancellableNodeClient, request); + return channel -> searchRequestBuilder.execute(new RestResponseListener<>(channel) { + + @Override + public RestResponse buildResponse(SearchResponse searchResponse) throws Exception { + try (BytesStream bytesOut = Streams.flushOnCloseStream(channel.bytesOutput())) { + // Even if there is no hits, we return a tile with the meta layer + final VectorTile.Tile.Builder tileBuilder = VectorTile.Tile.newBuilder(); + ensureOpen(); + final SearchHit[] hits = searchResponse.getHits().getHits(); + if (hits.length > 0) { + tileBuilder.addLayers(buildHitsLayer(hits, request)); + } + ensureOpen(); + final SimpleFeatureFactory geomBuilder = new SimpleFeatureFactory( + request.getZ(), + request.getX(), + request.getY(), + request.getExtent() + ); + final InternalGeoTileGrid grid = searchResponse.getAggregations() != null + ? searchResponse.getAggregations().get(GRID_FIELD) + : null; + // TODO: should we expose the total number of buckets on InternalGeoTileGrid? + if (grid != null && grid.getBuckets().size() > 0) { + tileBuilder.addLayers(buildAggsLayer(grid, request, geomBuilder)); + } + ensureOpen(); + final InternalGeoBounds bounds = searchResponse.getAggregations() != null + ? searchResponse.getAggregations().get(BOUNDS_FIELD) + : null; + final Aggregations aggsWithoutGridAndBounds = searchResponse.getAggregations() == null + ? null + : new Aggregations( + searchResponse.getAggregations() + .asList() + .stream() + .filter(a -> GRID_FIELD.equals(a.getName()) == false && BOUNDS_FIELD.equals(a.getName()) == false) + .collect(Collectors.toList()) + ); + final SearchResponse meta = new SearchResponse( + new SearchResponseSections( + new SearchHits( + SearchHits.EMPTY, + searchResponse.getHits().getTotalHits(), + searchResponse.getHits().getMaxScore() + ), // remove actual hits + aggsWithoutGridAndBounds, + searchResponse.getSuggest(), + searchResponse.isTimedOut(), + searchResponse.isTerminatedEarly(), + searchResponse.getProfileResults() == null + ? null + : new SearchProfileShardResults(searchResponse.getProfileResults()), + searchResponse.getNumReducePhases() + ), + searchResponse.getScrollId(), + searchResponse.getTotalShards(), + searchResponse.getSuccessfulShards(), + searchResponse.getSkippedShards(), + searchResponse.getTook().millis(), + searchResponse.getShardFailures(), + searchResponse.getClusters() + ); + tileBuilder.addLayers(buildMetaLayer(meta, bounds, request, geomBuilder)); + ensureOpen(); + tileBuilder.build().writeTo(bytesOut); + return new BytesRestResponse(RestStatus.OK, MIME_TYPE, bytesOut.bytes()); + } + } + }); + } + + private static SearchRequestBuilder searchRequestBuilder(RestCancellableNodeClient client, VectorTileRequest request) + throws IOException { + final SearchRequestBuilder searchRequestBuilder = client.prepareSearch(request.getIndexes()); + searchRequestBuilder.setSize(request.getSize()); + searchRequestBuilder.setFetchSource(false); + // TODO: I wonder if we can leverage field and format so what we get in the result is already the mvt commands. + searchRequestBuilder.addFetchField(new FieldAndFormat(request.getField(), null)); + for (FieldAndFormat field : request.getFieldAndFormats()) { + searchRequestBuilder.addFetchField(field); + } + searchRequestBuilder.setRuntimeMappings(request.getRuntimeMappings()); + QueryBuilder qBuilder = QueryBuilders.geoShapeQuery(request.getField(), request.getBoundingBox()); + if (request.getQueryBuilder() != null) { + final BoolQueryBuilder boolQueryBuilder = QueryBuilders.boolQuery(); + boolQueryBuilder.filter(request.getQueryBuilder()); + boolQueryBuilder.filter(qBuilder); + qBuilder = boolQueryBuilder; + } + searchRequestBuilder.setQuery(qBuilder); + if (request.getGridPrecision() > 0) { + final Rectangle rectangle = request.getBoundingBox(); + final GeoBoundingBox boundingBox = new GeoBoundingBox( + new GeoPoint(rectangle.getMaxLat(), rectangle.getMinLon()), + new GeoPoint(rectangle.getMinLat(), rectangle.getMaxLon()) + ); + final int extent = 1 << request.getGridPrecision(); + final GeoGridAggregationBuilder tileAggBuilder = new GeoTileGridAggregationBuilder(GRID_FIELD).field(request.getField()) + .precision(Math.min(GeoTileUtils.MAX_ZOOM, request.getZ() + request.getGridPrecision())) + .setGeoBoundingBox(boundingBox) + .size(extent * extent); + searchRequestBuilder.addAggregation(tileAggBuilder); + searchRequestBuilder.addAggregation(new StatsBucketPipelineAggregationBuilder(COUNT_TAG, GRID_FIELD + "." + COUNT_TAG)); + final AggregatorFactories.Builder otherAggBuilder = request.getAggBuilder(); + if (otherAggBuilder != null) { + tileAggBuilder.subAggregations(request.getAggBuilder()); + final Collection aggregations = otherAggBuilder.getAggregatorFactories(); + for (AggregationBuilder aggregation : aggregations) { + searchRequestBuilder.addAggregation( + new StatsBucketPipelineAggregationBuilder(aggregation.getName(), GRID_FIELD + ">" + aggregation.getName()) + ); + } + } + } + if (request.getExactBounds()) { + final GeoBoundsAggregationBuilder boundsBuilder = new GeoBoundsAggregationBuilder(BOUNDS_FIELD).field(request.getField()) + .wrapLongitude(false); + searchRequestBuilder.addAggregation(boundsBuilder); + } + for (SortBuilder sortBuilder : request.getSortBuilders()) { + searchRequestBuilder.addSort(sortBuilder); + } + return searchRequestBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildHitsLayer(SearchHit[] hits, VectorTileRequest request) { + final FeatureFactory featureFactory = new FeatureFactory(request.getZ(), request.getX(), request.getY(), request.getExtent()); + final GeometryParser parser = new GeometryParser(true, false, false); + final VectorTile.Tile.Layer.Builder hitsLayerBuilder = VectorTileUtils.createLayerBuilder(HITS_LAYER, request.getExtent()); + final List fields = request.getFieldAndFormats(); + for (SearchHit searchHit : hits) { + final IUserDataConverter tags = (userData, layerProps, featureBuilder) -> { + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, ID_TAG, searchHit.getId()); + if (fields != null) { + for (FieldAndFormat field : fields) { + final DocumentField documentField = searchHit.field(field.field); + if (documentField != null) { + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, field.field, documentField.getValue()); + } + } + } + }; + // TODO: See comment on field formats. + final Geometry geometry = parser.parseGeometry(searchHit.field(request.getField()).getValue()); + hitsLayerBuilder.addAllFeatures(featureFactory.getFeatures(geometry, tags)); + } + VectorTileUtils.addPropertiesToLayer(hitsLayerBuilder, featureFactory.getLayerProps()); + return hitsLayerBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildAggsLayer( + InternalGeoTileGrid grid, + VectorTileRequest request, + SimpleFeatureFactory geomBuilder + ) throws IOException { + final VectorTile.Tile.Layer.Builder aggLayerBuilder = VectorTileUtils.createLayerBuilder(AGGS_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (InternalGeoGridBucket bucket : grid.getBuckets()) { + featureBuilder.clear(); + // Add geometry + if (request.getGridType() == VectorTileRequest.GRID_TYPE.GRID) { + final Rectangle r = GeoTileUtils.toBoundingBox(bucket.getKeyAsString()); + geomBuilder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + } else { + // TODO: it should be the centroid of the data? + final GeoPoint point = (GeoPoint) bucket.getKey(); + geomBuilder.point(featureBuilder, point.lon(), point.lat()); + } + // Add count as key value pair + VectorTileUtils.addPropertyToFeature(featureBuilder, layerProps, COUNT_TAG, bucket.getDocCount()); + for (Aggregation aggregation : bucket.getAggregations()) { + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, aggregation); + } + aggLayerBuilder.addFeatures(featureBuilder); + } + VectorTileUtils.addPropertiesToLayer(aggLayerBuilder, layerProps); + return aggLayerBuilder; + } + + private static VectorTile.Tile.Layer.Builder buildMetaLayer( + SearchResponse response, + InternalGeoBounds bounds, + VectorTileRequest request, + SimpleFeatureFactory geomBuilder + ) throws IOException { + final VectorTile.Tile.Layer.Builder metaLayerBuilder = VectorTileUtils.createLayerBuilder(META_LAYER, request.getExtent()); + final MvtLayerProps layerProps = new MvtLayerProps(); + final VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + if (bounds != null && bounds.topLeft() != null) { + final GeoPoint topLeft = bounds.topLeft(); + final GeoPoint bottomRight = bounds.bottomRight(); + geomBuilder.box(featureBuilder, topLeft.lon(), bottomRight.lon(), bottomRight.lat(), topLeft.lat()); + } else { + final Rectangle tile = request.getBoundingBox(); + geomBuilder.box(featureBuilder, tile.getMinLon(), tile.getMaxLon(), tile.getMinLat(), tile.getMaxLat()); + } + VectorTileUtils.addToXContentToFeature(featureBuilder, layerProps, response); + metaLayerBuilder.addFeatures(featureBuilder); + VectorTileUtils.addPropertiesToLayer(metaLayerBuilder, layerProps); + return metaLayerBuilder; + } +} diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java new file mode 100644 index 0000000000000..c2f90b0a3db96 --- /dev/null +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java @@ -0,0 +1,309 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import org.elasticsearch.common.CheckedFunction; +import org.elasticsearch.common.ParseField; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.index.query.AbstractQueryBuilder; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.AggregatorFactories; +import org.elasticsearch.search.aggregations.PipelineAggregationBuilder; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.CardinalityAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MaxAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.MinAggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.SumAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.fetch.subphase.FieldAndFormat; +import org.elasticsearch.search.sort.SortBuilder; + +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Locale; +import java.util.Map; + +import static java.util.Collections.emptyList; +import static java.util.Collections.emptyMap; + +/** + * Transforms a rest request in a vector tile request + */ +class VectorTileRequest { + + protected static final String INDEX_PARAM = "index"; + protected static final String FIELD_PARAM = "field"; + protected static final String Z_PARAM = "z"; + protected static final String X_PARAM = "x"; + protected static final String Y_PARAM = "y"; + + protected static final ParseField GRID_PRECISION_FIELD = new ParseField("grid_precision"); + protected static final ParseField GRID_TYPE_FIELD = new ParseField("grid_type"); + protected static final ParseField EXTENT_FIELD = new ParseField("extent"); + protected static final ParseField EXACT_BOUNDS_FIELD = new ParseField("exact_bounds"); + + protected enum GRID_TYPE { + GRID, + POINT; + + private static GRID_TYPE fromString(String type) { + switch (type.toLowerCase(Locale.ROOT)) { + case "grid": + return GRID; + case "point": + return POINT; + default: + throw new IllegalArgumentException("Invalid grid type [" + type + "]"); + } + } + } + + protected static class Defaults { + // TODO: Should it be SearchService.DEFAULT_SIZE? + public static final int SIZE = 10000; + public static final List FETCH = emptyList(); + public static final Map RUNTIME_MAPPINGS = emptyMap(); + public static final QueryBuilder QUERY = null; + public static final AggregatorFactories.Builder AGGS = null; + public static final List> SORT = emptyList(); + // TODO: Should it be 0, no aggs by default? + public static final int GRID_PRECISION = 8; + public static final GRID_TYPE GRID_TYPE = VectorTileRequest.GRID_TYPE.GRID; + public static final int EXTENT = 4096; + public static final boolean EXACT_BOUNDS = false; + } + + private static final ObjectParser PARSER; + + static { + PARSER = new ObjectParser<>("vector-tile"); + PARSER.declareInt(VectorTileRequest::setSize, SearchSourceBuilder.SIZE_FIELD); + PARSER.declareField(VectorTileRequest::setFieldAndFormats, (p) -> { + List fetchFields = new ArrayList<>(); + while ((p.nextToken()) != XContentParser.Token.END_ARRAY) { + fetchFields.add(FieldAndFormat.fromXContent(p)); + } + return fetchFields; + }, SearchSourceBuilder.FETCH_FIELDS_FIELD, ObjectParser.ValueType.OBJECT_ARRAY); + PARSER.declareField( + VectorTileRequest::setQueryBuilder, + (CheckedFunction) AbstractQueryBuilder::parseInnerQueryBuilder, + SearchSourceBuilder.QUERY_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setRuntimeMappings, + XContentParser::map, + SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setAggBuilder, + AggregatorFactories::parseAggregators, + SearchSourceBuilder.AGGS_FIELD, + ObjectParser.ValueType.OBJECT + ); + PARSER.declareField( + VectorTileRequest::setSortBuilders, + SortBuilder::fromXContent, + SearchSourceBuilder.SORT_FIELD, + ObjectParser.ValueType.OBJECT_ARRAY + ); + // Specific for vector tiles + PARSER.declareInt(VectorTileRequest::setGridPrecision, GRID_PRECISION_FIELD); + PARSER.declareString(VectorTileRequest::setGridType, GRID_TYPE_FIELD); + PARSER.declareInt(VectorTileRequest::setExtent, EXTENT_FIELD); + PARSER.declareBoolean(VectorTileRequest::setExactBounds, EXACT_BOUNDS_FIELD); + } + + static VectorTileRequest parseRestRequest(RestRequest restRequest) throws IOException { + final VectorTileRequest request = new VectorTileRequest( + Strings.splitStringByCommaToArray(restRequest.param(INDEX_PARAM)), + restRequest.param(FIELD_PARAM), + Integer.parseInt(restRequest.param(Z_PARAM)), + Integer.parseInt(restRequest.param(X_PARAM)), + Integer.parseInt(restRequest.param(Y_PARAM)) + ); + if (restRequest.hasContent()) { + try (XContentParser contentParser = restRequest.contentParser()) { + PARSER.parse(contentParser, request, restRequest); + } + } + return request; + } + + private final String[] indexes; + private final String field; + private final int x; + private final int y; + private final int z; + private final Rectangle bbox; + private QueryBuilder queryBuilder = Defaults.QUERY; + private Map runtimeMappings = Defaults.RUNTIME_MAPPINGS; + private int gridPrecision = Defaults.GRID_PRECISION; + private GRID_TYPE gridType = Defaults.GRID_TYPE; + private int size = Defaults.SIZE; + private int extent = Defaults.EXTENT; + private AggregatorFactories.Builder aggBuilder = Defaults.AGGS; + private List fields = Defaults.FETCH; + private List> sortBuilders = Defaults.SORT; + private boolean exact_bounds = Defaults.EXACT_BOUNDS; + + private VectorTileRequest(String[] indexes, String field, int z, int x, int y) { + this.indexes = indexes; + this.field = field; + this.z = z; + this.x = x; + this.y = y; + // This should validate that z/x/y is a valid combination + this.bbox = GeoTileUtils.toBoundingBox(x, y, z); + } + + public String[] getIndexes() { + return indexes; + } + + public String getField() { + return field; + } + + public int getX() { + return x; + } + + public int getY() { + return y; + } + + public int getZ() { + return z; + } + + public Rectangle getBoundingBox() { + return bbox; + } + + public int getExtent() { + return extent; + } + + private void setExtent(int extent) { + if (extent < 0) { + throw new IllegalArgumentException("[extent] parameter cannot be negative, found [" + extent + "]"); + } + this.extent = extent; + } + + public boolean getExactBounds() { + return exact_bounds; + } + + private void setExactBounds(boolean exact_bounds) { + this.exact_bounds = exact_bounds; + } + + public List getFieldAndFormats() { + return fields; + } + + private void setFieldAndFormats(List fields) { + this.fields = fields; + } + + public QueryBuilder getQueryBuilder() { + return queryBuilder; + } + + private void setQueryBuilder(QueryBuilder queryBuilder) { + // TODO: validation + this.queryBuilder = queryBuilder; + } + + public Map getRuntimeMappings() { + return runtimeMappings; + } + + private void setRuntimeMappings(Map runtimeMappings) { + this.runtimeMappings = runtimeMappings; + } + + public int getGridPrecision() { + return gridPrecision; + } + + private void setGridPrecision(int gridPrecision) { + if (gridPrecision < 0) { + throw new IllegalArgumentException("[gridPrecision] parameter cannot be negative, found [" + gridPrecision + "]"); + } + if (gridPrecision > 8) { + throw new IllegalArgumentException("[gridPrecision] parameter cannot be bigger than 8, found [" + gridPrecision + "]"); + } + this.gridPrecision = gridPrecision; + } + + public GRID_TYPE getGridType() { + return gridType; + } + + private void setGridType(String gridType) { + this.gridType = GRID_TYPE.fromString(gridType); + } + + public int getSize() { + return size; + } + + private void setSize(int size) { + if (size < 0) { + throw new IllegalArgumentException("[size] parameter cannot be negative, found [" + size + "]"); + } + this.size = size; + } + + public AggregatorFactories.Builder getAggBuilder() { + return aggBuilder; + } + + private void setAggBuilder(AggregatorFactories.Builder aggBuilder) { + for (AggregationBuilder aggregation : aggBuilder.getAggregatorFactories()) { + final String type = aggregation.getType(); + switch (type) { + case MinAggregationBuilder.NAME: + case MaxAggregationBuilder.NAME: + case AvgAggregationBuilder.NAME: + case SumAggregationBuilder.NAME: + case CardinalityAggregationBuilder.NAME: + break; + default: + // top term and percentile should be supported + throw new IllegalArgumentException("Unsupported aggregation of type [" + type + "]"); + } + } + for (PipelineAggregationBuilder aggregation : aggBuilder.getPipelineAggregatorFactories()) { + // should not have pipeline aggregations + final String type = aggregation.getType(); + throw new IllegalArgumentException("Unsupported pipeline aggregation of type [" + type + "]"); + } + this.aggBuilder = aggBuilder; + } + + public List> getSortBuilders() { + return sortBuilders; + } + + private void setSortBuilders(List> sortBuilders) { + this.sortBuilders = sortBuilders; + } +} diff --git a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java similarity index 59% rename from x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java rename to x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java index 2ac20b9bba1ad..aa4015343fc37 100644 --- a/x-pack/plugin/spatial/src/main/java/org/elasticsearch/xpack/spatial/vectortile/VectorTileUtils.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java @@ -5,7 +5,7 @@ * 2.0. */ -package org.elasticsearch.xpack.spatial.vectortile; +package org.elasticsearch.xpack.vectortile.rest; import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; @@ -14,16 +14,18 @@ import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentHelper; import org.elasticsearch.common.xcontent.XContentType; -import org.elasticsearch.geometry.Rectangle; -import org.locationtech.jts.geom.Envelope; import java.io.IOException; import java.util.Map; /** - * Utility methods For vector tiles. Transforms WGS84 into spherical mercator. + * Utility methods for vector tiles. */ -public class VectorTileUtils { +class VectorTileUtils { + + private VectorTileUtils() { + // no instances + } /** * Creates a vector layer builder with the provided name and extent. @@ -61,6 +63,9 @@ public static void addPropertyToFeature(VectorTile.Tile.Feature.Builder feature, feature.addTags(layerProps.addValue(value)); } + /** + * Adds the given properties to the provided layer. + */ public static void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, MvtLayerProps layerProps) { // Add keys layer.addAllKeys(layerProps.getKeys()); @@ -70,47 +75,4 @@ public static void addPropertiesToLayer(VectorTile.Tile.Layer.Builder layer, Mvt layer.addValues(MvtValue.toValue(value)); } } - - /** - * Gets the JTS envelope for z/x/y/ tile in spherical mercator projection. - */ - public static Envelope getJTSTileBounds(int z, int x, int y) { - return new Envelope(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); - } - - /** - * Gets the {@link org.elasticsearch.geometry.Geometry} envelope for z/x/y/ tile - * in spherical mercator projection. - */ - public static Rectangle getTileBounds(int z, int x, int y) { - return new Rectangle(getLong(x, z), getLong(x + 1, z), getLat(y, z), getLat(y + 1, z)); - } - - private static double getLong(int x, int zoom) - { - return lonToSphericalMercator( x / Math.pow(2, zoom) * 360 - 180 ); - } - - private static double getLat(int y, int zoom) { - double r2d = 180 / Math.PI; - double n = Math.PI - 2 * Math.PI * y / Math.pow(2, zoom); - return latToSphericalMercator(r2d * Math.atan(0.5 * (Math.exp(n) - Math.exp(-n)))); - } - - private static double MERCATOR_FACTOR = 20037508.34 / 180.0; - - /** - * Transforms WGS84 longitude to a Spherical mercator longitude - */ - public static double lonToSphericalMercator(double lon) { - return lon * MERCATOR_FACTOR; - } - - /** - * Transforms WGS84 latitude to a Spherical mercator latitude - */ - public static double latToSphericalMercator(double lat) { - double y = Math.log(Math.tan((90 + lat) * Math.PI / 360)) / (Math.PI / 180); - return y * MERCATOR_FACTOR; - } } diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java new file mode 100644 index 0000000000000..7bef903810ddf --- /dev/null +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java @@ -0,0 +1,65 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.feature; + +import com.wdtinc.mapbox_vector_tile.VectorTile; +import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; +import org.apache.lucene.geo.GeoTestUtil; +import org.elasticsearch.geometry.Point; +import org.elasticsearch.geometry.Rectangle; +import org.elasticsearch.search.aggregations.bucket.geogrid.GeoTileUtils; +import org.elasticsearch.test.ESTestCase; +import org.hamcrest.Matchers; + +import java.util.List; + +public class FeatureFactoryTests extends ESTestCase { + + public void testPoint() { + int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + int extent = randomIntBetween(1 << 8, 1 << 14); + Rectangle rectangle = GeoTileUtils.toBoundingBox(x, y, z); + SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); + FeatureFactory factory = new FeatureFactory(z, x, y, extent); + VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (int i = 0; i < 10; i++) { + featureBuilder.clear(); + double lat = randomValueOtherThanMany((l) -> rectangle.getMinY() > l || rectangle.getMaxY() < l, GeoTestUtil::nextLatitude); + double lon = randomValueOtherThanMany((l) -> rectangle.getMinX() > l || rectangle.getMaxX() < l, GeoTestUtil::nextLongitude); + builder.point(featureBuilder, lon, lat); + byte[] b1 = featureBuilder.build().toByteArray(); + Point point = new Point(lon, lat); + List features = factory.getFeatures(point, new UserDataIgnoreConverter()); + assertThat(features.size(), Matchers.equalTo(1)); + byte[] b2 = features.get(0).toByteArray(); + assertArrayEquals(b1, b2); + } + } + + public void testRectangle() { + int z = randomIntBetween(1, 10); + int x = randomIntBetween(0, (1 << z) - 1); + int y = randomIntBetween(0, (1 << z) - 1); + int extent = randomIntBetween(1 << 8, 1 << 14); + SimpleFeatureFactory builder = new SimpleFeatureFactory(z, x, y, extent); + FeatureFactory factory = new FeatureFactory(z, x, y, extent); + Rectangle r = GeoTileUtils.toBoundingBox(x, y, z); + VectorTile.Tile.Feature.Builder featureBuilder = VectorTile.Tile.Feature.newBuilder(); + for (int i = 0; i < extent; i++) { + featureBuilder.clear(); + builder.box(featureBuilder, r.getMinLon(), r.getMaxLon(), r.getMinLat(), r.getMaxLat()); + byte[] b1 = featureBuilder.build().toByteArray(); + List features = factory.getFeatures(r, new UserDataIgnoreConverter()); + assertThat(features.size(), Matchers.equalTo(1)); + byte[] b2 = features.get(0).toByteArray(); + assertArrayEquals(extent + "", b1, b2); + } + } +} diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java new file mode 100644 index 0000000000000..16d96f1b2ad7d --- /dev/null +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java @@ -0,0 +1,213 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.vectortile.rest; + +import com.carrotsearch.randomizedtesting.generators.RandomPicks; +import org.elasticsearch.common.CheckedConsumer; +import org.elasticsearch.common.bytes.BytesReference; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.NamedXContentRegistry; +import org.elasticsearch.common.xcontent.ToXContent; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.index.query.QueryBuilder; +import org.elasticsearch.index.query.TermQueryBuilder; +import org.elasticsearch.rest.RestRequest; +import org.elasticsearch.search.SearchModule; +import org.elasticsearch.search.aggregations.AggregationBuilder; +import org.elasticsearch.search.aggregations.metrics.AvgAggregationBuilder; +import org.elasticsearch.search.builder.SearchSourceBuilder; +import org.elasticsearch.search.sort.FieldSortBuilder; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.test.rest.FakeRestRequest; +import org.hamcrest.Matchers; + +import java.io.IOException; +import java.util.Map; +import java.util.function.Consumer; + +import static java.util.Collections.emptyList; + +public class VectorTileRequestTests extends ESTestCase { + + @Override + protected NamedXContentRegistry xContentRegistry() { + SearchModule searchModule = new SearchModule(Settings.EMPTY, emptyList()); + return new NamedXContentRegistry(searchModule.getNamedXContents()); + } + + public void testDefaults() throws IOException { + assertRestRequest((builder) -> {}, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getSize(), Matchers.equalTo(VectorTileRequest.Defaults.SIZE)); + assertThat(vectorTileRequest.getExtent(), Matchers.equalTo(VectorTileRequest.Defaults.EXTENT)); + assertThat(vectorTileRequest.getAggBuilder(), Matchers.equalTo(VectorTileRequest.Defaults.AGGS)); + assertThat(vectorTileRequest.getFieldAndFormats(), Matchers.equalTo(VectorTileRequest.Defaults.FETCH)); + assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_TYPE)); + assertThat(vectorTileRequest.getGridPrecision(), Matchers.equalTo(VectorTileRequest.Defaults.GRID_PRECISION)); + assertThat(vectorTileRequest.getExactBounds(), Matchers.equalTo(VectorTileRequest.Defaults.EXACT_BOUNDS)); + assertThat(vectorTileRequest.getRuntimeMappings(), Matchers.equalTo(VectorTileRequest.Defaults.RUNTIME_MAPPINGS)); + assertThat(vectorTileRequest.getSortBuilders(), Matchers.equalTo(VectorTileRequest.Defaults.SORT)); + assertThat(vectorTileRequest.getQueryBuilder(), Matchers.equalTo(VectorTileRequest.Defaults.QUERY)); + }); + } + + public void testFieldSize() throws IOException { + final int size = randomIntBetween(0, 10000); + assertRestRequest( + (builder) -> { builder.field(SearchSourceBuilder.SIZE_FIELD.getPreferredName(), size); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getSize(), Matchers.equalTo(size)); } + ); + } + + public void testFieldExtent() throws IOException { + final int extent = randomIntBetween(256, 8192); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.EXTENT_FIELD.getPreferredName(), extent); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getExtent(), Matchers.equalTo(extent)); } + ); + } + + public void testFieldFetch() throws IOException { + final String fetchField = randomAlphaOfLength(10); + assertRestRequest( + (builder) -> { builder.field(SearchSourceBuilder.FETCH_FIELDS_FIELD.getPreferredName(), new String[] { fetchField }); }, + (vectorTileRequest) -> { + assertThat(vectorTileRequest.getFieldAndFormats(), Matchers.iterableWithSize(1)); + assertThat(vectorTileRequest.getFieldAndFormats().get(0).field, Matchers.equalTo(fetchField)); + } + ); + } + + public void testFieldGridType() throws IOException { + final VectorTileRequest.GRID_TYPE grid_type = RandomPicks.randomFrom(random(), VectorTileRequest.GRID_TYPE.values()); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.GRID_TYPE_FIELD.getPreferredName(), grid_type.name()); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridType(), Matchers.equalTo(grid_type)); } + ); + } + + public void testFieldGridPrecision() throws IOException { + final int grid_precision = randomIntBetween(1, 8); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.GRID_PRECISION_FIELD.getPreferredName(), grid_precision); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getGridPrecision(), Matchers.equalTo(grid_precision)); } + ); + } + + public void testFieldExactBounds() throws IOException { + final boolean exactBounds = randomBoolean(); + assertRestRequest( + (builder) -> { builder.field(VectorTileRequest.EXACT_BOUNDS_FIELD.getPreferredName(), exactBounds); }, + (vectorTileRequest) -> { assertThat(vectorTileRequest.getExactBounds(), Matchers.equalTo(exactBounds)); } + ); + } + + public void testFieldQuery() throws IOException { + final QueryBuilder queryBuilder = new TermQueryBuilder(randomAlphaOfLength(10), randomAlphaOfLength(10)); + assertRestRequest((builder) -> { + builder.field(SearchSourceBuilder.QUERY_FIELD.getPreferredName()); + queryBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + }, (vectorTileRequest) -> { assertThat(vectorTileRequest.getQueryBuilder(), Matchers.equalTo(queryBuilder)); }); + } + + public void testFieldAgg() throws IOException { + final AggregationBuilder aggregationBuilder = new AvgAggregationBuilder("xxx").field("xxxx"); + assertRestRequest((builder) -> { + builder.startObject(SearchSourceBuilder.AGGS_FIELD.getPreferredName()); + aggregationBuilder.toXContent(builder, ToXContent.EMPTY_PARAMS); + builder.endObject(); + }, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getAggBuilder().getAggregatorFactories(), Matchers.iterableWithSize(1)); + assertThat(vectorTileRequest.getAggBuilder().getAggregatorFactories().contains(aggregationBuilder), Matchers.equalTo(true)); + }); + } + + public void testFieldRuntimeMappings() throws IOException { + final String fieldName = randomAlphaOfLength(10); + assertRestRequest((builder) -> { + builder.startObject(SearchSourceBuilder.RUNTIME_MAPPINGS_FIELD.getPreferredName()) + .startObject(fieldName) + .field("script", "emit('foo')") + .field("type", "string") + .endObject() + .endObject(); + }, (vectorTileRequest) -> { + assertThat(vectorTileRequest.getRuntimeMappings(), Matchers.aMapWithSize(1)); + assertThat(vectorTileRequest.getRuntimeMappings().get(fieldName), Matchers.notNullValue()); + }); + } + + public void testFieldSort() throws IOException { + final String sortName = randomAlphaOfLength(10); + assertRestRequest( + (builder) -> { + builder.startArray(SearchSourceBuilder.SORT_FIELD.getPreferredName()) + .startObject() + .field(sortName, "desc") + .endObject() + .endArray(); + }, + (vectorTileRequest) -> { + assertThat(vectorTileRequest.getSortBuilders(), Matchers.iterableWithSize(1)); + FieldSortBuilder sortBuilder = (FieldSortBuilder) vectorTileRequest.getSortBuilders().get(0); + assertThat(sortBuilder.getFieldName(), Matchers.equalTo(sortName)); + } + ); + } + + public void testWrongTile() { + final int z = randomIntBetween(1, 10); + final int x = -randomIntBetween(0, (1 << z) - 1); + final int y = -randomIntBetween(0, (1 << z) - 1); + final String index = randomAlphaOfLength(10); + final String field = randomAlphaOfLength(10); + final FakeRestRequest request = getBasicRequestBuilder(index, field, z, x, y).build(); + final IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> VectorTileRequest.parseRestRequest(request)); + assertThat(ex.getMessage(), Matchers.equalTo("Zoom/X/Y combination is not valid: " + z + "/" + x + "/" + y)); + } + + private void assertRestRequest(CheckedConsumer consumer, Consumer asserter) + throws IOException { + final int z = randomIntBetween(1, 10); + final int x = randomIntBetween(0, (1 << z) - 1); + final int y = randomIntBetween(0, (1 << z) - 1); + final String index = randomAlphaOfLength(10); + final String field = randomAlphaOfLength(10); + final FakeRestRequest.Builder requestBuilder = getBasicRequestBuilder(index, field, z, x, y); + final XContentBuilder builder = JsonXContent.contentBuilder(); + builder.startObject(); + consumer.accept(builder); + builder.endObject(); + final FakeRestRequest request = requestBuilder.withContent(BytesReference.bytes(builder), builder.contentType()).build(); + final VectorTileRequest vectorTileRequest = VectorTileRequest.parseRestRequest(request); + assertThat(vectorTileRequest.getIndexes(), Matchers.equalTo(new String[] { index })); + assertThat(vectorTileRequest.getField(), Matchers.equalTo(field)); + assertThat(vectorTileRequest.getZ(), Matchers.equalTo(z)); + assertThat(vectorTileRequest.getX(), Matchers.equalTo(x)); + assertThat(vectorTileRequest.getY(), Matchers.equalTo(y)); + asserter.accept(vectorTileRequest); + } + + private FakeRestRequest.Builder getBasicRequestBuilder(String index, String field, int z, int x, int y) { + return new FakeRestRequest.Builder(xContentRegistry()).withMethod(RestRequest.Method.GET) + .withParams( + Map.of( + VectorTileRequest.INDEX_PARAM, + index, + VectorTileRequest.FIELD_PARAM, + field, + VectorTileRequest.Z_PARAM, + "" + z, + VectorTileRequest.X_PARAM, + "" + x, + VectorTileRequest.Y_PARAM, + "" + y + ) + ); + } +} From f3648830edbc0d4882be1d6d56de2cc544232d18 Mon Sep 17 00:00:00 2001 From: Ignacio Vera Date: Mon, 7 Jun 2021 12:55:02 +0200 Subject: [PATCH 10/15] Fix extra line in build.gradle from spatial module (#73816) --- x-pack/plugin/spatial/build.gradle | 1 - 1 file changed, 1 deletion(-) diff --git a/x-pack/plugin/spatial/build.gradle b/x-pack/plugin/spatial/build.gradle index acd8d27d5a7e5..9169e7f4c41e7 100644 --- a/x-pack/plugin/spatial/build.gradle +++ b/x-pack/plugin/spatial/build.gradle @@ -14,7 +14,6 @@ dependencies { compileOnly project(path: xpackModule('core')) testImplementation(testArtifact(project(xpackModule('core')))) yamlRestTestImplementation(testArtifact(project(xpackModule('core')))) - restTestConfig project(path: ':modules:geo', configuration: 'restTests') api project(path: ':modules:geo') restTestConfig project(path: ':modules:geo', configuration: 'restTests') } From 88a43f9133542261560b7b30070b2f188e182e8b Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Jun 2021 11:28:42 -1000 Subject: [PATCH 11/15] Update x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java Co-authored-by: Adrien Grand --- .../xpack/vectortile/feature/FeatureFactoryUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java index be588e395bdb4..ced22f16f6d51 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryUtils.java @@ -35,7 +35,7 @@ public static Rectangle getTileBounds(int z, int x, int y) { } private static double getLong(int x, int zoom) { - return lonToSphericalMercator(x / Math.pow(2, zoom) * 360 - 180); + return lonToSphericalMercator(Math.scalb(x, -zoom) * 360 - 180); } private static double getLat(int y, int zoom) { From 106a9652b1db791f80fafa367fc7732171159bd4 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Jun 2021 11:29:10 -1000 Subject: [PATCH 12/15] Update x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java Co-authored-by: Adrien Grand --- .../xpack/vectortile/feature/SimpleFeatureFactory.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java index 5033d28443b06..3fa37081a22bb 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java @@ -26,7 +26,7 @@ public class SimpleFeatureFactory { public SimpleFeatureFactory(int z, int x, int y, int extent) { this.extent = extent; rectangle = FeatureFactoryUtils.getTileBounds(z, x, y); - pointXScale = 1d / ((rectangle.getMaxLon() - rectangle.getMinLon()) / (double) extent); + pointXScale = (double) extent / (rectangle.getMaxLon() - rectangle.getMinLon()); pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); } From 19ade1e1737602382a07fe7a70f5f660efeb222c Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Jun 2021 11:33:53 -1000 Subject: [PATCH 13/15] Fix vector tile plugin after master merge --- .../xpack/vectortile/rest/VectorTileRequest.java | 4 ++-- .../xpack/vectortile/rest/VectorTileRequestTests.java | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java index c2f90b0a3db96..34a136aba21f4 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequest.java @@ -7,11 +7,11 @@ package org.elasticsearch.xpack.vectortile.rest; -import org.elasticsearch.common.CheckedFunction; -import org.elasticsearch.common.ParseField; import org.elasticsearch.common.Strings; import org.elasticsearch.common.xcontent.ObjectParser; +import org.elasticsearch.common.xcontent.ParseField; import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.core.CheckedFunction; import org.elasticsearch.geometry.Rectangle; import org.elasticsearch.index.query.AbstractQueryBuilder; import org.elasticsearch.index.query.QueryBuilder; diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java index 16d96f1b2ad7d..0f9905a2d3fc2 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java @@ -8,13 +8,13 @@ package org.elasticsearch.xpack.vectortile.rest; import com.carrotsearch.randomizedtesting.generators.RandomPicks; -import org.elasticsearch.common.CheckedConsumer; import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentBuilder; import org.elasticsearch.common.xcontent.json.JsonXContent; +import org.elasticsearch.core.CheckedConsumer; import org.elasticsearch.index.query.QueryBuilder; import org.elasticsearch.index.query.TermQueryBuilder; import org.elasticsearch.rest.RestRequest; From 29f939a143602fae263ad55a62613344cf339a6a Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Jun 2021 11:34:11 -1000 Subject: [PATCH 14/15] Address review comments --- .../java/org/elasticsearch/common/util/Maps.java | 12 ++++++++++++ .../xpack/vectortile/VectorTilePlugin.java | 13 ++----------- .../vectortile/feature/SimpleFeatureFactory.java | 2 +- 3 files changed, 15 insertions(+), 12 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/common/util/Maps.java b/server/src/main/java/org/elasticsearch/common/util/Maps.java index 273678e338215..02a041fdc9ff7 100644 --- a/server/src/main/java/org/elasticsearch/common/util/Maps.java +++ b/server/src/main/java/org/elasticsearch/common/util/Maps.java @@ -140,6 +140,18 @@ public static boolean deepEquals(Map left, Map right) { .allMatch(e -> right.containsKey(e.getKey()) && Objects.deepEquals(e.getValue(), right.get(e.getKey()))); } + /** + * Returns an array where all internal maps and optionally arrays are flattened into the root map. + * + * For example the map {"foo": {"bar": 1, "baz": [2, 3]}} will become {"foo.bar": 1, "foo.baz.0": 2, "foo.baz.1": 3}. Note that if + * maps contains keys with "." or numbers it is possible that such keys will be silently overridden. For example the map + * {"foo": {"bar": 1}, "foo.bar": 2} will become {"foo.bar": 1} or {"foo.bar": 2}. + * + * @param map - input to be flattened + * @param flattenArrays - if false, arrays will be ignored + * @param ordered - if true the resulted map will be sorted + * @return + */ public static Map flatten(Map map, boolean flattenArrays, boolean ordered) { return flatten(map, flattenArrays, ordered, null); } diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java index 3d659e826fca2..8ab4fae0900dd 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/VectorTilePlugin.java @@ -13,6 +13,7 @@ import org.elasticsearch.common.settings.IndexScopedSettings; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.settings.SettingsFilter; +import org.elasticsearch.core.Booleans; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.plugins.ActionPlugin; import org.elasticsearch.plugins.Plugin; @@ -41,17 +42,7 @@ protected XPackLicenseState getLicenseState() { if (Build.CURRENT.isSnapshot() && property != null) { throw new IllegalArgumentException("es.vector_tile_feature_flag_registered is only supported in non-snapshot builds"); } - if ("true".equals(property)) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = true; - } else if ("false".equals(property)) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = false; - } else if (property == null) { - VECTOR_TILE_FEATURE_FLAG_REGISTERED = null; - } else { - throw new IllegalArgumentException( - "expected es.vector_tile_feature_flag_registered to be unset or [true|false] but was [" + property + "]" - ); - } + VECTOR_TILE_FEATURE_FLAG_REGISTERED = Booleans.parseBoolean(property, null); } public boolean isVectorTileEnabled() { diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java index 3fa37081a22bb..dcd69b723e142 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java @@ -27,7 +27,7 @@ public SimpleFeatureFactory(int z, int x, int y, int extent) { this.extent = extent; rectangle = FeatureFactoryUtils.getTileBounds(z, x, y); pointXScale = (double) extent / (rectangle.getMaxLon() - rectangle.getMinLon()); - pointYScale = -1d / ((rectangle.getMaxLat() - rectangle.getMinLat()) / (double) extent); + pointYScale = -(double) extent / (rectangle.getMaxLat() - rectangle.getMinLat()); } public void point(VectorTile.Tile.Feature.Builder featureBuilder, double lon, double lat) { From 8ae6cf44a097b85497bb11ecd10e91861f815a79 Mon Sep 17 00:00:00 2001 From: Igor Motov Date: Wed, 16 Jun 2021 15:27:36 -1000 Subject: [PATCH 15/15] Fix spotless issues --- .../org/elasticsearch/xpack/vectortile/VectorTileRestIT.java | 1 + .../elasticsearch/xpack/vectortile/feature/FeatureFactory.java | 1 + .../xpack/vectortile/feature/SimpleFeatureFactory.java | 1 + .../xpack/vectortile/rest/RestVectorTileAction.java | 1 + .../org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java | 1 + .../xpack/vectortile/feature/FeatureFactoryTests.java | 1 + .../xpack/vectortile/rest/VectorTileRequestTests.java | 1 + 7 files changed, 7 insertions(+) diff --git a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java index debb50c5045df..390f4dd81fa60 100644 --- a/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java +++ b/x-pack/plugin/vector-tile/src/javaRestTest/java/org/elasticsearch/xpack/vectortile/VectorTileRestIT.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.vectortile; import com.wdtinc.mapbox_vector_tile.VectorTile; + import org.apache.http.HttpStatus; import org.apache.http.client.methods.HttpDelete; import org.apache.http.client.methods.HttpGet; diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java index 03771650922c7..7f3d1884fe280 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactory.java @@ -14,6 +14,7 @@ import com.wdtinc.mapbox_vector_tile.adapt.jts.TileGeomResult; import com.wdtinc.mapbox_vector_tile.build.MvtLayerParams; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; + import org.elasticsearch.geometry.Circle; import org.elasticsearch.geometry.Geometry; import org.elasticsearch.geometry.GeometryCollection; diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java index dcd69b723e142..7a6a1cf4cc5b6 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/feature/SimpleFeatureFactory.java @@ -10,6 +10,7 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.encoding.GeomCmd; import com.wdtinc.mapbox_vector_tile.encoding.GeomCmdHdr; + import org.apache.lucene.util.BitUtil; import org.elasticsearch.geometry.Rectangle; diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java index d968a7f5133c9..66b13f4ac00eb 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/RestVectorTileAction.java @@ -9,6 +9,7 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.adapt.jts.IUserDataConverter; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; + import org.elasticsearch.action.search.SearchRequestBuilder; import org.elasticsearch.action.search.SearchResponse; import org.elasticsearch.action.search.SearchResponseSections; diff --git a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java index aa4015343fc37..09fd711942965 100644 --- a/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java +++ b/x-pack/plugin/vector-tile/src/main/java/org/elasticsearch/xpack/vectortile/rest/VectorTileUtils.java @@ -10,6 +10,7 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.build.MvtLayerProps; import com.wdtinc.mapbox_vector_tile.encoding.MvtValue; + import org.elasticsearch.common.util.Maps; import org.elasticsearch.common.xcontent.ToXContent; import org.elasticsearch.common.xcontent.XContentHelper; diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java index 7bef903810ddf..c486168c4b472 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/feature/FeatureFactoryTests.java @@ -9,6 +9,7 @@ import com.wdtinc.mapbox_vector_tile.VectorTile; import com.wdtinc.mapbox_vector_tile.adapt.jts.UserDataIgnoreConverter; + import org.apache.lucene.geo.GeoTestUtil; import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.Rectangle; diff --git a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java index 0f9905a2d3fc2..3a0b718716398 100644 --- a/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java +++ b/x-pack/plugin/vector-tile/src/test/java/org/elasticsearch/xpack/vectortile/rest/VectorTileRequestTests.java @@ -8,6 +8,7 @@ package org.elasticsearch.xpack.vectortile.rest; import com.carrotsearch.randomizedtesting.generators.RandomPicks; + import org.elasticsearch.common.bytes.BytesReference; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.common.xcontent.NamedXContentRegistry;