diff --git a/analytics/api/src/main/java/org/locationtech/geowave/analytic/kryo/GeometrySerializer.java b/analytics/api/src/main/java/org/locationtech/geowave/analytic/kryo/GeometrySerializer.java new file mode 100644 index 00000000000..2c6a7c452fd --- /dev/null +++ b/analytics/api/src/main/java/org/locationtech/geowave/analytic/kryo/GeometrySerializer.java @@ -0,0 +1,34 @@ +package org.locationtech.geowave.analytic.kryo; + +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.io.ParseException; +import org.locationtech.jts.io.WKBReader; +import org.locationtech.jts.io.WKBWriter; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import com.esotericsoftware.kryo.Kryo; +import com.esotericsoftware.kryo.Serializer; +import com.esotericsoftware.kryo.io.Input; +import com.esotericsoftware.kryo.io.Output; + +public class GeometrySerializer extends Serializer { + static final Logger LOGGER = LoggerFactory.getLogger(GeometrySerializer.class); + + @Override + public Geometry read(final Kryo arg0, final Input arg1, final Class arg2) { + final byte[] data = arg1.readBytes(arg1.readInt()); + try { + return new WKBReader().read(data); + } catch (final ParseException e) { + LOGGER.warn("Unable to deserialize geometry", e); + } + return null; + } + + @Override + public void write(final Kryo arg0, final Output arg1, final Geometry arg2) { + final byte[] data = new WKBWriter().write(arg2); + arg1.writeInt(data.length); + arg1.write(data); + } +} diff --git a/analytics/spark/src/main/java/org/locationtech/geowave/analytic/spark/GeoWaveRegistrator.java b/analytics/spark/src/main/java/org/locationtech/geowave/analytic/spark/GeoWaveRegistrator.java index 300f4f8a95c..388f6d47f0f 100644 --- a/analytics/spark/src/main/java/org/locationtech/geowave/analytic/spark/GeoWaveRegistrator.java +++ b/analytics/spark/src/main/java/org/locationtech/geowave/analytic/spark/GeoWaveRegistrator.java @@ -12,12 +12,19 @@ import org.geotools.feature.simple.SimpleFeatureImpl; import org.locationtech.geowave.adapter.raster.adapter.GridCoverageWritable; import org.locationtech.geowave.analytic.kryo.FeatureSerializer; +import org.locationtech.geowave.analytic.kryo.GeometrySerializer; import org.locationtech.geowave.analytic.kryo.GridCoverageWritableSerializer; import org.locationtech.geowave.analytic.kryo.PersistableSerializer; import org.locationtech.geowave.core.index.ByteArray; import org.locationtech.geowave.core.index.persist.PersistableFactory; import org.locationtech.geowave.mapreduce.input.GeoWaveInputKey; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryCollection; +import org.locationtech.jts.geom.MultiLineString; +import org.locationtech.jts.geom.MultiPoint; +import org.locationtech.jts.geom.MultiPolygon; +import org.locationtech.jts.geom.Point; +import org.locationtech.jts.geom.Polygon; import org.locationtech.jts.geom.prep.PreparedGeometry; import com.esotericsoftware.kryo.Kryo; @@ -29,13 +36,20 @@ public void registerClasses(final Kryo kryo) { final FeatureSerializer simpleFeatureSerializer = new FeatureSerializer(); final GridCoverageWritableSerializer gcwSerializer = new GridCoverageWritableSerializer(); final PersistableSerializer persistSerializer = new PersistableSerializer(); + final GeometrySerializer geometrySerializer = new GeometrySerializer(); PersistableFactory.getInstance().getClassIdMapping().entrySet().forEach( e -> kryo.register(e.getKey(), persistSerializer)); kryo.register(GeoWaveRDD.class); kryo.register(GeoWaveIndexedRDD.class); - kryo.register(Geometry.class); + kryo.register(Geometry.class, geometrySerializer); + kryo.register(Point.class, geometrySerializer); + kryo.register(MultiLineString.class, geometrySerializer); + kryo.register(Polygon.class, geometrySerializer); + kryo.register(MultiPolygon.class, geometrySerializer); + kryo.register(MultiPoint.class, geometrySerializer); + kryo.register(GeometryCollection.class, geometrySerializer); kryo.register(PreparedGeometry.class); kryo.register(ByteArray.class); kryo.register(GeoWaveInputKey.class); diff --git a/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/ExplainCommand.java b/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/ExplainCommand.java index 3b428890c56..5630961e809 100644 --- a/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/ExplainCommand.java +++ b/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/ExplainCommand.java @@ -170,9 +170,11 @@ public static StringBuilder explainMainParameter(final JCommander commander) { } final boolean assigned = mainParameter.isAssigned(); + System.out.println("ASSIGNED: " + assigned); builder.append("Specified: "); final List mP = (List) mainParameter.getParameterized().get(mainParameter.getObject()); + System.out.println("MP: " + mP); if (!assigned || (mP.size() == 0)) { builder.append(""); } else { diff --git a/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/GeoWaveTopLevelSection.java b/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/GeoWaveTopLevelSection.java index 7723c8e12e4..bd057d35ab5 100644 --- a/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/GeoWaveTopLevelSection.java +++ b/core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/GeoWaveTopLevelSection.java @@ -9,13 +9,11 @@ package org.locationtech.geowave.core.cli.operations; import org.apache.logging.log4j.core.appender.ConsoleAppender; -import org.apache.logging.log4j.core.config.Configuration; import org.apache.logging.log4j.core.config.Configurator; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.core.Logger; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.core.layout.PatternLayout; -import org.apache.logging.log4j.core.layout.PatternLayout.Builder; import org.locationtech.geowave.core.cli.VersionUtils; import org.locationtech.geowave.core.cli.annotations.GeowaveOperation; import org.locationtech.geowave.core.cli.api.DefaultOperation; diff --git a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/binning/SpatialBinningType.java b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/binning/SpatialBinningType.java index 1e90ef11f61..d169ba50143 100644 --- a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/binning/SpatialBinningType.java +++ b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/binning/SpatialBinningType.java @@ -8,29 +8,113 @@ */ package org.locationtech.geowave.core.geotime.binning; +import org.geotools.geometry.jts.JTS; +import org.geotools.referencing.CRS; import org.locationtech.geowave.core.index.ByteArray; import org.locationtech.geowave.core.store.api.BinConstraints.ByteArrayConstraints; import org.locationtech.jts.geom.Geometry; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.NoSuchAuthorityCodeException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.referencing.operation.TransformException; public enum SpatialBinningType implements SpatialBinningHelper { H3(new H3BinningHelper()), S2(new S2BinningHelper()), GEOHASH(new GeohashBinningHelper()); private SpatialBinningHelper helperDelegate; + public static int WGS84_SRID = 4326; + public static String WGS84_SRID_EPSG = "EPSG:4326"; + public static String WEB_MERCATOR = "EPSG:3857"; + private SpatialBinningType(final SpatialBinningHelper helperDelegate) { this.helperDelegate = helperDelegate; } + /** + * Converts a JTS geometry to WGS84 CRS. + * + * @param geometry {Geometry} The input geometry to be processed. + * @return {Geometry} Returns the JTS geometry in WGS84 CRS. + */ + public static Geometry convertToWGS84(Geometry geometry) { + + // Get the source CRS from the user data that is set in OptimizedSimpleFeatureBuilder.java + CoordinateReferenceSystem sourceCRS = (CoordinateReferenceSystem) geometry.getUserData(); + // System.out.println("SBT - SOURCE CRS is null? " + (sourceCRS == null)); + + // if (sourceCRS == null) { + // try { + // sourceCRS = CRS.decode(WEB_MERCATOR); + // } catch (NoSuchAuthorityCodeException e) { + // e.printStackTrace(); + // } catch (FactoryException e) { + // e.printStackTrace(); + // } + // } + + MathTransform transform; + Geometry targetGeometry = null; + + if (sourceCRS != null) { + // Only proceed if CRS is not WGS84 + Boolean isWGS84 = sourceCRS.getName().getCode().equals("WGS 84"); + + if (!isWGS84) { + try { + // Decode the target CRS of "EPSG:4326" + CoordinateReferenceSystem targetCRS = CRS.decode(WGS84_SRID_EPSG); + + // Get the transform from source CRS to target CRS with leniency + transform = CRS.findMathTransform(sourceCRS, targetCRS, true); + try { + // Transform the JTS geometry + targetGeometry = JTS.transform(geometry, transform); + + // Set the SRID, although this is not necessary + targetGeometry.setSRID(WGS84_SRID); + } catch (MismatchedDimensionException | TransformException e) { + e.printStackTrace(); + } + } catch (FactoryException e) { + e.printStackTrace(); + } + } + } + + return targetGeometry != null ? targetGeometry : geometry; + } + + /** + * Gets the spatial bins. Note: Spatial binning aggregations call this (runs on each individual + * SimpleFeature). + * + * @param geometry {Geometry} The input geometry to be processed. + * @param precision {Integer} The spatial binning precision. + * @return {ByteArray[]} Returns a ByteArray of spatial bins. + */ @Override public ByteArray[] getSpatialBins(final Geometry geometry, final int precision) { - // TODO if geometry is not WGS84 we need to transform it - return helperDelegate.getSpatialBins(geometry, precision); + Geometry targetGeometry = convertToWGS84(geometry); + + return helperDelegate.getSpatialBins(targetGeometry, precision); } + /** + * Gets the geometry constraints. Note: Spatial binning statistics call this - runs once on whole + * extent. + * + * @param geom {Geometry} The input geometry to be processed. + * @param precision {Integer} The spatial binning precision. + * @return {ByteArrayConstraints} Returns a ByteArrayConstraints of geometry constraints. + */ @Override public ByteArrayConstraints getGeometryConstraints(final Geometry geom, final int precision) { - // TODO if geometry is not WGS84 we need to transform it - return helperDelegate.getGeometryConstraints(geom, precision); + Geometry targetGeometry = convertToWGS84(geom); + + return helperDelegate.getGeometryConstraints(targetGeometry, precision); } diff --git a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/query/aggregate/SpatialBinningStrategy.java b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/query/aggregate/SpatialBinningStrategy.java index 9024624a2e8..8d4487afe50 100644 --- a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/query/aggregate/SpatialBinningStrategy.java +++ b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/query/aggregate/SpatialBinningStrategy.java @@ -101,6 +101,7 @@ public ByteArray[] getBins( } if (ComplexGeometryBinningOption.USE_CENTROID_ONLY.equals(complexGeometryBinning)) { final Point centroid = geometry.getCentroid(); + centroid.setUserData(geometry.getUserData()); return type.getSpatialBins(centroid, precision); } return type.getSpatialBins(geometry, precision); diff --git a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/statistics/binning/SpatialFieldValueBinningStrategy.java b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/statistics/binning/SpatialFieldValueBinningStrategy.java index b28511ee43f..b7284ca6c84 100644 --- a/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/statistics/binning/SpatialFieldValueBinningStrategy.java +++ b/core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/statistics/binning/SpatialFieldValueBinningStrategy.java @@ -23,6 +23,8 @@ import org.locationtech.geowave.core.store.statistics.binning.FieldValueBinningStrategy; import org.locationtech.jts.geom.Envelope; import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.Point; +import org.opengis.referencing.crs.CoordinateReferenceSystem; import com.beust.jcommander.IStringConverter; import com.beust.jcommander.Parameter; import com.beust.jcommander.ParameterException; @@ -128,7 +130,12 @@ public Class[] supportedConstraintClasses() { private ByteArray[] getSpatialBinsFromObj(final Object value) { if (value instanceof Geometry) { if (ComplexGeometryBinningOption.USE_CENTROID_ONLY.equals(complexGeometry)) { - return getSpatialBins(((Geometry) value).getCentroid()); + + Point centroid = ((Geometry) value).getCentroid(); + centroid.setUserData(((Geometry) value).getUserData()); + + return getSpatialBins(centroid); + // return getSpatialBins(((Geometry) value).getCentroid()); } return getSpatialBins((Geometry) value); } diff --git a/core/store/src/main/java/org/locationtech/geowave/core/store/cli/stats/StatsCommandLineOptions.java b/core/store/src/main/java/org/locationtech/geowave/core/store/cli/stats/StatsCommandLineOptions.java index f2b8bd90732..1d7f2235a84 100644 --- a/core/store/src/main/java/org/locationtech/geowave/core/store/cli/stats/StatsCommandLineOptions.java +++ b/core/store/src/main/java/org/locationtech/geowave/core/store/cli/stats/StatsCommandLineOptions.java @@ -141,7 +141,7 @@ public List>> resolveMatchingStatistics( } final DataTypeAdapter adapter = dataStore.getType(typeName); if (adapter == null) { - throw new ParameterException("Unable to find an type named: " + typeName); + throw new ParameterException("Unable to find a type named: " + typeName); } try (CloseableIterator>> stats = statsStore.getDataTypeStatistics(adapter, statisticType, tag)) { @@ -154,7 +154,7 @@ public List>> resolveMatchingStatistics( } final DataTypeAdapter adapter = dataStore.getType(typeName); if (adapter == null) { - throw new ParameterException("Unable to find an type named: " + typeName); + throw new ParameterException("Unable to find a type named: " + typeName); } if (fieldName == null) { throw new ParameterException( diff --git a/examples/java-api/src/main/java/org/locationtech/geowave/examples/ingest/SimpleIngest.java b/examples/java-api/src/main/java/org/locationtech/geowave/examples/ingest/SimpleIngest.java index fe8b3ce9f19..a1767b26cc8 100644 --- a/examples/java-api/src/main/java/org/locationtech/geowave/examples/ingest/SimpleIngest.java +++ b/examples/java-api/src/main/java/org/locationtech/geowave/examples/ingest/SimpleIngest.java @@ -40,7 +40,6 @@ public static void main(final String[] args) { DataStoreFactory.createDataStore(new MemoryRequiredOptions()); si.writeExampleData(geowaveDataStore); - System.out.println("Finished ingesting data"); } /** * Here we will change the ingest mechanism to use a producer/consumer pattern */ @@ -65,7 +64,7 @@ protected void writeExampleData(final DataStore geowaveDataStore) { try (Writer indexWriter = geowaveDataStore.createWriter(dataTypeAdapter.getTypeName())) { // build a grid of points across the globe at each whole - // lattitude/longitude intersection + // latitude/longitude intersection for (final SimpleFeature sft : getGriddedFeatures(pointBuilder, 1000)) { indexWriter.write(sft); @@ -204,6 +203,9 @@ public static SimpleFeatureType createPointFeatureType() { builder.add(ab.binding(String.class).nillable(true).buildDescriptor("TrajectoryID")); builder.add(ab.binding(String.class).nillable(true).buildDescriptor("Comment")); + // Create a SIZE field for sum aggregation and statistics tests + builder.add(ab.binding(Double.class).nillable(true).buildDescriptor("SIZE")); + return builder.buildFeatureType(); } } diff --git a/extensions/adapters/vector/src/main/java/org/geotools/feature/simple/OptimizedSimpleFeatureBuilder.java b/extensions/adapters/vector/src/main/java/org/geotools/feature/simple/OptimizedSimpleFeatureBuilder.java index 43938b90791..939e9276a5a 100644 --- a/extensions/adapters/vector/src/main/java/org/geotools/feature/simple/OptimizedSimpleFeatureBuilder.java +++ b/extensions/adapters/vector/src/main/java/org/geotools/feature/simple/OptimizedSimpleFeatureBuilder.java @@ -8,7 +8,9 @@ */ package org.geotools.feature.simple; +import org.locationtech.jts.geom.Geometry; import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.referencing.crs.CoordinateReferenceSystem; /** * Variation of SimpleFeatureBuilder that skips object conversion, since GeoWave handles that @@ -26,6 +28,12 @@ public void set(int index, Object value) { throw new ArrayIndexOutOfBoundsException( "Can handle " + values.length + " attributes only, index is " + index); + // Add the CRS of the geometry to the user data + if (value instanceof Geometry) { + ((Geometry) value).setUserData(getFeatureType().getCoordinateReferenceSystem()); + } + values[index] = value; + } } diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/FeatureDataAdapter.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/FeatureDataAdapter.java index 05b8b602cfd..558ddc02c0c 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/FeatureDataAdapter.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/FeatureDataAdapter.java @@ -325,6 +325,14 @@ public SimpleFeature fromWritable(final FeatureWritable writable) { @Override public Object getFieldValue(final SimpleFeature entry, final String fieldName) { + + // Object fieldValue = entry.getAttribute(fieldName); + // if ((fieldValue instanceof Geometry) + // && !(((Geometry) fieldValue).getUserData() instanceof CoordinateReferenceSystem)) { + // ((Geometry) fieldValue).setUserData(getFeatureType().getCoordinateReferenceSystem()); + // } + + return entry.getAttribute(fieldName); } diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureCollection.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureCollection.java index 2c5dcb6aea2..e2fa5dd100b 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureCollection.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureCollection.java @@ -36,15 +36,22 @@ import org.opengis.filter.Filter; import org.opengis.geometry.BoundingBox; import org.opengis.referencing.FactoryException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +// import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.operation.TransformException; import org.slf4j.Logger; import org.slf4j.LoggerFactory; +import org.locationtech.geowave.adapter.vector.plugin.heatmap.HeatMapUtils; /** * This class is a helper for the GeoWave GeoTools data store. It represents a collection of feature * data by encapsulating a GeoWave reader and a query object in order to open the appropriate cursor * to iterate over data. It uses Keys within the Query hints to determine whether to perform special - * purpose queries such as decimation or distributed rendering. + * purpose queries such as decimation, distributed rendering, subsampling, and heatmap processes. + * + * @apiNote Changelog:
3-25-2022 M. Zagorski: Added code for custom HeatMapProcess using + * spatial binning.
+ * */ public class GeoWaveFeatureCollection extends DataFeatureCollection { private static final Logger LOGGER = LoggerFactory.getLogger(GeoWaveFeatureCollection.class); @@ -52,6 +59,7 @@ public class GeoWaveFeatureCollection extends DataFeatureCollection { private CloseableIterator featureCursor; private final Query query; private static SimpleFeatureType distributedRenderFeatureType; + private static SimpleFeatureType heatmapFeatureType; public GeoWaveFeatureCollection(final GeoWaveFeatureReader reader, final Query query) { this.reader = reader; @@ -61,6 +69,7 @@ public GeoWaveFeatureCollection(final GeoWaveFeatureReader reader, final Query q @Override public int getCount() { + if (query.getFilter().equals(Filter.INCLUDE)) { // GEOWAVE-60 optimization final CountValue count = @@ -94,6 +103,7 @@ public ReferencedEnvelope getBounds() { double minx = Double.MAX_VALUE, maxx = -Double.MAX_VALUE, miny = Double.MAX_VALUE, maxy = -Double.MAX_VALUE; + try { // GEOWAVE-60 optimization final BoundingBoxValue boundingBox = @@ -121,6 +131,7 @@ public ReferencedEnvelope getBounds() { maxy = Math.max(bbox.getMaxY(), maxy); } close(iterator); + } catch (final Exception e) { LOGGER.warn("Error calculating bounds", e); return new ReferencedEnvelope(-180, 180, -90, 90, GeometryUtils.getDefaultCRS()); @@ -132,10 +143,23 @@ public ReferencedEnvelope getBounds() { public SimpleFeatureType getSchema() { if (isDistributedRenderQuery()) { return getDistributedRenderFeatureType(); + } else if (isHeatmapQuery()) { + return getHeatmapFeatureType(); } return reader.getFeatureType(); } + public static SimpleFeatureType getSchema( + final Query query, + final SimpleFeatureType defaultType) { + if (isDistributedRenderQuery(query)) { + return getDistributedRenderFeatureType(); + } else if (isHeatmapQuery(query)) { + return getHeatmapFeatureType(); + } + return defaultType; + } + public static synchronized SimpleFeatureType getDistributedRenderFeatureType() { if (distributedRenderFeatureType == null) { distributedRenderFeatureType = createDistributedRenderFeatureType(); @@ -143,6 +167,13 @@ public static synchronized SimpleFeatureType getDistributedRenderFeatureType() { return distributedRenderFeatureType; } + public static synchronized SimpleFeatureType getHeatmapFeatureType() { + if (heatmapFeatureType == null) { + heatmapFeatureType = HeatMapUtils.createHeatmapFeatureType(); + } + return heatmapFeatureType; + } + private static SimpleFeatureType createDistributedRenderFeatureType() { final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder(); typeBuilder.setName("distributed_render"); @@ -151,6 +182,23 @@ private static SimpleFeatureType createDistributedRenderFeatureType() { return typeBuilder.buildFeatureType(); } + protected boolean isHeatmapQuery() { + return GeoWaveFeatureCollection.isHeatmapQuery(query); + } + + protected static final boolean isHeatmapQuery(final Query query) { + + final Object heatmapEnabled = query.getHints().get(GeoWaveHeatMapProcess.HEATMAP_ENABLED); + final Object useBinning = query.getHints().get(GeoWaveHeatMapProcess.USE_BINNING); + if ((heatmapEnabled instanceof Boolean) + && (Boolean) heatmapEnabled + && (useBinning instanceof Boolean) + && (Boolean) useBinning) { + return true; + } + return false; + } + protected boolean isDistributedRenderQuery() { return GeoWaveFeatureCollection.isDistributedRenderQuery(query); } @@ -160,18 +208,24 @@ protected static final boolean isDistributedRenderQuery(final Query query) { } private static SimpleFeatureType getSchema(final GeoWaveFeatureReader reader, final Query query) { + if (GeoWaveFeatureCollection.isDistributedRenderQuery(query)) { return getDistributedRenderFeatureType(); } + if (GeoWaveFeatureCollection.isHeatmapQuery(query)) { + return getHeatmapFeatureType(); + } return reader.getComponents().getFeatureType(); } protected QueryConstraints getQueryConstraints() throws TransformException, FactoryException { + final ReferencedEnvelope referencedEnvelope = getEnvelope(query); final Geometry jtsBounds; final TemporalConstraintsSet timeBounds; - if (reader.getGeoWaveFilter() == null - || query.getHints().containsKey(SubsampleProcess.SUBSAMPLE_ENABLED)) { + if ((reader.getGeoWaveFilter() == null) + || query.getHints().containsKey(SubsampleProcess.SUBSAMPLE_ENABLED) + || query.getHints().containsKey(GeoWaveHeatMapProcess.HEATMAP_ENABLED)) { jtsBounds = getBBox(query, referencedEnvelope); timeBounds = getBoundedTime(query); } else { @@ -182,13 +236,14 @@ protected QueryConstraints getQueryConstraints() throws TransformException, Fact Integer limit = getLimit(query); final Integer startIndex = getStartIndex(query); - // limit becomes a 'soft' constraint since GeoServer will inforce + // limit becomes a 'soft' constraint since GeoServer will enforce // the limit final Long max = (limit != null) ? limit.longValue() + (startIndex == null ? 0 : startIndex.longValue()) : null; // limit only used if less than an integer max value. limit = ((max != null) && (max.longValue() < Integer.MAX_VALUE)) ? max.intValue() : null; + return new QueryConstraints(jtsBounds, timeBounds, referencedEnvelope, limit); } @@ -205,7 +260,7 @@ protected Iterator openIterator() { private Iterator openIterator(final QueryConstraints constraints) { - if (reader.getGeoWaveFilter() == null + if ((reader.getGeoWaveFilter() == null) && (((constraints.jtsBounds != null) && constraints.jtsBounds.isEmpty()) || ((constraints.timeBounds != null) && constraints.timeBounds.isEmpty()))) { // return nothing if either constraint is empty @@ -236,6 +291,24 @@ private Iterator openIterator(final QueryConstraints constraints) constraints.referencedEnvelope, constraints.limit); + } else if (query.getHints().containsKey(GeoWaveHeatMapProcess.OUTPUT_WIDTH) + && query.getHints().containsKey(GeoWaveHeatMapProcess.OUTPUT_HEIGHT) + && query.getHints().containsKey(GeoWaveHeatMapProcess.OUTPUT_BBOX) + && query.getHints().containsKey(GeoWaveHeatMapProcess.USE_BINNING) + && ((Boolean) query.getHints().get(GeoWaveHeatMapProcess.USE_BINNING) == true)) { + + // GeoWave Heatmap Process + featureCursor = + new CloseableIterator.Wrapper<>( + DataUtilities.iterator( + reader.getDataHeatMap( + constraints.jtsBounds, + constraints.timeBounds, + (ReferencedEnvelope) query.getHints().get(GeoWaveHeatMapProcess.OUTPUT_BBOX), + (Integer) query.getHints().get(GeoWaveHeatMapProcess.OUTPUT_WIDTH), + (Integer) query.getHints().get(GeoWaveHeatMapProcess.OUTPUT_HEIGHT), + constraints.limit))); // TODO: is limit needed? + } else { featureCursor = reader.getData(constraints.jtsBounds, constraints.timeBounds, constraints.limit); @@ -245,15 +318,46 @@ private Iterator openIterator(final QueryConstraints constraints) private ReferencedEnvelope getEnvelope(final Query query) throws TransformException, FactoryException { + if (query.getHints().containsKey(SubsampleProcess.OUTPUT_BBOX)) { return ((ReferencedEnvelope) query.getHints().get(SubsampleProcess.OUTPUT_BBOX)).transform( reader.getFeatureType().getCoordinateReferenceSystem(), true); } + + // Return the heatmap referenced envelope + if (query.getHints().containsKey(GeoWaveHeatMapProcess.OUTPUT_BBOX)) { + + final ReferencedEnvelope bbox = + (ReferencedEnvelope) query.getHints().get(GeoWaveHeatMapProcess.OUTPUT_BBOX); + final CoordinateReferenceSystem bboxCRS = bbox.getCoordinateReferenceSystem(); + System.out.println("COLLECTION - BBOX CRS: " + bboxCRS.getName()); + + final CoordinateReferenceSystem featureCRS = + reader.getFeatureType().getCoordinateReferenceSystem(); + System.out.println("COLLECTION - FEATURE CRS: " + featureCRS.getName()); + + // Find out if the CRS is WGS84 + final Boolean isWGS84 = featureCRS.getName().getCode().equals("WGS 84"); + System.out.println("COLLECTION - isWGS84? " + isWGS84); + + return ((ReferencedEnvelope) query.getHints().get( + GeoWaveHeatMapProcess.OUTPUT_BBOX)).transform( + reader.getFeatureType().getCoordinateReferenceSystem(), + true); + + // TODO: Does jtsBounds need to have the same CRS as the output bbox? + // return ((ReferencedEnvelope) query.getHints().get( + // GeoWaveHeatMapProcess.OUTPUT_BBOX)).transform( + // bbox.getCoordinateReferenceSystem(), + // true); + } + return null; } private Geometry getBBox(final Query query, final ReferencedEnvelope envelope) { + if (envelope != null) { return new GeometryFactory().toGeometry(envelope); } @@ -272,14 +376,17 @@ private Geometry getBBox(final Query query, final ReferencedEnvelope envelope) { } private Query validateQuery(final String typeName, final Query query) { + return query == null ? new Query(typeName, Filter.EXCLUDE) : query; } private Integer getStartIndex(final Query query) { + return query.getStartIndex(); } private Integer getLimit(final Query query) { + if (!query.isMaxFeaturesUnlimited() && (query.getMaxFeatures() >= 0)) { return query.getMaxFeatures(); } @@ -290,6 +397,7 @@ private Integer getLimit(final Query query) { public void accepts( final org.opengis.feature.FeatureVisitor visitor, final org.opengis.util.ProgressListener progress) throws IOException { + if (!GeoWaveGTPluginUtils.accepts( reader.getComponents().getStatsStore(), reader.getComponents().getAdapter(), @@ -305,6 +413,7 @@ public void accepts( * @return the temporal constraints of the query */ protected TemporalConstraintsSet getBoundedTime(final Query query) { + if (query == null) { return null; } @@ -316,26 +425,31 @@ protected TemporalConstraintsSet getBoundedTime(final Query query) { @Override public FeatureReader reader() { + return reader; } @Override protected void closeIterator(final Iterator close) { + featureCursor.close(); } public Iterator getOpenIterator() { + return featureCursor; } @Override public void close(final FeatureIterator iterator) { + featureCursor = null; super.close(iterator); } @Override public boolean isEmpty() { + try { return !reader.hasNext(); } catch (final IOException e) { @@ -362,4 +476,5 @@ public QueryConstraints( this.limit = limit; } } + } diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureReader.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureReader.java index bb693a7e0d3..a8d6447829a 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureReader.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureReader.java @@ -20,6 +20,9 @@ import java.util.Set; import org.geotools.data.FeatureReader; import org.geotools.data.Query; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.geotools.data.simple.SimpleFeatureIterator; +import org.geotools.feature.FeatureIterator; import org.geotools.feature.simple.SimpleFeatureBuilder; import org.geotools.filter.AttributeExpressionImpl; import org.geotools.filter.FidFilterImpl; @@ -28,6 +31,9 @@ import org.geotools.geometry.jts.ReferencedEnvelope; import org.geotools.referencing.operation.transform.ProjectiveTransform; import org.geotools.renderer.lite.RendererUtilities; +import org.locationtech.geowave.adapter.vector.plugin.heatmap.HeatMapAggregations; +import org.locationtech.geowave.adapter.vector.plugin.heatmap.HeatMapStatistics; +import org.locationtech.geowave.adapter.vector.plugin.heatmap.HeatMapUtils; import org.locationtech.geowave.adapter.vector.plugin.transaction.GeoWaveTransaction; import org.locationtech.geowave.adapter.vector.plugin.transaction.StatisticsCache; import org.locationtech.geowave.adapter.vector.render.DistributedRenderAggregation; @@ -69,6 +75,7 @@ import org.opengis.filter.expression.Expression; import org.opengis.filter.expression.PropertyName; import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; import org.opengis.referencing.operation.MathTransform2D; import org.opengis.referencing.operation.TransformException; import org.slf4j.Logger; @@ -77,9 +84,14 @@ import com.google.common.collect.Sets; /** - * This class wraps a geotools data store as well as one for statistics (for example to display - * Heatmaps) into a GeoTools FeatureReader for simple feature data. It acts as a helper for - * GeoWave's GeoTools data store. + * This class wraps a geotools data store as well as one for statistics (e.g. to display Heatmaps) + * into a GeoTools FeatureReader for simple feature data. It acts as a helper for GeoWave's GeoTools + * data store. + * + * + * @apiNote Changelog:
3-25-2022 M. Zagorski: Added code for custom HeatMapProcess using + * spatial binning.
+ * */ public class GeoWaveFeatureReader implements FeatureReader { private static final Logger LOGGER = LoggerFactory.getLogger(GeoWaveFeatureReader.class); @@ -99,10 +111,10 @@ public GeoWaveFeatureReader( this.transaction = transaction; featureCollection = new GeoWaveFeatureCollection(this, query); this.query = query; - this.filter = getFilter(query); + filter = getFilter(query); Object gwfilter = null; try { - gwfilter = this.filter.accept(new CQLToGeoWaveFilterVisitor(components.getAdapter()), null); + gwfilter = filter.accept(new CQLToGeoWaveFilterVisitor(components.getAdapter()), null); } catch (CQLToGeoWaveConversionException | InvalidFilterException e) { // Incompatible with GeoWave filter expressions, fall back to regular optimal CQL query } @@ -110,19 +122,23 @@ public GeoWaveFeatureReader( } public GeoWaveTransaction getTransaction() { + return transaction; } public GeoWaveDataStoreComponents getComponents() { + return components; } public org.locationtech.geowave.core.store.query.filter.expression.Filter getGeoWaveFilter() { + return (org.locationtech.geowave.core.store.query.filter.expression.Filter) geoWaveFilter; } @Override public void close() throws IOException { + if (featureCollection.getOpenIterator() != null) { featureCollection.closeIterator(featureCollection.getOpenIterator()); } @@ -130,11 +146,13 @@ public void close() throws IOException { @Override public SimpleFeatureType getFeatureType() { + return components.getFeatureType(); } @Override public boolean hasNext() throws IOException { + Iterator it = featureCollection.getOpenIterator(); if (it != null) { // protect againt GeoTools forgetting to call close() @@ -150,6 +168,7 @@ public boolean hasNext() throws IOException { @Override public SimpleFeature next() throws IOException, IllegalArgumentException, NoSuchElementException { + Iterator it = featureCollection.getOpenIterator(); if (it != null) { return it.next(); @@ -159,10 +178,12 @@ public SimpleFeature next() throws IOException, IllegalArgumentException, NoSuch } public CloseableIterator getNoData() { + return new CloseableIterator.Empty<>(); } public long getCount() { + return featureCollection.getCount(); } @@ -170,6 +191,7 @@ protected long getCountInternal( final Geometry jtsBounds, final TemporalConstraintsSet timeBounds, final Integer limit) { + final CountQueryIssuer countIssuer = new CountQueryIssuer(limit); issueQuery(jtsBounds, timeBounds, countIssuer); return countIssuer.count; @@ -178,6 +200,7 @@ protected long getCountInternal( private BasicQueryByClass getQuery( final Geometry jtsBounds, final TemporalConstraintsSet timeBounds) { + final GeoConstraintsWrapper geoConstraints = QueryIndexHelper.composeGeometricConstraints(getFeatureType(), jtsBounds); @@ -207,20 +230,59 @@ private BasicQueryByClass getQuery( query.setExact(timeBounds.isExact()); return query; } + } + + + /** + * Issues the heatmap query. + * + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param issuer {QueryIssuerHeatMap} The issuer that issues the query. + * @return {FeatureIterator} Returns a FeatureIterator for SimpleFeatures. + */ + public FeatureIterator issueQueryHeatmap( + final Geometry jtsBounds, + final QueryIssuerHeatMap issuer) { + + // Set defaults (to be overridden by user preferences) + String queryType = GeoWaveHeatMapProcess.CNT_AGGR; + String weightAttr = "count"; // TODO: what should this be set to? + int pixelsPerCell = 1; + Boolean createStats = false; + + if (query.getHints().containsKey(GeoWaveHeatMapProcess.HEATMAP_ENABLED) + && (Boolean) query.getHints().get(GeoWaveHeatMapProcess.HEATMAP_ENABLED)) { + + // Get user specified parameters + queryType = (String) query.getHints().get(GeoWaveHeatMapProcess.QUERY_TYPE); + weightAttr = (String) query.getHints().get(GeoWaveHeatMapProcess.WEIGHT_ATTR); + pixelsPerCell = (Integer) query.getHints().get(GeoWaveHeatMapProcess.PIXELS_PER_CELL); + createStats = (Boolean) query.getHints().get(GeoWaveHeatMapProcess.CREATE_STATS); + } + return issuer.query(queryType, weightAttr, pixelsPerCell, createStats); } + public CloseableIterator issueQuery( final Geometry jtsBounds, final TemporalConstraintsSet timeBounds, final QueryIssuer issuer) { final List> results = new ArrayList<>(); boolean spatialOnly = false; - if (this.query.getHints().containsKey(SubsampleProcess.SUBSAMPLE_ENABLED) - && (Boolean) this.query.getHints().get(SubsampleProcess.SUBSAMPLE_ENABLED)) { + if (query.getHints().containsKey(SubsampleProcess.SUBSAMPLE_ENABLED) + && (Boolean) query.getHints().get(SubsampleProcess.SUBSAMPLE_ENABLED)) { spatialOnly = true; } - if (!spatialOnly && getGeoWaveFilter() != null) { + + // If heatmap process is enabled, set spatialOnly to true + if (query.getHints().containsKey(GeoWaveHeatMapProcess.HEATMAP_ENABLED) + && (Boolean) query.getHints().get(GeoWaveHeatMapProcess.HEATMAP_ENABLED)) { + spatialOnly = true; + } + + if (!spatialOnly && (getGeoWaveFilter() != null)) { results.add(issuer.query(null, null, spatialOnly)); } else { final BasicQueryByClass query = getQuery(jtsBounds, timeBounds); @@ -257,6 +319,7 @@ public void close() throws IOException { } protected static boolean hasTime(final Index index) { + if ((index == null) || (index.getIndexStrategy() == null) || (index.getIndexStrategy().getOrderedDimensionDefinitions() == null)) { @@ -274,6 +337,7 @@ private QueryConstraints createQueryConstraints( final Index index, final BasicQueryByClass baseQuery, final boolean spatialOnly) { + if (getGeoWaveFilter() != null) { return new OptimalExpressionQuery( getGeoWaveFilter(), @@ -293,6 +357,7 @@ private QueryConstraints createQueryConstraints( } public Filter getFilter(final Query query) { + final Filter filter = query.getFilter(); if (filter instanceof BBOXImpl) { final BBOXImpl bbox = ((BBOXImpl) filter); @@ -317,6 +382,7 @@ public BaseIssuer(final Integer limit) { super(); this.limit = limit; + } @Override @@ -324,6 +390,7 @@ public CloseableIterator query( final Index index, final BasicQueryByClass query, final boolean spatialOnly) { + VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder().addTypeName( components.getAdapter().getTypeName()).setAuthorizations( @@ -343,11 +410,13 @@ public CloseableIterator query( @Override public Filter getFilter() { + return filter; } @Override public Integer getLimit() { + return limit; } } @@ -357,13 +426,17 @@ private class CountQueryIssuer extends BaseIssuer implements QueryIssuer { public CountQueryIssuer(final Integer limit) { super(limit); + } + @SuppressWarnings("unchecked") @Override public CloseableIterator query( final Index index, final BasicQueryByClass query, final boolean spatialOnly) { + + @SuppressWarnings("rawtypes") VectorAggregationQueryBuilder bldr = (VectorAggregationQueryBuilder) VectorAggregationQueryBuilder.newBuilder().count( components.getAdapter().getTypeName()).setAuthorizations( @@ -400,6 +473,7 @@ public EnvelopeQueryIssuer( this.height = height; this.pixelSize = pixelSize; this.envelope = envelope; + } @Override @@ -407,6 +481,7 @@ public CloseableIterator query( final Index index, final BasicQueryByClass query, final boolean spatialOnly) { + VectorQueryBuilder bldr = VectorQueryBuilder.newBuilder().addTypeName( components.getAdapter().getTypeName()).setAuthorizations( @@ -471,19 +546,151 @@ public CloseableIterator query( } } + + /** + * Private class that starts the heatmap query issuer. + * + * @author M. Zagorski + * + */ + private class HeatMapQueryIssuer extends BaseIssuer implements QueryIssuerHeatMap { + private QueryConstraints queryConstraints; + final Geometry jtsBounds; + final ReferencedEnvelope outputBbox; + final int width; + final int height; + + public HeatMapQueryIssuer( + final QueryConstraints queryConstraints, + final Geometry jtsBounds, + final ReferencedEnvelope outputBbox, + final int width, + final int height, + final Integer limit) { + super(limit); + this.queryConstraints = queryConstraints; + this.jtsBounds = jtsBounds; + this.outputBbox = outputBbox; + this.width = width; + this.height = height; + } + + @Override + public FeatureIterator query( + final String queryType, + final String weightAttr, + final Integer pixelsPerCell, + final Boolean createStats) { + + SimpleFeatureCollection newFeatures = null; + + // double bboxArea = outputBbox.getArea(); + // double jtsBoundsArea = jtsBounds.getArea(); + + // Get CRS if it exists + final CoordinateReferenceSystem sourceCRS = outputBbox.getCoordinateReferenceSystem(); + + // Add the source CRS to the user data for the jtsBounds + jtsBounds.setUserData(sourceCRS); + + // Test the Geohash precision determined by the comparative method TODO: does something weird + // with binning values + // int geohashPrec = HeatMapUtils.getGeohashPrecisionComp(width, jtsBounds, pixelsPerCell); + + // Get an appropriate Geohash precision for the GeoServer extent + final int geohashPrec = + HeatMapUtils.autoSelectGeohashPrecision( + height, + width, + pixelsPerCell, + jtsBounds, + sourceCRS); + + // TODO: implement tiling + // Temporary histogram builder + // TDigestNumericHistogram histogram = new TDigestNumericHistogram(); + // Create a method that utilizes histogram.add(cell values); + + // Build the count aggregation query and get the resulting SimpleFeatureCollection + if (queryType.equals(GeoWaveHeatMapProcess.CNT_AGGR)) { + newFeatures = + HeatMapAggregations.buildCountAggrQuery( + // histogram, + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr); + } + + // Build the sum aggregation query and get the resulting SimpleFeatureCollection + if (queryType.equals(GeoWaveHeatMapProcess.SUM_AGGR)) { + newFeatures = + HeatMapAggregations.buildFieldSumAggrQuery( + // histogram, + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr); + } + + // Build the count statistics query and get the resulting SimpleFeatureCollection + if (queryType.equals(GeoWaveHeatMapProcess.CNT_STATS)) { + newFeatures = + HeatMapStatistics.buildCountStatsQuery( + // histogram, + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr, + createStats); + } + + // Build the sum statistics query and get the resulting SimpleFeatureCollection + if (queryType.equals(GeoWaveHeatMapProcess.SUM_STATS)) { + newFeatures = + HeatMapStatistics.buildFieldStatsQuery( + // histogram, + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr, + createStats); + } + + if (newFeatures == null) { + LOGGER.warn( + "YOU MUST SPECIFICY A QUERY TYPE: CNT_AGGR, SUM_AGGR, CNT_STATS, or SUM_STATS."); + return null; + } + + final SimpleFeatureIterator simpFeatIter = newFeatures.features(); + return simpFeatIter; + + } + } + + private class RenderQueryIssuer extends BaseIssuer implements QueryIssuer { final DistributedRenderOptions renderOptions; public RenderQueryIssuer(final Integer limit, final DistributedRenderOptions renderOptions) { super(limit); this.renderOptions = renderOptions; + } + @SuppressWarnings("unchecked") @Override public CloseableIterator query( final Index index, final BasicQueryByClass query, final boolean spatialOnly) { + + @SuppressWarnings("rawtypes") final VectorAggregationQueryBuilder bldr = (VectorAggregationQueryBuilder) VectorAggregationQueryBuilder.newBuilder().setAuthorizations( transaction.composeAuthorizations()); @@ -509,9 +716,12 @@ public CloseableIterator renderData( final TemporalConstraintsSet timeBounds, final Integer limit, final DistributedRenderOptions renderOptions) { + return issueQuery(jtsBounds, timeBounds, new RenderQueryIssuer(limit, renderOptions)); } + + // Customizable way to get data as an iterator public CloseableIterator getData( final Geometry jtsBounds, final TemporalConstraintsSet timeBounds, @@ -520,16 +730,48 @@ public CloseableIterator getData( final double pixelSize, final ReferencedEnvelope envelope, final Integer limit) { + return issueQuery( jtsBounds, timeBounds, new EnvelopeQueryIssuer(width, height, pixelSize, limit, envelope)); } + /** + * Get data for heatmap query issuers. + * + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param outputBbox {ReferencedEnvelope} The bounding box of the dataset. + * @param width {Integer} The width of the bounding box. + * @param height {Integer} The height of the bounding box. + * @param limit {Integer} A constraints limit. //TODO: is this needed? + * @return {FeatureIterator} Returns a FeatureIterator for SimpleFeatures. + */ + public FeatureIterator getDataHeatMap( + final Geometry jtsBounds, + final TemporalConstraintsSet timeBounds, + final ReferencedEnvelope outputBbox, + final int width, + final int height, + final Integer limit) { + + return issueQueryHeatmap( + jtsBounds, + new HeatMapQueryIssuer( + getQuery(jtsBounds, timeBounds), + jtsBounds, + outputBbox, + width, + height, + limit)); + } + public CloseableIterator getData( final Geometry jtsBounds, final TemporalConstraintsSet timeBounds, final Integer limit) { + if (filter instanceof FidFilterImpl) { final Set fids = ((FidFilterImpl) filter).getFidsSet(); final byte[][] ids = new byte[fids.size()][]; @@ -559,6 +801,7 @@ public CloseableIterator getData( } public GeoWaveFeatureCollection getFeatureCollection() { + return featureCollection; } @@ -566,11 +809,13 @@ private CloseableIterator interweaveTransaction( final Integer limit, final Filter filter, final CloseableIterator it) { + return transaction.interweaveTransaction(limit, filter, it); } protected TemporalConstraintsSet clipIndexedTemporalConstraints( final TemporalConstraintsSet constraintsSet) { + return QueryIndexHelper.clipIndexedTemporalConstraints( transaction.getDataStatistics(), components.getAdapter().getTimeDescriptors(), @@ -578,6 +823,7 @@ protected TemporalConstraintsSet clipIndexedTemporalConstraints( } protected Geometry clipIndexedBBOXConstraints(final Geometry bbox) { + return QueryIndexHelper.clipIndexedBBOXConstraints( transaction.getDataStatistics(), components.getAdapter().getFeatureType(), @@ -586,13 +832,16 @@ protected Geometry clipIndexedBBOXConstraints(final Geometry bbox) { } private boolean subsetRequested() { + if (query == null) { return false; } return !(query.getPropertyNames() == Query.ALL_NAMES); } + @SuppressWarnings("unchecked") private String[] getSubset() { + if (query == null) { return new String[0]; } diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureSource.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureSource.java index 0609a45d0e9..8eb9bc3f11f 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureSource.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveFeatureSource.java @@ -81,7 +81,7 @@ protected ReferencedEnvelope getBoundsInternal(final Query query) throws IOExcep maxx = bboxStats.getMaxX(); miny = bboxStats.getMinY(); maxy = bboxStats.getMaxY(); - BoundingBoxStatistic statistic = (BoundingBoxStatistic) bboxStats.getStatistic(); + final BoundingBoxStatistic statistic = (BoundingBoxStatistic) bboxStats.getStatistic(); if (statistic.getDestinationCrs() != null) { bboxCRS = statistic.getDestinationCrs(); } else { @@ -134,7 +134,7 @@ protected int getCountInternal(final Query query) throws IOException { } public SimpleFeatureType getFeatureType() { - return components.getFeatureType(); + return GeoWaveFeatureCollection.getSchema(query, components.getFeatureType()); } @Override diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveGSProcessFactory.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveGSProcessFactory.java index 9ea2db5339c..9f3813bce55 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveGSProcessFactory.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveGSProcessFactory.java @@ -23,6 +23,7 @@ public GeoWaveGSProcessFactory() { Text.text("GeoWave Process Factory"), "geowave", SubsampleProcess.class, - DistributedRenderProcess.class); + DistributedRenderProcess.class, + GeoWaveHeatMapProcess.class); } } diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveHeatMapProcess.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveHeatMapProcess.java new file mode 100644 index 00000000000..146852cd557 --- /dev/null +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/GeoWaveHeatMapProcess.java @@ -0,0 +1,562 @@ +/** + * Copyright (c) 2013-2022 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin; + +import java.io.IOException; +import org.geotools.coverage.CoverageFactoryFinder; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.coverage.grid.GridCoverageFactory; +import org.geotools.data.Query; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.geotools.data.simple.SimpleFeatureIterator; +import org.geotools.filter.text.cql2.CQLException; +import org.geotools.filter.text.ecql.ECQL; +import org.geotools.geojson.feature.FeatureJSON; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.process.ProcessException; +import org.geotools.process.factory.DescribeParameter; +import org.geotools.process.factory.DescribeProcess; +import org.geotools.process.factory.DescribeResult; +import org.geotools.process.vector.BBOXExpandingFilterVisitor; +import org.geotools.process.vector.BilinearInterpolator; +import org.geotools.process.vector.HeatmapSurface; +import org.geotools.process.vector.VectorProcess; +import org.geotools.referencing.CRS; +import org.geotools.util.factory.GeoTools; +import org.geotools.util.factory.Hints; +import org.locationtech.geowave.core.geotime.util.GeometryUtils; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.util.Stopwatch; +import org.opengis.coverage.grid.GridCoverage; +import org.opengis.coverage.grid.GridGeometry; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.filter.Filter; +import org.opengis.filter.expression.Expression; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.NoSuchAuthorityCodeException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.util.ProgressListener; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * A Process that uses a {@link HeatmapSurface} to compute a heatmap surface over a set of irregular + * data points as a {@link GridCoverage}. Heatmaps are known more formally as Multivariate Kernel + * Density Estimation. + * + *

The appearance of the heatmap is controlled by the kernel radius, which determines the "radius + * of influence" of input points. The radius is specified by the radiusPixels parameter, which is in + * output pixels. Using pixels allows easy estimation of a value which will give a visually + * effective result, and ensures the heatmap appearance changes to match the zoom level. + * + *

By default each input point has weight 1. Optionally the weights of points may be supplied by + * an attribute specified by the weightAttr parameter. + * + *

All geometry types are allowed as input. For non-point geometries the centroid is used. + * + *

To improve performance, the surface grid can be computed at a lower resolution than the + * requested output image using the pixelsPerCell parameter. The grid is upsampled to + * match the required image size. Upsampling uses Bilinear Interpolation to maintain visual quality. + * This gives a large improvement in performance, with minimal impact on visual quality for small + * cell sizes (for instance, 10 pixels or less). + * + *

To ensure that the computed surface is stable (i.e. does not display obvious edge artifacts + * under zooming and panning), the data extent is expanded to be larger than the specified output + * extent. The expansion distance is equal to the size of radiusPixels in the input + * CRS. + * + *

Parameters

+ * + * M = mandatory, O = optional + * + *
  • data (M) - the FeatureCollection containing the point observations + *
  • radiusPixels (M)- the density kernel radius, in pixels
  • weightAttr (M)- the + * feature type attribute containing the observed surface value
  • pixelsPerCell (O) - The + * pixels-per-cell value determines the resolution of the computed grid. Larger values improve + * performance, but degrade appearance. (Default = 1)
  • outputBBOX (M) - The georeferenced + * bounding box of the output area
  • outputWidth (M) - The width of the output raster + *
  • outputHeight (M) - The height of the output raster
+ * + * The output of the process is a {@linkplain GridCoverage2D} with a single band, with cell values + * in the range [0, 1]. + * + *

Computation of the surface takes places in the CRS of the output. If the data CRS is different + * to the output CRS, the input points are transformed into the output CRS. + * + *

Using the process as a Rendering Transformation

+ * + * This process can be used as a RenderingTransformation, since it implements the + * invertQuery(... Query, GridGeometry) method. In this case the queryBuffer + * parameter should be specified to expand the query extent appropriately. The output raster + * parameters may be provided from the request extents, using the following SLD environment + * variables: + * + *
  • outputBBOX - env var = wms_bbox
  • outputWidth - env var = + * wms_width
  • outputHeight - env var = wms_height
+ * + * When used as an Rendering Transformation the data query is rewritten to expand the query BBOX, to + * ensure that enough data points are queried to make the computed surface stable under panning and + * zooming. + * + *

+ * + * @author M. Zagorski (customizations for GeoWave Heatmap rendering using aggregation and statistic + * spatial binning queries).
+ * @apiNode Note: based on the GeoTools version of HeatmapProcess by Martin Davis - OpenGeo. + * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + * + * + */ +@SuppressWarnings("deprecation") +@DescribeProcess( + title = "GeoWaveHeatMapProcess", + description = "Computes a heatmap surface over a set of data points and outputs as a single-band raster.") +public class GeoWaveHeatMapProcess implements VectorProcess { + + private static final Logger LOGGER = LoggerFactory.getLogger(GeoWaveHeatMapProcess.class); + + // For testing and verification of accuracy only (keep set to false in production) + Boolean writeGeoJson = false; + + // Query types + public static final String CNT_AGGR = "CNT_AGGR"; + public static final String SUM_AGGR = "SUM_AGGR"; + public static final String CNT_STATS = "CNT_STATS"; + public static final String SUM_STATS = "SUM_STATS"; + + + public static final Hints.Key HEATMAP_ENABLED = new Hints.Key(Boolean.class); + public static final Hints.Key OUTPUT_BBOX = new Hints.Key(ReferencedEnvelope.class); + public static final Hints.Key OUTPUT_WIDTH = new Hints.Key(Integer.class); + public static final Hints.Key OUTPUT_HEIGHT = new Hints.Key(Integer.class); + public static final Hints.Key GEOHASH_PREC = new Hints.Key(Integer.class); + public static final Hints.Key AGGR_QUERY = new Hints.Key(Boolean.class); + public static final Hints.Key STATS_QUERY = new Hints.Key(Boolean.class); + public static final Hints.Key QUERY_TYPE = new Hints.Key(String.class); + + // The value of the weight attribute must be numeric (e.g. cannot be a geometry, etc.) + public static final Hints.Key WEIGHT_ATTR = new Hints.Key(String.class); + public static final Hints.Key PIXELS_PER_CELL = new Hints.Key(Integer.class); + public static final Hints.Key CREATE_STATS = new Hints.Key(Boolean.class); + public static final Hints.Key USE_BINNING = new Hints.Key(Boolean.class); + + + @DescribeResult(name = "result", description = "Output raster") + public GridCoverage2D execute( + + // process data + @DescribeParameter( + name = "data", + description = "Input features") final SimpleFeatureCollection obsFeatures, + + // process parameters + @DescribeParameter( + name = "radiusPixels", + description = "Radius of the density kernel in pixels") final Integer argRadiusPixels, + @DescribeParameter( + name = "weightAttr", + description = "Name of the attribute to use for data point weight", + min = 0, + max = 1) final String valueAttr, + @DescribeParameter( + name = "pixelsPerCell", + description = "Resolution at which to compute the heatmap (in pixels). Default = 1", + defaultValue = "1", + min = 0, + max = 1) final Integer argPixelsPerCell, + + // output image parameters + @DescribeParameter( + name = "outputBBOX", + description = "Bounding box of the output") final ReferencedEnvelope argOutputEnv, + @DescribeParameter( + name = "outputWidth", + description = "Width of output raster in pixels") final Integer argOutputWidth, + @DescribeParameter( + name = "outputHeight", + description = "Height of output raster in pixels") final Integer argOutputHeight, + + // CUSTOM GEOWAVE PARAMETERS + // Options for queryType include: CNT_AGGR, SUM_AGGR, CNT_STATS, SUM_STATS + @DescribeParameter( + name = "queryType", + description = "Height of the output raster") final String queryType, + @DescribeParameter( + name = "createStats", + description = "Option to run statistics if they do not exist in datastore - must have queryType set to CNT_STATS or SUM_STATS.") final Boolean createStats, + @DescribeParameter( + name = "useSpatialBinning", + description = "Option to use spatial binning.") final Boolean useSpatialBinning, + + + final ProgressListener monitor) throws ProcessException { + + /** -------- Extract required information from process arguments ------------- */ + int pixelsPerCell = 1; + if ((argPixelsPerCell != null) && (argPixelsPerCell > 1)) { + pixelsPerCell = argPixelsPerCell; + } + final int outputWidth = argOutputWidth; + final int outputHeight = argOutputHeight; + int gridWidth = outputWidth; + int gridHeight = outputHeight; + if (pixelsPerCell > 1) { + gridWidth = outputWidth / pixelsPerCell; + gridHeight = outputHeight / pixelsPerCell; + } + + /** Compute transform to convert input coords into output CRS */ + final CoordinateReferenceSystem srcCRS = + useSpatialBinning ? GeometryUtils.getDefaultCRS() + : obsFeatures.getSchema().getCoordinateReferenceSystem(); + + final CoordinateReferenceSystem dstCRS = argOutputEnv.getCoordinateReferenceSystem(); + + MathTransform trans = null; + try { + trans = CRS.findMathTransform(srcCRS, dstCRS); + } catch (final FactoryException e) { + throw new ProcessException(e); + } + + // ------------ Kernel Radius + /* + * // not used for now - only pixel radius values are supported double distanceConversionFactor + * = distanceConversionFactor(srcCRS, dstCRS); double dstRadius = argRadius * + * distanceConversionFactor; + */ + int radiusCells = 100; + if (argRadiusPixels != null) { + radiusCells = argRadiusPixels; + } + if (pixelsPerCell > 1) { + radiusCells /= pixelsPerCell; + } + + /** + * -------------- Extract the input observation points and add them to the heatmap ----------- + */ + final HeatmapSurface heatMap = + new HeatmapSurface(radiusCells, argOutputEnv, gridWidth, gridHeight); + try { + extractPoints(obsFeatures, useSpatialBinning ? "weight" : valueAttr, trans, heatMap); + } catch (final CQLException e) { + throw new ProcessException(e); + } + + /** --------------- Do the processing on the heatmap------------------------------ */ + // KEEP the stopwatch for testing and verification purposes only + // Stopwatch sw = new Stopwatch(); + + // compute the heatmap at the specified resolution + float[][] heatMapGrid = heatMap.computeSurface(); + + // flip now, since grid size may be smaller + heatMapGrid = flipXY(heatMapGrid); + + // upsample to output resolution if necessary + float[][] outGrid = heatMapGrid; + if (pixelsPerCell > 1) { + outGrid = upsample(heatMapGrid, -999, outputWidth, outputHeight); + } + + // convert to the GridCoverage2D required for output + final GridCoverageFactory gcf = + CoverageFactoryFinder.getGridCoverageFactory(GeoTools.getDefaultHints()); + final GridCoverage2D gridCov = gcf.create("Process Results", outGrid, argOutputEnv); + + // KEEP THIS System.out for testing and verification purposes only + // System.out.println("************** Heatmap FINAL computed in " + sw.getTimeString()); + + return gridCov; + } + + /** + * Flips an XY matrix along the X=Y axis, and inverts the Y axis. Used to convert from "map + * orientation" into the "image orientation" used by GridCoverageFactory. The surface + * interpolation is done on an XY grid, with Y=0 being the bottom of the space. GridCoverages are + * stored in an image format, in a YX grid with Y=0 being the top. + * + * @param grid the grid to flip + * @return the flipped grid + */ + private float[][] flipXY(final float[][] grid) { + final int xsize = grid.length; + final int ysize = grid[0].length; + + final float[][] grid2 = new float[ysize][xsize]; + for (int ix = 0; ix < xsize; ix++) { + for (int iy = 0; iy < ysize; iy++) { + final int iy2 = ysize - iy - 1; + grid2[iy2][ix] = grid[ix][iy]; + } + } + return grid2; + } + + private float[][] upsample( + final float[][] grid, + final float noDataValue, + final int width, + final int height) { + final BilinearInterpolator bi = new BilinearInterpolator(grid, noDataValue); + final float[][] outGrid = bi.interpolate(width, height, false); + return outGrid; + } + + /** + * Given a target query and a target grid geometry returns the query to be used to read the input + * data of the process involved in rendering. In this process this method is used to: + * + *

  • determine the extent & CRS of the output grid
  • expand the query envelope to ensure + * stable surface generation
  • modify the query hints to ensure point features are returned + *
+ * + * Note that in order to pass validation, all parameters named here must also appear in the + * parameter list of the execute method, even if they are not used there. + * + * @param argRadiusPixels the feature type attribute that contains the observed surface value + * @param targetQuery the query used against the data source + * @param targetGridGeometry the grid geometry of the destination image + * @return The transformed query + */ + public Query invertQuery( + @DescribeParameter( + name = "radiusPixels", + description = "Radius to use for the kernel", + min = 0, + max = 1) final Integer argRadiusPixels, + @DescribeParameter( + name = "pixelsPerCell", + description = "Resolution at which to compute the heatmap (in pixels). Default = 1", + defaultValue = "1", + min = 0, + max = 1) final Integer argPixelsPerCell, + @DescribeParameter( + name = "weightAttr", + description = "Name of the attribute to use for data point weight", + min = 0, + max = 1) final String valueAttr, + // output image parameters + @DescribeParameter( + name = "outputBBOX", + description = "Georeferenced bounding box of the output") final ReferencedEnvelope argOutputEnv, + @DescribeParameter( + name = "outputWidth", + description = "Width of the output raster") final Integer argOutputWidth, + @DescribeParameter( + name = "outputHeight", + description = "Height of the output raster") final Integer argOutputHeight, + // Can be: CNT_AGGR, SUM_AGGR, CNT_STATS, SUM_STATS + @DescribeParameter( + name = "queryType", + description = "Height of the output raster") final String queryType, + @DescribeParameter( + name = "createStats", + description = "Option to run statistics if they do not exist in datastore - must have queryType set to CNT_STATS or SUM_STATS.") final Boolean createStats, + @DescribeParameter( + name = "useSpatialBinning", + description = "Option to use spatial binning.") final Boolean useSpatialBinning, + final Query targetQuery, + final GridGeometry targetGridGeometry) throws ProcessException { + + // Get hints for this process + final Hints hints = targetQuery.getHints(); + + // State that the hints for this process are enabled (for GeoWaveFeatureCollection.java) + hints.put(HEATMAP_ENABLED, true); + hints.put(PIXELS_PER_CELL, argPixelsPerCell); + hints.put(OUTPUT_WIDTH, argOutputWidth); + hints.put(OUTPUT_HEIGHT, argOutputHeight); + hints.put(OUTPUT_BBOX, argOutputEnv); + // hints.put(GEOHASH_PREC, 4); + // hints.put(AGGR_QUERY, true); + // hints.put(STATS_QUERY, false); + + // Add one of these values in the SLD: CNT_AGGR, SUM_AGGR, CNT_STATS, SUM_STATS. + hints.put(QUERY_TYPE, queryType); + + hints.put(WEIGHT_ATTR, valueAttr); + hints.put(CREATE_STATS, createStats); + hints.put(USE_BINNING, useSpatialBinning); + + final int radiusPixels = argRadiusPixels > 0 ? argRadiusPixels : 0; + // input parameters are required, so should be non-null + final double queryBuffer = + radiusPixels / pixelSize(argOutputEnv, argOutputWidth, argOutputHeight); + /* + * if (argQueryBuffer != null) { queryBuffer = argQueryBuffer; } + */ + targetQuery.setFilter(expandBBox(targetQuery.getFilter(), queryBuffer)); + + // clear properties to force all attributes to be read + // (required because the SLD processor cannot see the value attribute specified in the + // transformation) + // TODO: set the properties to read only the specified value attribute + targetQuery.setProperties(null); + + // set the decimation hint to ensure points are read + // Hints hints = targetQuery.getHints(); + hints.put(Hints.GEOMETRY_DISTANCE, 0.0); + + return targetQuery; + } + + private double pixelSize( + final ReferencedEnvelope outputEnv, + final int outputWidth, + final int outputHeight) { + // error-proofing + if (outputEnv.getWidth() <= 0) { + return 0; + } + // assume view is isotropic + return outputWidth / outputEnv.getWidth(); + } + + protected Filter expandBBox(final Filter filter, final double distance) { + return (Filter) filter.accept( + new BBOXExpandingFilterVisitor(distance, distance, distance, distance), + null); + } + + /** + * Extract points from a feature collection, and stores them in the heatmap + * + * @param obsPoints features to extract + * @param attrName expression or property name used to evaluate the geometry from a feature + * @param trans transform for extracted points + * @param heatMap heatmap to add points to + * @throws CQLException if attrName can't be parsed + */ + protected void extractPoints( + final SimpleFeatureCollection obsPoints, + final String attrName, + final MathTransform trans, + final HeatmapSurface heatMap) throws CQLException { + + Expression attrExpr = null; + if (attrName != null) { + attrExpr = ECQL.toExpression(attrName); + } + + int counter = 0; + + try (SimpleFeatureIterator obsIt = obsPoints.features()) { + final double[] srcPt = new double[2]; + final double[] dstPt = new double[2]; + + // Iterate over the results + while (obsIt.hasNext()) { + final SimpleFeature feature = obsIt.next(); + + // try { + // get the weight value, if any + double val = 1; + if (attrExpr != null) { + val = getPointValue(feature, attrExpr); + } + + // Get the information (testing and verification purposes only) + if (writeGeoJson) { + final Expression geohashIdExpr = ECQL.toExpression("geohashId"); + final String geohashId = geohashIdExpr.evaluate(feature, String.class); + + final Expression sourceExpr = ECQL.toExpression("source"); + final String source = sourceExpr.evaluate(feature, String.class); + + final Expression geohashPrecExpr = ECQL.toExpression("geohashPrec"); + final Integer geohashPrec = geohashPrecExpr.evaluate(feature, Integer.class); + + final Expression fieldNameExpr = ECQL.toExpression("field_name"); + final String fieldName = fieldNameExpr.evaluate(feature, String.class); + + // Create geojson file (for testing and verification purposes only) + counter++; + if (counter <= 30) { + final FeatureJSON fjson = new FeatureJSON(); + final String name = + "/xxx/xxx" + + fieldName + + "_GEOHASH_" + + geohashPrec + + "_" + + geohashId + + "_" + + source + + "_val_" + + val + + ".geojson"; + try { + fjson.writeFeature(feature, name); + } catch (final IOException e) { + e.printStackTrace(); + } + } + } + + // get the point location from the geometry + final Geometry geom = (Geometry) feature.getDefaultGeometry(); + final Coordinate p = getPoint(geom); + srcPt[0] = p.x; + srcPt[1] = p.y; + + try { + trans.transform(srcPt, 0, dstPt, 0, 1); + + final Coordinate pobs = new Coordinate(dstPt[0], dstPt[1], val); + + heatMap.addPoint(pobs.x, pobs.y, val); + } catch (final Exception e) { + LOGGER.warn( + "Expression {} failed to evaluate to a numeric value {} due to: {}", + attrExpr, + val, + e); + e.printStackTrace(); + } + } + } + } + + /** + * Gets a point to represent the Geometry. If the Geometry is a point, this is returned. + * Otherwise, the centroid is used. + * + * @param g the geometry to find a point for + * @return a point representing the Geometry + */ + private static Coordinate getPoint(final Geometry g) { + if (g.getNumPoints() == 1) { + return g.getCoordinate(); + } + return g.getCentroid().getCoordinate(); + } + + /** + * Gets the value for a point from the supplied attribute. The value is checked for validity, and + * a default of 1 is used if necessary. + * + * @param feature the feature to extract the value from + * @param attrExpr the expression specifying the attribute to read + * @return the value for the point + */ + private static double getPointValue(final SimpleFeature feature, final Expression attrExpr) { + final Double valObj = attrExpr.evaluate(feature, Double.class); + if (valObj != null) { + return valObj; + } + return 1; + } +} diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/QueryIssuerHeatMap.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/QueryIssuerHeatMap.java new file mode 100644 index 00000000000..fb40c3dd985 --- /dev/null +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/QueryIssuerHeatMap.java @@ -0,0 +1,39 @@ +/** + * Copyright (c) 2013-2020 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin; + +import org.geotools.feature.FeatureIterator; +import org.locationtech.geowave.core.store.CloseableIterator; +import org.locationtech.geowave.core.store.api.Index; +import org.locationtech.geowave.core.store.query.constraints.BasicQueryByClass; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.filter.Filter; + +/** + * Special class for the heatmap query. + * + * @author M. Zagorski
+ * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + */ + +public interface QueryIssuerHeatMap { + FeatureIterator query( + String queryType, + String weightAttr, + Integer pixelsPerCell, + Boolean createStats); + + Filter getFilter(); + + Integer getLimit(); + +} diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapAggregations.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapAggregations.java new file mode 100644 index 00000000000..d00ccf6dece --- /dev/null +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapAggregations.java @@ -0,0 +1,197 @@ +/** + * Copyright (c) 2013-2020 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin.heatmap; + +import java.math.BigDecimal; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Map.Entry; +import org.geotools.data.DataUtilities; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveDataStoreComponents; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveFeatureCollection; +import org.locationtech.geowave.core.geotime.binning.SpatialBinningType; +import org.locationtech.geowave.core.geotime.store.query.aggregate.SpatialSimpleFeatureBinningStrategy; +import org.locationtech.geowave.core.index.ByteArray; +import org.locationtech.geowave.core.store.adapter.statistics.histogram.TDigestNumericHistogram; +import org.locationtech.geowave.core.store.api.AggregationQuery; +import org.locationtech.geowave.core.store.api.AggregationQueryBuilder; +import org.locationtech.geowave.core.store.api.Index; +import org.locationtech.geowave.core.store.query.aggregate.FieldNameParam; +import org.locationtech.geowave.core.store.query.aggregate.FieldSumAggregation; +import org.locationtech.geowave.core.store.query.aggregate.OptimalCountAggregation; +import org.locationtech.geowave.core.store.query.constraints.QueryConstraints; +import org.locationtech.jts.geom.Geometry; +import org.opengis.feature.simple.SimpleFeature; + +/** + * Methods for HeatMap aggregation queries. + * + * @author M. Zagorski
+ * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + */ +public class HeatMapAggregations { + + public static String SUM_AGGR = "sum_aggr"; + public static String CNT_AGGR = "cnt_aggr"; + + + /** + * Builds the field sum aggregation query and returns a SimpleFeatureCollection. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + * @param weightAttr {String} The name of the field in the dataset to which the query is applied. + * @return {SimpleFeatureCollection} Returns a SimpleFeatureCollection of spatial bin centroids + * attributed with the aggregation value of their bin. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static SimpleFeatureCollection buildFieldSumAggrQuery( + final GeoWaveDataStoreComponents components, + final QueryConstraints queryConstraints, + final Geometry jtsBounds, + final Integer geohashPrec, + final String weightAttr) { + + // Initialize empty SimpleFeature list + final List newSimpleFeatures = new ArrayList<>(); + + // Initialize new query builder + final AggregationQueryBuilder queryBuilder = + AggregationQueryBuilder.newBuilder(); + + // Add spatial constraint to optimize the datastore query + queryBuilder.constraints(queryConstraints); + + // Set up the aggregate + queryBuilder.aggregate( + components.getAdapter().getTypeName(), + new FieldSumAggregation(new FieldNameParam(weightAttr))); + + // Set the index name from the data store + final Index[] indices = components.getDataStore().getIndices(); + final String indexName = indices[0].getName(); + queryBuilder.indexName(indexName); + + // Build the query with binning strategy + final AggregationQuery, SimpleFeature> agg = + queryBuilder.buildWithBinningStrategy( + new SpatialSimpleFeatureBinningStrategy(SpatialBinningType.GEOHASH, geohashPrec, true), + -1); + + // Apply aggregate query to the datastore and get the results + final Map results = components.getDataStore().aggregate(agg); + + // Loop over results and create new SimpleFeature using the centroid of the spatial bin + for (final Entry entry : results.entrySet()) { + final ByteArray geoHashId = entry.getKey(); + final BigDecimal weightValBigDec = entry.getValue(); + final Double weightVal = weightValBigDec.doubleValue(); + + final SimpleFeature simpFeature = + HeatMapUtils.buildSimpleFeature( + GeoWaveFeatureCollection.getHeatmapFeatureType(), + geoHashId, + weightVal, + geohashPrec, + weightAttr, + SUM_AGGR); + + newSimpleFeatures.add(simpFeature); + } + + // Add the new simple features to the SimpleFeatureCollection + final SimpleFeatureCollection newFeatures = DataUtilities.collection(newSimpleFeatures); + + return newFeatures; + } + + + /** + * Builds the count aggregation query and returns a SimpleFeatureCollection. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + * @param weightAttr {String} The name of the field in the dataset to which the query is applied. + * @return {SimpleFeatureCollection} Returns a SimpleFeatureCollection of spatial bin centroids + * attributed with the aggregation value of their bin. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + public static SimpleFeatureCollection buildCountAggrQuery( + // TDigestNumericHistogram histogram, + final GeoWaveDataStoreComponents components, + final QueryConstraints queryConstraints, + final Geometry jtsBounds, + final Integer geohashPrec, + final String weightAttr) { + + // Initialize empty SimpleFeature list + final List newSimpleFeatures = new ArrayList<>(); + + // Initialize new query builder + final AggregationQueryBuilder queryBuilder = + AggregationQueryBuilder.newBuilder(); + + // Add spatial constraint to optimize the datastore query + queryBuilder.constraints(queryConstraints); + + // Set up the aggregation based on the name of the geometry field + queryBuilder.aggregate( + components.getAdapter().getTypeName(), + new OptimalCountAggregation.FieldCountAggregation( + new FieldNameParam(HeatMapUtils.getGeometryFieldName(components)))); + + // Set the index name from the data store + final Index[] indices = components.getDataStore().getIndices(); + final String indexName = indices[0].getName(); + queryBuilder.indexName(indexName); + + // Build the query with binning strategy + final AggregationQuery, SimpleFeature> agg = + queryBuilder.buildWithBinningStrategy( + new SpatialSimpleFeatureBinningStrategy(SpatialBinningType.GEOHASH, geohashPrec, true), + -1); + + // Apply aggregate query to the datastore and get the results + final Map results = components.getDataStore().aggregate(agg); + + // Loop over results and create new SimpleFeatures using the centroid of the spatial bin + for (final Entry entry : results.entrySet()) { + final ByteArray geoHashId = entry.getKey(); + final Long weightValLong = entry.getValue(); + final Double weightVal = weightValLong.doubleValue(); + + final SimpleFeature simpFeature = + HeatMapUtils.buildSimpleFeature( + // histogram, + GeoWaveFeatureCollection.getHeatmapFeatureType(), + geoHashId, + weightVal, + geohashPrec, + weightAttr, + CNT_AGGR); + + newSimpleFeatures.add(simpFeature); + } + + // Add the new simple features to SimpleFeatureCollection + final SimpleFeatureCollection newFeatures = DataUtilities.collection(newSimpleFeatures); + + return newFeatures; + } + +} diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapStatistics.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapStatistics.java new file mode 100644 index 00000000000..ee8dffa958f --- /dev/null +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapStatistics.java @@ -0,0 +1,449 @@ +/** + * Copyright (c) 2013-2020 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin.heatmap; + +import java.util.ArrayList; +import java.util.List; +import org.apache.commons.lang3.tuple.Pair; +import org.geotools.data.DataUtilities; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveDataStoreComponents; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveFeatureCollection; +import org.locationtech.geowave.core.geotime.binning.SpatialBinningType; +import org.locationtech.geowave.core.geotime.store.statistics.binning.SpatialFieldValueBinningStrategy; +import org.locationtech.geowave.core.index.ByteArray; +import org.locationtech.geowave.core.store.CloseableIterator; +import org.locationtech.geowave.core.store.api.BinConstraints; +import org.locationtech.geowave.core.store.api.DataTypeStatistic; +import org.locationtech.geowave.core.store.api.FieldStatistic; +import org.locationtech.geowave.core.store.query.constraints.QueryConstraints; +import org.locationtech.geowave.core.store.statistics.adapter.CountStatistic; +import org.locationtech.geowave.core.store.statistics.adapter.CountStatistic.CountValue; +import org.locationtech.geowave.core.store.statistics.field.NumericStatsStatistic; +import org.locationtech.geowave.core.store.statistics.field.NumericStatsStatistic.NumericStatsValue; +import org.locationtech.geowave.core.store.statistics.field.Stats; +import org.locationtech.jts.geom.Geometry; +import org.opengis.feature.simple.SimpleFeature; + +/** + * Methods for HeatMap statistics queries.
+ * + * @author M. Zagorski
+ * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + */ +public class HeatMapStatistics { + + public static String SUM_STATS = "sum_stats"; + public static String CNT_STATS = "cnt_stats"; + public static String GEOHASH_STR = "geohash"; + + + /** + * Get the SimpleFeatures with count statistics. + * + * @param components {GeoWaveDataStoreComponents} The GeoWave datastore components. + * @param typeName {String} The type name of the feature type from components. + * @param weightAttr {String} The data field name being processed in the statistics. + * @param geohashPrec {Integer} The Geohash precision for binning purposes. + * @param jtsBounds {Geometry} The JTS geometry representing the bounds of the data. + * @return {List} Returns an array list of Simple Features. + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List getSimpleFeaturesWithCountStatistics( + GeoWaveDataStoreComponents components, + String typeName, + String weightAttr, + Integer geohashPrec, + Geometry jtsBounds) { + + // Initialize empty SimpleFeature list + List newSimpleFeatures = new ArrayList<>(); + + // Get all data type statistics from the datastore + DataTypeStatistic[] stats = components.getDataStore().getDataTypeStatistics(typeName); + + for (DataTypeStatistic stat : stats) { + + // Get the tag for the statistic + String statTag = stat.getTag(); + + // Only proceed if the tag contains "geohash" + if (statTag.contains(GEOHASH_STR)) { + + // Get the statistic Geohash precision from the tag + Integer statGeohashPrec = Integer.valueOf(statTag.split("-")[3]); + + // Find out if the statistic precision matches the geohash precision + Boolean matchPrec = statGeohashPrec.equals(geohashPrec); + + // Continue if a count statistic and an instance of spatial field value binning strategy + if ((stat.getStatisticType() == CountStatistic.STATS_TYPE) + && (stat.getBinningStrategy() instanceof SpatialFieldValueBinningStrategy) + && matchPrec) { + + // Get the spatial binning strategy + SpatialFieldValueBinningStrategy spatialBinningStrategy = + (SpatialFieldValueBinningStrategy) stat.getBinningStrategy(); + + // Continue only if spatial binning strategy type is GEOHASH + if (spatialBinningStrategy.getType() == SpatialBinningType.GEOHASH) { + + DataTypeStatistic geohashCount = stat; + + // Create new SimpleFeatures from the GeoHash centroid, add the statistic as attribute + try (CloseableIterator> it = + components.getDataStore().getBinnedStatisticValues( + geohashCount, + BinConstraints.ofObject(jtsBounds))) { + + // Iterate over all bins and build the SimpleFeature list + while (it.hasNext()) { + final Pair pair = it.next(); + ByteArray geohashId = pair.getLeft(); + Long weightValLong = pair.getRight(); + Double weightVal = weightValLong.doubleValue(); + + SimpleFeature simpFeature = + HeatMapUtils.buildSimpleFeature( + GeoWaveFeatureCollection.getHeatmapFeatureType(), + geohashId, + weightVal, + geohashPrec, + weightAttr, + CNT_STATS); + + newSimpleFeatures.add(simpFeature); + } + } + break; + } + } + } + } + + return newSimpleFeatures; + } + + + /** + * Builds the count statistics query and returns a SimpleFeatureCollection. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + * @param weightAttr {String} The name of the field in the dataset to which the query is applied. + * @param createStats {Boolean} User-specified preference to build and calculate the statistics if + * they do not exist in the datastore (otherwise, the query will default to the equivalent + * aggregation query). + * @return {SimpleFeatureCollection} Returns a SimpleFeatureCollection of spatial bin centroids + * attributed with the aggregation value of their bin. + */ + public static SimpleFeatureCollection buildCountStatsQuery( + final GeoWaveDataStoreComponents components, + final QueryConstraints queryConstraints, + final Geometry jtsBounds, + final Integer geohashPrec, + final String weightAttr, + final Boolean createStats) { + + // Get type name + final String typeName = components.getFeatureType().getTypeName(); + // Note - Another way to get the typeName: String typeName = + // components.getAdapter().getTypeName(); + + // Get the simple features with count statistics + List newSimpleFeatures = + getSimpleFeaturesWithCountStatistics( + components, + typeName, + weightAttr, + geohashPrec, + jtsBounds); + + // Add the new simple features to SimpleFeatureCollection (ok if empty at this point in time) + SimpleFeatureCollection newFeatures = DataUtilities.collection(newSimpleFeatures); + + // Only proceed if newFeatures is empty (requested statistics do not exist in datastore) + if (newFeatures.size() == 0) { + + // Add GeoHash field count statistic to datastore and render it when createStats is true + if (createStats) { + addGeoHashCountStatisticToDataStore(components, typeName, geohashPrec); + + newSimpleFeatures = + getSimpleFeaturesWithCountStatistics( + components, + typeName, + weightAttr, + geohashPrec, + jtsBounds); + + newFeatures = DataUtilities.collection(newSimpleFeatures); + } else { + + // Default to the count aggregation query for rendered results + newFeatures = + HeatMapAggregations.buildCountAggrQuery( + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr); + } + } + + return newFeatures; + } + + + /** + * Programmatically add a GeoHash count statistic to the DataStore. This should only be done once + * as needed. The count is the number of instance geometries per GeoHash grid cell. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param typeName {String} The name of the data layer or dataset. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + */ + private static void addGeoHashCountStatisticToDataStore( + GeoWaveDataStoreComponents components, + String typeName, + Integer geohashPrec) { + + // Set up the count statistic + final CountStatistic geohashCount = new CountStatistic(typeName); + + // Set a tag for information purposes + String tagStr = "count-stat-geohash-" + geohashPrec; + geohashCount.setTag(tagStr); + + // Set up spatial binning strategy + final SpatialFieldValueBinningStrategy geohashSpatialBinning = + new SpatialFieldValueBinningStrategy(HeatMapUtils.getGeometryFieldName(components)); + + // Set the type to GeoHash + geohashSpatialBinning.setType(SpatialBinningType.GEOHASH); + + // Set the GeoHash precision + geohashSpatialBinning.setPrecision(geohashPrec); + + // Set the binning strategy + geohashCount.setBinningStrategy(geohashSpatialBinning); + + // Add statistics to datastore + components.getDataStore().addStatistic(geohashCount); + } + + + /** + * Get the SimpleFeatures with field statistics. + * + * @param components {GeoWaveDataStoreComponents} The GeoWave datastore components. + * @param typeName {String} The type name of the feature type from components. + * @param weightAttr {String} The data field name being processed in the statistics. + * @param geohashPrec {Integer} The Geohash precision for binning purposes. + * @param jtsBounds {Geometry} The JTS geometry representing the bounds of the data. + * @return {List} Returns an array list of Simple Features. + */ + @SuppressWarnings({"rawtypes", "unchecked"}) + private static List getSimpleFeaturesWithFieldStatistics( + GeoWaveDataStoreComponents components, + String typeName, + String weightAttr, + Integer geohashPrec, + Geometry jtsBounds) { + + // Initialize empty SimpleFeature list + List newSimpleFeatures = new ArrayList<>(); + + // Get all data type statistics from the datastore + FieldStatistic[] stats = components.getDataStore().getFieldStatistics(typeName, weightAttr); + + for (FieldStatistic stat : stats) { + + // Get the tag for the statistic + final String statTag = stat.getTag(); + + // Only proceed if the tag contains "geohash" + if (statTag.contains(GEOHASH_STR)) { + + // Get the stored Geohash precision from the tag + final Integer statGeohashPrec = Integer.valueOf(statTag.split("-")[3]); + + // Find out if the statistic precision matches the geohash precision + final Boolean matchPrec = statGeohashPrec.equals(geohashPrec); + + // Continue if a field sum statistic and an instance of spatial field value binning strategy + if ((stat.getStatisticType() == NumericStatsStatistic.STATS_TYPE) + && (stat.getBinningStrategy() instanceof SpatialFieldValueBinningStrategy) + && matchPrec) { + + // Get the spatial binning strategy + final SpatialFieldValueBinningStrategy spatialBinningStrategy = + (SpatialFieldValueBinningStrategy) stat.getBinningStrategy(); + + // Continue only if spatial binning strategy type is GEOHASH + if (spatialBinningStrategy.getType() == SpatialBinningType.GEOHASH) { + + final FieldStatistic geohashNumeric = stat; + + // Create new SimpleFeatures from the GeoHash centroid and add the statistic and other + try (CloseableIterator> it = + components.getDataStore().getBinnedStatisticValues( + geohashNumeric, + BinConstraints.ofObject(jtsBounds))) { + + // Iterate over all bins and build the SimpleFeature list + while (it.hasNext()) { + final Pair pair = it.next(); + final ByteArray geoHashId = pair.getLeft(); + final Double fieldSum = pair.getRight().sum(); + + // KEEP THIS - Other types of field statistics: + // Long fieldCount = pair.getRight().count(); + // Double fieldMean = pair.getRight().mean(); + // Double fieldMax = pair.getRight().max(); + // Double fieldMin = pair.getRight().min(); + + final SimpleFeature simpFeature = + HeatMapUtils.buildSimpleFeature( + GeoWaveFeatureCollection.getHeatmapFeatureType(), + geoHashId, + fieldSum, // TODO: this could be made dynamic + geohashPrec, + weightAttr, + SUM_STATS); + + newSimpleFeatures.add(simpFeature); + } + } + break; + } + } + } + } + return newSimpleFeatures; + } + + + /** + * Builds the field statistics query and returns a SimpleFeatureCollection. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param jtsBounds {Geometry} The geometry representing the bounds of the GeoServer map viewer + * extent. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + * @param weightAttr {String} The name of the field in the dataset to which the query is applied. + * @param createStats {Boolean} User-specified preference to build and calculate the statistics if + * they do not exist in the datastore (otherwise, the query will default to the equivalent + * aggregation query). + * @return {SimpleFeatureCollection} Returns a SimpleFeatureCollection of spatial bin centroids + * attributed with the aggregation value of their bin. + */ + public static SimpleFeatureCollection buildFieldStatsQuery( + final GeoWaveDataStoreComponents components, + final QueryConstraints queryConstraints, + final Geometry jtsBounds, + final Integer geohashPrec, + final String weightAttr, + final Boolean createStats) { + + // Get type name + final String typeName = components.getFeatureType().getTypeName(); + + // Get the simple features if the statistics already exist in the datastore + List newSimpleFeatures = + getSimpleFeaturesWithFieldStatistics( + components, + typeName, + weightAttr, + geohashPrec, + jtsBounds); + + // Add the new simple features to SimpleFeatureCollection (ok if empty at this point in time) + SimpleFeatureCollection newFeatures = DataUtilities.collection(newSimpleFeatures); + + // Only proceed if newFeatures is empty (requested statistics do not exist in datastore) + if (newFeatures.size() == 0) { + + // Add GeoHash field statistic to datastore and render it when createStats is true + if (createStats) { + addGeoHashFieldStatisticsToDataStore(components, typeName, geohashPrec, weightAttr); + + newSimpleFeatures = + getSimpleFeaturesWithFieldStatistics( + components, + typeName, + weightAttr, + geohashPrec, + jtsBounds); + + newFeatures = DataUtilities.collection(newSimpleFeatures); + + } else { + // Default to the count aggregation query for rendered results + newFeatures = + HeatMapAggregations.buildFieldSumAggrQuery( + components, + queryConstraints, + jtsBounds, + geohashPrec, + weightAttr); + } + } + + return newFeatures; + } + + + /** + * Programmatically add a GeoHash field statistic to the DataStore. This should only be done once + * as needed. The default statistic is sum, but could be count, mean, max, or min of the selected + * numeric field. + * + * @param components {GeoWaveDataStoreComponents} The base components of the dataset. + * @param typeName {String} The name of the data layer or dataset. + * @param geohashPrec {Integer} The Geohash precision to use for binning. + * @param weightAttr {String} The name of the field in the dataset to which the query is applied. + */ + private static void addGeoHashFieldStatisticsToDataStore( + final GeoWaveDataStoreComponents components, + final String typeName, + final Integer geohashPrec, + final String weightAttr) { + + // Set up the field statistic + final NumericStatsStatistic geohashFieldStat = new NumericStatsStatistic(typeName, weightAttr); + + // Set a tag for information purposes + String tagStr = "field-stat-geohash-" + geohashPrec; + geohashFieldStat.setTag(tagStr); + + // Set up spatial binning strategy + final SpatialFieldValueBinningStrategy geohashSpatialBinning = + new SpatialFieldValueBinningStrategy( + components.getFeatureType().getGeometryDescriptor().getLocalName()); + + // Set the type to GeoHash + geohashSpatialBinning.setType(SpatialBinningType.GEOHASH); + + // Set the GeoHash precision + geohashSpatialBinning.setPrecision(geohashPrec); + + // Set the binning strategy + geohashFieldStat.setBinningStrategy(geohashSpatialBinning); + + // Add statistics to datastore + components.getDataStore().addStatistic(geohashFieldStat); + } + +} diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtils.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtils.java new file mode 100644 index 00000000000..11396949374 --- /dev/null +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtils.java @@ -0,0 +1,456 @@ +/** + * Copyright (c) 2013-2020 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin.heatmap; + +import org.geotools.feature.simple.SimpleFeatureBuilder; +import org.geotools.feature.simple.SimpleFeatureTypeBuilder; +import org.geotools.geometry.jts.JTS; +import org.geotools.referencing.CRS; +import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.locationtech.geowave.core.geotime.binning.SpatialBinningType; +import org.locationtech.geowave.core.geotime.util.GeometryUtils; +import org.locationtech.geowave.core.index.ByteArray; +import org.locationtech.geowave.core.store.adapter.statistics.histogram.TDigestNumericHistogram; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.Point; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.geometry.MismatchedDimensionException; +import org.opengis.referencing.FactoryException; +import org.opengis.referencing.crs.CoordinateReferenceSystem; +import org.opengis.referencing.operation.MathTransform; +import org.opengis.referencing.operation.TransformException; +import com.github.davidmoten.geo.GeoHash; +import com.github.davidmoten.geo.LatLong; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveDataStoreComponents; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; + +/** + * Utility methods to support HeatMap queries. + * + * @author M. Zagorski
+ * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + */ +public class HeatMapUtils { + + public static int SQ_KM_CONV = 1000 * 1000; + private static int axis = 6378137; + + static final Logger LOGGER = LoggerFactory.getLogger(HeatMapUtils.class); + + /** + * Creates the heatmap feature type + * + * @return {SimpleFeatureType} Returns the SimpleFeatureType + */ + public static SimpleFeatureType createHeatmapFeatureType() { + + // Initialize new SimpleFeatureTypeBuilder + final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder(); + + // Set Name and CRS + typeBuilder.setName("heatmap_bins"); + typeBuilder.setCRS(GeometryUtils.getDefaultCRS()); + + // Add keys to the typeBuilder + typeBuilder.add("the_geom", Geometry.class); + typeBuilder.add("field_name", String.class); + typeBuilder.add("weight", Double.class); + typeBuilder.add("geohashId", String.class); + typeBuilder.add("source", String.class); + typeBuilder.add("geohashPrec", Integer.class); + + // Build the new type + return typeBuilder.buildFeatureType(); + } + + /** + * Builds a simple feature. + * + * @param featureType {SimpleFeatureType} The feature type of the simple feature. + * @param geohashId {ByteArray} The geohash grid cell ID. + * @param value {Double} The value calculated by the aggregation or statistics query. + * @param precision {Integer} The Geohash precision level (1-12). + * @param weightAttr {String} The target data field name. + * @param source {String} The code that indicates the type of query. + * @return {SimpleFeature} Returns a SimpleFeature containing the query value and relevant + * information. + */ + public static SimpleFeature buildSimpleFeature( + // final TDigestNumericHistogram histogram, + final SimpleFeatureType featureType, + final ByteArray geohashId, + final Double value, + final Integer precision, + final String weightAttr, + final String source) { + + // // Get the coordinate reference system + // CoordinateReferenceSystem oldCRS = featureType.getCoordinateReferenceSystem(); + // String oldName = featureType.getTypeName(); + + // Convert the value to a double + double valDbl = value.doubleValue(); + + // Get the histogram-weighted value + // valDbl = histogram.cdf(valDbl); + + // Convert GeoHash ID to string + String geoHashIdStr = geohashId.getString(); + + // Get centroid of GeoHash cell + final LatLong ll = GeoHash.decodeHash(geohashId.getString()); + Geometry centroid = + GeometryUtils.GEOMETRY_FACTORY.createPoint(new Coordinate(ll.getLon(), ll.getLat())); + + // // Initialize new SimpleFeatureTypeBuilder + // final SimpleFeatureTypeBuilder typeBuilder = new SimpleFeatureTypeBuilder(); + + // // Set Name and CRS + // typeBuilder.setName(oldName); + // typeBuilder.setCRS(oldCRS); + + // // Add keys to the typeBuilder + // typeBuilder.add("the_geom", Geometry.class); + // typeBuilder.add("field_name", String.class); + // typeBuilder.add(weightAttr, Double.class); + // typeBuilder.add("geohashId", String.class); + // typeBuilder.add("source", String.class); + // typeBuilder.add("geohashPrec", Integer.class); + + // // Build the new type + // SimpleFeatureType newType = typeBuilder.buildFeatureType(); + + // Create heatmap feature type + // SimpleFeatureType heatmapType = createHeatmapFeatureType(); + + // Initialize the new SimpleFeatureBuilder using the new type + final SimpleFeatureBuilder builder = new SimpleFeatureBuilder(featureType); + // final SimpleFeatureBuilder builder = new SimpleFeatureBuilder(heatmapType); + + // Set values + builder.set("the_geom", centroid); + builder.set("field_name", weightAttr); + builder.set("weight", valDbl); + builder.set("geohashId", geoHashIdStr); + builder.set("source", source); + builder.set("geohashPrec", precision); + + return builder.buildFeature(geoHashIdStr); + } + + + /** + * Get an appropriate Geohash precision based on the approximate area (square kilometers) of a + * grid cell. + * + * @param cellArea {double} The area (square kilometers) of the grid cell (from the GeoServer + * mapping extent). + * @return Returns an integer for the Geohash precision (1-12). + */ + public static int getGeohashPrecision(double cellArea) { + if (cellArea >= 10000000) + return 1; + if (cellArea >= 500000) + return 2; + if (cellArea >= 15000) + return 3; + if (cellArea >= 500) + return 4; + if (cellArea >= 15) + return 5; + if (cellArea >= 1) + return 6; + if (cellArea >= 0.01) + return 7; + if (cellArea >= 0.0005) + return 8; + if (cellArea >= 0.00002) + return 9; + if (cellArea >= 0.000005) + return 10; + if (cellArea >= 0.00000002) + return 11; + if (cellArea >= 0) + return 12; + return 4; + } + + /** + * Calculate the approximate area of a geometry based. To be used for geometries projected in a + * metric based projection. + * + * @param geom {Geometry} The input geometry to be processed. + * @param sourceCRS {CoordinateReferenceSystem} The source CRS. + * @return {Double} Returns the area in square kilometers. + */ + public static double getAreaMetricProjections( + Geometry geom, + CoordinateReferenceSystem sourceCRS) { + return geom.getArea() / SQ_KM_CONV; + } + + /** + * Calculate the approximate area of a geometry based on its envelope. To be used for geometries + * projected in a non-metric based projection. + * + * @param geom {Geometry} The input geometry to be processed. + * @param sourceCRS {CoordinateReferenceSystem} The source CRS. + * @return {Double} Returns the area in square kilometers. + */ + public static double getAreaNonMetricProjections( + Geometry geom, + CoordinateReferenceSystem sourceCRS) { + + // Convert geometry to WGS84 + geom = convertToWGS84(geom, sourceCRS); + + // Get the area and length from the geometry envelope + double area = geom.getEnvelope().getArea(); + double length = geom.getEnvelope().getLength(); + + // Calculate the width of the envelope based on its area and length + double width = area / length; + + // Calculate the length, width in meters and the area is square kilometers + double lengthMeters = length * (Math.PI / 180) * axis; + double widthMeters = width * (Math.PI / 180) * axis; + double geomArea = (lengthMeters * widthMeters) / SQ_KM_CONV; + + return geomArea; + } + + + /** + * Calculate the area of a geometry in square kilometers. + * + * @param geom {Geometry} The input geometry to be processed. + * @return {double} Returns a double representing the area of the input geometry. + */ + public static double calcAreaSqKm(Geometry geom, CoordinateReferenceSystem sourceCRS) { + + double geomArea = 0; + + // Get centroid of geometry + Point centroid = geom.getCentroid(); + + // Find out if the CRS is WGS84 + Boolean isWGS84 = sourceCRS.getName().getCode().equals("WGS 84"); + + // Get area of non-WGS84 CRSs + if (!isWGS84) { + + // Get the units of the projection + String projectionUnits = sourceCRS.getCoordinateSystem().getAxis(0).getUnit().toString(); + + // Determine if projection is metric based + Boolean isUnitMeters = projectionUnits.equals("m"); + + // Calculate the area + if (isUnitMeters) { + geomArea = getAreaMetricProjections(geom, sourceCRS); + } else { + geomArea = getAreaNonMetricProjections(geom, sourceCRS); + } + } + + // Project the geometry in order to get an accurate area + if (isWGS84) { + + // Get longitude coordinate of centroid + double longitude = centroid.getX(); + + // Get latitude coordinate of centroid + double latitude = centroid.getY(); + + // Get the location + String code = "AUTO:42001," + longitude + "," + latitude; + + // Initialize empty CRS + CoordinateReferenceSystem crs; + + try { + // Decode the location to get the CRS + crs = CRS.decode(code); + + // Get the transform and use leniency + MathTransform transform = CRS.findMathTransform(DefaultGeographicCRS.WGS84, crs, true); + + // Project the geometry using the transform + Geometry geomProj = JTS.transform(geom, transform); + + // Calculate the area (square kilometers) based on the projected geometry + geomArea = geomProj.getArea() / SQ_KM_CONV; + + } catch (FactoryException e) { + e.printStackTrace(); + } catch (MismatchedDimensionException e) { + e.printStackTrace(); + } catch (TransformException e) { + e.printStackTrace(); + } + } + return geomArea; + } + + /** + * Returns the cell count of the GeoServer map viewer extent. + * + * @param width {Integer} The width of the GeoServer map viewer extent. + * @param height {Integer} The height of the GeoServer map viewer extent. + * @param pixelsPerCell {Integer} The count of pixels per cell. + * @return {Integer} Returns an integer representing the cell count in the GeoServer map viewer + * extent. + */ + public static int getExtentCellCount(int width, int height, int pixelsPerCell) { + + // Get the count of grid cells for the width and height of the extent + int cntCellsWidth = width / pixelsPerCell; + int cntCellsHeight = height / pixelsPerCell; + + // Get the total count of grid cells in the extent + int extentCellCount = cntCellsWidth * cntCellsHeight; + + return extentCellCount; + } + + + /** + * Returns the approximate area of a single cell in the GeoServer map viewer extent. + * + * @param extentAreaSqKm {Double} The area of the GeoServer map viewer extent in square + * kilometers. + * @param totCellsTarget {Integer} The total count of cells in the GeoServer map viewer extent. + * @return {Double} Returns a double representing the approximate area of each cell in the + * GeoServer map viewer extent. + */ + public static double getCellArea(double extentAreaSqKm, int totCellsTarget) { + return extentAreaSqKm / totCellsTarget; + } + + + /** + * Automatic selection of an appropriate Geohash precision. + * + * @param height {Integer} The height of the GeoServer map viewer extent. + * @param width {Integer} The width of the GeoServer map viewer extent. + * @param pixelsPerCell {Integer} The number of pixels per GeoServer map viewer cell. + * @param jtsBounds {Geometry} The geometry that represents the GeoServer map viewer extent. + * @return {Integer} Returns an integer representing an appropriate Geohash precision. + */ + public static int autoSelectGeohashPrecision( + int height, + int width, + int pixelsPerCell, + Geometry jtsBounds, + CoordinateReferenceSystem sourceCRS) { + + // Get total count of cells in GeoServer map viewer extent + int totCellsTarget = HeatMapUtils.getExtentCellCount(width, height, pixelsPerCell); + + // Get the area of the GeoServer map viewer extent in square kilometers + double extentAreaSqKm = HeatMapUtils.calcAreaSqKm(jtsBounds, sourceCRS); + + // Get approximate area of a single cell in square kilometers + double cellArea = HeatMapUtils.getCellArea(extentAreaSqKm, totCellsTarget); + + // Get the most appropriate Geohash precision (e.g. 1-12) based on the cell area + int geohashPrec = HeatMapUtils.getGeohashPrecision(cellArea); + + return geohashPrec; + } + + + /** + * Get the field name of the geometry column from the input data. + * + * @param components {GeoWaveDataStoreComponents} The base components of the data. + * @return {String} Returns a string representing the field name of the geometry column from the + * input data. + */ + public static String getGeometryFieldName(GeoWaveDataStoreComponents components) { + return components.getFeatureType().getGeometryDescriptor().getLocalName(); + } + + /** + * Convert a geometry to WGS84 CRS. + * + * @param geometry {Geometry} + * @param sourceCRS {CoordinateReferenceSystem} + * @return {Geometry} Returns the geometry in WGS84 CRS. + */ + public static Geometry convertToWGS84(Geometry geometry, CoordinateReferenceSystem sourceCRS) { + + MathTransform transform; + Geometry targetGeometry = null; + try { + // Decode WGS84 + CoordinateReferenceSystem targetCRS = CRS.decode("EPSG:4326"); + + // Find the math transform from the source CRS to WGS84 + transform = CRS.findMathTransform(sourceCRS, targetCRS, true); + try { + // Transform the geometry + targetGeometry = JTS.transform(geometry, transform); + + // Set SRID - this is not necessary + targetGeometry.setSRID(4326); + } catch (MismatchedDimensionException | TransformException e) { + e.printStackTrace(); + } + } catch (FactoryException e) { + e.printStackTrace(); + } + + return targetGeometry; + } + + /** + * Get the approximate Geohash precision based on a comparative method. Note: this method runs a + * bit slower than autoSelectGeohashPrecision. + * + * @param width {Integer} The width (in pixels) of the map viewer. + * @param jtsBounds {Geometry} The geometry representing the extent of the map viewer. + * @param pixelsPerCell {Integer} The number of pixels per cell. + * @return + */ + public static Integer getGeohashPrecisionComp( + Integer width, + Geometry jtsBounds, + Integer pixelsPerCell) { + + int holdAbsDiff = 0; + int geohashPrec = 1; + + // Get total cell counts for each GeoHash precision + int totCellsTarget = width / pixelsPerCell; + + // Iterate over Geohash precisions 3 through 12 and find closest match to totCellsTarget + // best + for (int i = 3; i <= 12; i++) { + int cntCellsAtPrec = (SpatialBinningType.GEOHASH.getSpatialBins(jtsBounds, i)).length; + int absDiff = Math.abs(cntCellsAtPrec - totCellsTarget); + + if (absDiff > holdAbsDiff && holdAbsDiff != 0) { + break; + } + + holdAbsDiff = absDiff; + geohashPrec = i + 1; + } + + return geohashPrec; + } + +} diff --git a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/transaction/StatisticsCache.java b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/transaction/StatisticsCache.java index e386b76af93..0be42e4f57c 100644 --- a/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/transaction/StatisticsCache.java +++ b/extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/plugin/transaction/StatisticsCache.java @@ -59,9 +59,11 @@ public , R> V getFieldStatistic( statisticsStore.getFieldStatistics(adapter, statisticType, fieldName, null)) { if (statsIter.hasNext()) { Statistic stat = (Statistic) statsIter.next(); - V value = statisticsStore.getStatisticValue(stat, authorizations); - if (value != null) { - retVal = value; + if (stat.getBinningStrategy() == null) { + V value = statisticsStore.getStatisticValue(stat, authorizations); + if (value != null) { + retVal = value; + } } } } @@ -81,9 +83,11 @@ public , R> V getAdapterStatistic( statisticsStore.getDataTypeStatistics(adapter, statisticType, null)) { if (statsIter.hasNext()) { Statistic stat = (Statistic) statsIter.next(); - V value = statisticsStore.getStatisticValue(stat, authorizations); - if (value != null) { - retVal = value; + if (stat.getBinningStrategy() == null) { + V value = statisticsStore.getStatisticValue(stat, authorizations); + if (value != null) { + retVal = value; + } } } } diff --git a/extensions/adapters/vector/src/test/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtilsTest.java b/extensions/adapters/vector/src/test/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtilsTest.java new file mode 100644 index 00000000000..2c31afa4543 --- /dev/null +++ b/extensions/adapters/vector/src/test/java/org/locationtech/geowave/adapter/vector/plugin/heatmap/HeatMapUtilsTest.java @@ -0,0 +1,88 @@ +/** + * Copyright (c) 2013-2022 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.adapter.vector.plugin.heatmap; + +import static org.junit.Assert.assertEquals; +import org.geotools.filter.FilterFactoryImpl; +import org.junit.Ignore; +import org.junit.Test; +import org.locationtech.geowave.core.index.ByteArray; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.PrecisionModel; +import org.opengis.feature.simple.SimpleFeature; +import org.opengis.filter.spatial.BBOX; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveFeatureCollection; + +/** + * Unit tests for HeatMapUtils.
+ * + * @author M. Zagorski
+ * @apiNote Date: 5-10-2022
+ * + * @apiNote Changelog:
+ * + */ + +public class HeatMapUtilsTest { + final GeometryFactory factory = new GeometryFactory(new PrecisionModel(PrecisionModel.FIXED)); + + @Test + public void getGeohashPrecisionTest() { + assertEquals(1, HeatMapUtils.getGeohashPrecision(10000000)); + assertEquals(2, HeatMapUtils.getGeohashPrecision(500000)); + assertEquals(3, HeatMapUtils.getGeohashPrecision(15000)); + assertEquals(4, HeatMapUtils.getGeohashPrecision(500)); + assertEquals(5, HeatMapUtils.getGeohashPrecision(15)); + assertEquals(6, HeatMapUtils.getGeohashPrecision(1)); + assertEquals(7, HeatMapUtils.getGeohashPrecision(0.01)); + assertEquals(8, HeatMapUtils.getGeohashPrecision(0.0005)); + assertEquals(9, HeatMapUtils.getGeohashPrecision(0.00002)); + assertEquals(10, HeatMapUtils.getGeohashPrecision(0.000005)); + assertEquals(11, HeatMapUtils.getGeohashPrecision(0.00000002)); + assertEquals(12, HeatMapUtils.getGeohashPrecision(0.00000001)); + } + + @Test + public void getExtentCellCountTest() { + assertEquals(3312, HeatMapUtils.getExtentCellCount(920, 360, 10)); + } + + @Test + public void getCellAreaTest() { + assertEquals(200, Math.round(HeatMapUtils.getCellArea(20000, 100))); + } + + @SuppressWarnings("static-access") + @Ignore + @Test + public void buildSimpleFeatureTest() { + + final FilterFactoryImpl factory = new FilterFactoryImpl(); + BBOX bb = factory.bbox("geometry", 28, 41, 28.5, 41.5, "EPSG:4326"); + + String geohashID = "9trg"; + byte[] geohashId = geohashID.getBytes(); + final Double weightVal = 2.0; + final Integer geohashPrec = 4; + final String weightAttr = "SIZE"; + + ByteArray geohashIdB = new ByteArray(); + geohashIdB.fromBytes(geohashId); + + SimpleFeature simpFeature = + HeatMapUtils.buildSimpleFeature( + GeoWaveFeatureCollection.getHeatmapFeatureType(), + geohashIdB, + weightVal, + geohashPrec, + weightAttr, + "CNT_STATS"); + } + +} diff --git a/test/src/main/java/org/locationtech/geowave/test/TestUtils.java b/test/src/main/java/org/locationtech/geowave/test/TestUtils.java index 15bbfd62cdc..340fa34102f 100644 --- a/test/src/main/java/org/locationtech/geowave/test/TestUtils.java +++ b/test/src/main/java/org/locationtech/geowave/test/TestUtils.java @@ -147,7 +147,11 @@ public Index[] getDefaultIndices() { // CRS for Web Mercator public static String CUSTOM_CRSCODE = "EPSG:3857"; + // CRS for WGS84 + public static String CUSTOM_CRSCODE_WGS84 = "EPSG:4326"; + public static final CoordinateReferenceSystem CUSTOM_CRS; + public static final CoordinateReferenceSystem CUSTOM_CRS_WGS84; public static final double DOUBLE_EPSILON = 1E-8d; @@ -160,6 +164,15 @@ public Index[] getDefaultIndices() { } } + static { + try { + CUSTOM_CRS_WGS84 = CRS.decode(CUSTOM_CRSCODE_WGS84, true); + } catch (final FactoryException e) { + LOGGER.error("Unable to decode " + CUSTOM_CRSCODE_WGS84 + "CRS", e); + throw new RuntimeException("Unable to initialize " + CUSTOM_CRSCODE_WGS84 + " CRS"); + } + } + public static Index createWebMercatorSpatialIndex() { final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider(); final SpatialOptions so = sdp.createOptions(); @@ -177,6 +190,23 @@ public static Index createWebMercatorSpatialTemporalIndex() { return primaryIndex; } + public static Index createWGS84SpatialIndex() { + final SpatialDimensionalityTypeProvider sdp = new SpatialDimensionalityTypeProvider(); + final SpatialOptions so = sdp.createOptions(); + so.setCrs(CUSTOM_CRSCODE_WGS84); + final Index primaryIndex = SpatialDimensionalityTypeProvider.createIndexFromOptions(so); + return primaryIndex; + } + + public static Index createWGS84SpatialTemporalIndex() { + final SpatialTemporalDimensionalityTypeProvider p = + new SpatialTemporalDimensionalityTypeProvider(); + final SpatialTemporalOptions o = p.createOptions(); + o.setCrs(CUSTOM_CRSCODE_WGS84); + final Index primaryIndex = SpatialTemporalDimensionalityTypeProvider.createIndexFromOptions(o); + return primaryIndex; + } + public static final String S3_INPUT_PATH = "s3://geowave-test/data/gdelt"; public static final String S3URL = "s3.amazonaws.com"; @@ -738,6 +768,7 @@ public static void testTileAgainstReference( // test under default style for (int x = 0; x < expected.getWidth(); x++) { for (int y = 0; y < expected.getHeight(); y++) { + if (actual.getRGB(x, y) != expected.getRGB(x, y)) { errorPixels++; if (errorPixels > maxErrorPixels) { @@ -892,6 +923,7 @@ public static void assertStatusCode( final Response response) { final String assertionMsg = msg + String.format(": A %s response code should be received", expectedCode); + Assert.assertEquals(assertionMsg, expectedCode, response.getStatus()); } diff --git a/test/src/main/java/org/locationtech/geowave/test/services/ServicesTestEnvironment.java b/test/src/main/java/org/locationtech/geowave/test/services/ServicesTestEnvironment.java index 1bd40623913..98377e5d674 100644 --- a/test/src/main/java/org/locationtech/geowave/test/services/ServicesTestEnvironment.java +++ b/test/src/main/java/org/locationtech/geowave/test/services/ServicesTestEnvironment.java @@ -67,6 +67,11 @@ public static synchronized ServicesTestEnvironment getInstance() { protected static final String TEST_STYLE_NAME_MINOR_SUBSAMPLE = "SubsamplePoints-10px"; protected static final String TEST_STYLE_NAME_MAJOR_SUBSAMPLE = "SubsamplePoints-100px"; protected static final String TEST_STYLE_NAME_DISTRIBUTED_RENDER = "DistributedRender"; + protected static final String TEST_STYLE_NAME_HEATMAP = "HeatMap-no-spatial-binning"; + protected static final String TEST_STYLE_NAME_HEATMAP_CNT_AGGR = "HeatMap-cnt-aggr"; + protected static final String TEST_STYLE_NAME_HEATMAP_SUM_AGGR = "HeatMap-sum-aggr"; + protected static final String TEST_STYLE_NAME_HEATMAP_CNT_STATS = "HeatMap-cnt-stats"; + protected static final String TEST_STYLE_NAME_HEATMAP_SUM_STATS = "HeatMap-sum-stats"; protected static final String TEST_STYLE_PATH = "src/test/resources/sld/"; protected static final String TEST_GEOSERVER_LOGGING_PATH = "src/test/resources/logging.xml"; protected static final String TEST_LOG_PROPERTIES_PATH = @@ -83,6 +88,16 @@ public static synchronized ServicesTestEnvironment getInstance() { TEST_STYLE_PATH + TEST_STYLE_NAME_MAJOR_SUBSAMPLE + ".sld"; protected static final String TEST_SLD_DISTRIBUTED_RENDER_FILE = TEST_STYLE_PATH + TEST_STYLE_NAME_DISTRIBUTED_RENDER + ".sld"; + protected static final String TEST_SLD_HEATMAP_FILE = + TEST_STYLE_PATH + TEST_STYLE_NAME_HEATMAP + ".sld"; + protected static final String TEST_SLD_HEATMAP_FILE_CNT_AGGR = + TEST_STYLE_PATH + TEST_STYLE_NAME_HEATMAP_CNT_AGGR + ".sld"; + protected static final String TEST_SLD_HEATMAP_FILE_SUM_AGGR = + TEST_STYLE_PATH + TEST_STYLE_NAME_HEATMAP_SUM_AGGR + ".sld"; + protected static final String TEST_SLD_HEATMAP_FILE_CNT_STATS = + TEST_STYLE_PATH + TEST_STYLE_NAME_HEATMAP_CNT_STATS + ".sld"; + protected static final String TEST_SLD_HEATMAP_FILE_SUM_STATS = + TEST_STYLE_PATH + TEST_STYLE_NAME_HEATMAP_SUM_STATS + ".sld"; private Server jettyServer; diff --git a/test/src/test/java/org/locationtech/geowave/test/basic/GeoWaveSpatialBinningStatisticsIT.java b/test/src/test/java/org/locationtech/geowave/test/basic/GeoWaveSpatialBinningStatisticsIT.java index 8388b901b71..b68f0871446 100644 --- a/test/src/test/java/org/locationtech/geowave/test/basic/GeoWaveSpatialBinningStatisticsIT.java +++ b/test/src/test/java/org/locationtech/geowave/test/basic/GeoWaveSpatialBinningStatisticsIT.java @@ -242,7 +242,7 @@ private static void testGeometry(final SimpleFeatureType featureType, final Data final Map> perBinResults = new HashMap<>(); stats.stream().forEach(s -> { - final Map results = new HashMap<>();; + final Map results = new HashMap<>(); perBinResults.put( new BinningStrategyKey((SpatialFieldValueBinningStrategy) s.getBinningStrategy()), results); diff --git a/test/src/test/java/org/locationtech/geowave/test/services/GeoServerIngestIT.java b/test/src/test/java/org/locationtech/geowave/test/services/GeoServerIngestIT.java index c1c4c409d79..0b2c74c1b98 100644 --- a/test/src/test/java/org/locationtech/geowave/test/services/GeoServerIngestIT.java +++ b/test/src/test/java/org/locationtech/geowave/test/services/GeoServerIngestIT.java @@ -58,6 +58,14 @@ import org.slf4j.Logger; import org.slf4j.LoggerFactory; +/** + * Integration tests for GeoServer ingest. + * + * @apiNote Changelog:
3-25-2022 M. Zagorski: Added tests for custom HeatMapProcess using + * spatial binning.
+ * + */ + @RunWith(GeoWaveITRunner.class) @Environments({Environment.SERVICES}) public class GeoServerIngestIT extends BaseServiceIT { @@ -67,12 +75,63 @@ public class GeoServerIngestIT extends BaseServiceIT { private static ConfigServiceClient configServiceClient; private static StoreServiceClient storeServiceClient; private static final String WORKSPACE = "testomatic"; + private static final String WORKSPACE2 = "testomatic2"; private static final String WMS_VERSION = "1.3"; private static final String WMS_URL_PREFIX = "/geoserver/wms"; private static final String REFERENCE_WMS_IMAGE_PATH = TestUtils.isOracleJRE() ? "src/test/resources/wms/wms-grid-oraclejdk.gif" : "src/test/resources/wms/wms-grid.gif"; + // TODO: create a heatmap .gif using non-Oracle JRE. + private static final String REFERENCE_WMS_HEATMAP_NO_SB = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-no-spat-bin-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-no-spat-bin.gif"; + + private static final String REFERENCE_WMS_HEATMAP_CNT_AGGR = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-cnt-aggr-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-cnt-aggr.gif"; + + private static final String REFERENCE_WMS_HEATMAP_SUM_AGGR = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-sum-aggr-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-sum-aggr.gif"; + + private static final String REFERENCE_WMS_HEATMAP_CNT_STATS = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-cnt-stats-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-cnt-stats.gif"; + + private static final String REFERENCE_WMS_HEATMAP_SUM_STATS = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-sum-stats-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-sum-stats.gif"; + + private static final String REFERENCE_WMS_HEATMAP_CNT_AGGR_ZOOM = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom.gif"; + + private static final String REFERENCE_WMS_HEATMAP_SUM_AGGR_ZOOM = + TestUtils.isOracleJRE() ? "src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom-oraclejdk.gif" + : "src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom.gif"; + + + private static final String REFERENCE_WMS_HEATMAP_NO_SB_WGS84 = + TestUtils.isOracleJRE() + ? "src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84-oraclejdk.gif" + : "src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84.gif"; + + private static final String REFERENCE_WMS_HEATMAP_CNT_AGGR_WGS84 = + TestUtils.isOracleJRE() ? "src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-oraclejdk.gif" + : "src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84.gif"; + + private static final String REFERENCE_WMS_HEATMAP_CNT_AGGR_ZOOM_WGS84 = + TestUtils.isOracleJRE() + ? "src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom-oraclejdk.gif" + : "src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom.gif"; + + private static final String REFERENCE_WMS_HEATMAP_SUM_AGGR_ZOOM_WGS84 = + TestUtils.isOracleJRE() + ? "src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom-oraclejdk.gif" + : "src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom.gif"; + + private static final String testName = "GeoServerIngestIT"; @GeoWaveTestStore( @@ -87,9 +146,11 @@ public class GeoServerIngestIT extends BaseServiceIT { // GeoServer and this thread have different class // loaders so the RocksDB "singleton" instances are not shared in // this JVM and GeoServer, for file-based geoserver data sources, using the REST - // "importer" will be more handy than adding a layer by referencing the local file system + // "importer" will be more handy than adding a layer by referencing the local + // file system GeoWaveStoreType.ROCKSDB, - // filesystem sporadically fails with a null response on spatial-temporal subsampling + // filesystem sporadically fails with a null response on spatial-temporal + // subsampling // (after the spatial index is removed and the services restarted) GeoWaveStoreType.FILESYSTEM}, namespace = testName) @@ -112,6 +173,13 @@ public static void reportTest() { TestUtils.printEndOfTest(LOGGER, testName, startMillis); } + /** + * Create a grid of points that are temporal. + * + * @param pointBuilder {SimpleFeatureBuilder} The simple feature builder. + * @param firstFeatureId {Integer} The first feature ID. + * @return List A list of simple features. + */ private static List getGriddedTemporalFeatures( final SimpleFeatureBuilder pointBuilder, final int firstFeatureId) { @@ -127,7 +195,7 @@ private static List getGriddedTemporalFeatures( dates[2] = cal.getTime(); // put 3 points on each grid location with different temporal attributes final List feats = new ArrayList<>(); - // extremes are close to -180,180,-90,and 90 wiuthout exactly matching + // extremes are close to -180,180,-90,and 90 without exactly matching // because coordinate transforms are illegal on the boundary for (int longitude = -36; longitude <= 36; longitude++) { for (int latitude = -18; latitude <= 18; latitude++) { @@ -138,9 +206,9 @@ private static List getGriddedTemporalFeatures( pointBuilder.set("TimeStamp", dates[date]); pointBuilder.set("Latitude", latitude); pointBuilder.set("Longitude", longitude); - // Note since trajectoryID and comment are marked as - // nillable we - // don't need to set them (they default ot null). + + // Add value of 2 to the SIZE field for sum aggregation and statistics + pointBuilder.set("SIZE", 2); final SimpleFeature sft = pointBuilder.buildFeature(String.valueOf(featureId)); feats.add(sft); @@ -151,28 +219,93 @@ private static List getGriddedTemporalFeatures( return feats; } + + @SuppressWarnings({"unchecked", "rawtypes"}) + private ArrayList getZoomedCoordinates(BoundingBoxValue env) { + + ArrayList coordSet = new ArrayList(); + + double widthX = env.getWidth() / 64; + double heightY = env.getHeight() / 64; + double centerX = (env.getMinX() + env.getMaxX()) / 2; + double centerY = (env.getMinY() + env.getMaxY()) / 2; + + Double minX = centerX - widthX; + Double maxX = centerX + widthX; + Double minY = centerY - heightY; + Double maxY = centerY + heightY; + + coordSet.add(minX); + coordSet.add(maxX); + coordSet.add(minY); + coordSet.add(maxY); + + return coordSet; + } + + /** + * Test projected data. + * + * @throws Exception + */ + @SuppressWarnings("unchecked") @Test - public void testExamplesIngest() throws Exception { + public void testExamplesIngestProjected() throws Exception { final DataStore ds = dataStorePluginOptions.createDataStore(); final SimpleFeatureType sft = SimpleIngest.createPointFeatureType(); + + // Keep these Booleans for local testing purposes + Boolean runNoSpatialBinning = true; + Boolean runCntAggr = true; + Boolean runCntAggrZoom = true; + Boolean runSumAggr = true; + Boolean runSumAggrZoom = true; + Boolean runCntStats = false; + Boolean runSumStats = true; + + // Use Web Mercator projection final Index spatialIdx = TestUtils.createWebMercatorSpatialIndex(); + + // Set the spatial temporal index final Index spatialTemporalIdx = TestUtils.createWebMercatorSpatialTemporalIndex(); + + @SuppressWarnings("rawtypes") + // Create data adapter final GeotoolsFeatureDataAdapter fda = SimpleIngest.createDataAdapter(sft); + + // Create grid of temporal points final List features = getGriddedTemporalFeatures(new SimpleFeatureBuilder(sft), 8675309); LOGGER.info( String.format("Beginning to ingest a uniform grid of %d features", features.size())); - int ingestedFeatures = 0; + + // Get a subset count final int featuresPer5Percent = features.size() / 20; + + // Add the type to the datastore ds.addType(fda, spatialIdx, spatialTemporalIdx); + + // Initialize a bounding box statistic final BoundingBoxStatistic mercatorBounds = new BoundingBoxStatistic(fda.getTypeName(), sft.getGeometryDescriptor().getLocalName()); + + // Set the source CRS mercatorBounds.setSourceCrs( fda.getFeatureType().getGeometryDescriptor().getCoordinateReferenceSystem()); + + // Set the destination CRS mercatorBounds.setDestinationCrs(TestUtils.CUSTOM_CRS); + + // Set the tag mercatorBounds.setTag("MERCATOR_BOUNDS"); + + // Add the statistic to the datastore ds.addStatistic(mercatorBounds); - try (Writer writer = ds.createWriter(fda.getTypeName())) { + + // Write a subset of features to the datastore + int ingestedFeatures = 0; + try (@SuppressWarnings("rawtypes") + Writer writer = ds.createWriter(fda.getTypeName())) { for (final SimpleFeature feat : features) { writer.write(feat); ingestedFeatures++; @@ -184,11 +317,15 @@ public void testExamplesIngest() throws Exception { } } } + + // Get the bounding box envelope final BoundingBoxValue env = ds.aggregateStatistics( StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName( fda.getTypeName()).fieldName(sft.getGeometryDescriptor().getLocalName()).tag( "MERCATOR_BOUNDS").build()); + + // Check the status codes of various processes TestUtils.assertStatusCode( "Should Create 'testomatic' Workspace", 201, @@ -206,12 +343,14 @@ public void testExamplesIngest() throws Exception { dataStorePluginOptions.getGeoWaveNamespace(), "testomatic", dataStorePluginOptions.getGeoWaveNamespace())); + TestUtils.assertStatusCode( "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_NO_DIFFERENCE + "' Style", 201, geoServerServiceClient.addStyle( ServicesTestEnvironment.TEST_SLD_NO_DIFFERENCE_FILE, ServicesTestEnvironment.TEST_STYLE_NAME_NO_DIFFERENCE)); + muteLogging(); TestUtils.assertStatusCode( "Should return 400, that layer was already added", @@ -227,18 +366,63 @@ public void testExamplesIngest() throws Exception { geoServerServiceClient.addStyle( ServicesTestEnvironment.TEST_SLD_MINOR_SUBSAMPLE_FILE, ServicesTestEnvironment.TEST_STYLE_NAME_MINOR_SUBSAMPLE)); + TestUtils.assertStatusCode( "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_MAJOR_SUBSAMPLE + "' Style", 201, geoServerServiceClient.addStyle( ServicesTestEnvironment.TEST_SLD_MAJOR_SUBSAMPLE_FILE, ServicesTestEnvironment.TEST_STYLE_NAME_MAJOR_SUBSAMPLE)); + TestUtils.assertStatusCode( "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_DISTRIBUTED_RENDER + "' Style", 201, geoServerServiceClient.addStyle( ServicesTestEnvironment.TEST_SLD_DISTRIBUTED_RENDER_FILE, ServicesTestEnvironment.TEST_STYLE_NAME_DISTRIBUTED_RENDER)); + + // ----------------HEATMAP RESPONSE TESTS------------------------------------ + // Test response code for heatmap - no spatial binning + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP)); + + // Test response code for heatmap CNT_AGGR + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_CNT_AGGR, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR)); + + // Test response code for heatmap SUM_AGGR + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_SUM_AGGR, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR)); + + // Test response code for heatmap CNT_STATS + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_STATS + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_CNT_STATS, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_STATS)); + + // Test response code for heatmap SUM_STATS + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_STATS + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_SUM_STATS, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_STATS)); + // ----------------------------------------------------------------------------------------- + TestUtils.assertStatusCode( "Should Publish '" + SimpleIngest.FEATURE_NAME + "' Layer", 201, @@ -248,6 +432,7 @@ public void testExamplesIngest() throws Exception { null, null, "point")); + if (!(ds instanceof Closeable)) { // this is kinda hacky, but its only for the integration test - the // problem is that GeoServer and this thread have different class @@ -267,6 +452,7 @@ public void testExamplesIngest() throws Exception { "point")); unmuteLogging(); } + final BufferedImage biDirectRender = getWMSSingleTile( env.getMinX(), @@ -278,6 +464,7 @@ public void testExamplesIngest() throws Exception { 920, 360, null, + true, true); final BufferedImage ref = ImageIO.read(new File(REFERENCE_WMS_IMAGE_PATH)); @@ -296,7 +483,8 @@ public void testExamplesIngest() throws Exception { 920, 360, null, - false); + false, + true); Assert.assertNotNull(ref); // being a little lenient because of differences in O/S rendering @@ -313,7 +501,8 @@ public void testExamplesIngest() throws Exception { 920, 360, null, - false); + false, + true); TestUtils.testTileAgainstReference(biSubsamplingWithExpectedError, ref, 0.01, 0.15); BufferedImage biSubsamplingWithLotsOfError = @@ -327,7 +516,8 @@ public void testExamplesIngest() throws Exception { 920, 360, null, - false); + false, + true); TestUtils.testTileAgainstReference(biSubsamplingWithLotsOfError, ref, 0.3, 0.4); final BufferedImage biDistributedRendering = @@ -341,13 +531,203 @@ public void testExamplesIngest() throws Exception { 920, 360, null, + true, true); TestUtils.testTileAgainstReference(biDistributedRendering, ref, 0, 0.07); + // ------------------------------HEATMAP PROJECTED RENDERING---------------------- + + // Get coordinates for zoomed-in tests + ArrayList zoomCoords = getZoomedCoordinates(env); + + Double minXzoom = zoomCoords.get(0); + Double maxXzoom = zoomCoords.get(1); + Double minYzoom = zoomCoords.get(2); + Double maxYzoom = zoomCoords.get(3); + + // Test the count aggregation heatmap rendering (NO SPATIAL BINNING) + if (runNoSpatialBinning) { + BufferedImage heatMapRenderingNoSpatBin; + + heatMapRenderingNoSpatBin = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapNoSpatialBinning = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_NO_SB)); + + TestUtils.testTileAgainstReference( + heatMapRenderingNoSpatBin, + refHeatMapNoSpatialBinning, + 0, + 0.07); + } + + // Test the count aggregation heatmap rendering (CNT_AGGR) + if (runCntAggr) { + final BufferedImage heatMapRenderingCntAggr = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapCntAggr = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_CNT_AGGR)); + + TestUtils.testTileAgainstReference(heatMapRenderingCntAggr, refHeatMapCntAggr, 0, 0.07); + + } + + if (runCntAggrZoom) { + + // Test zoomed-in version of heatmap count aggregation + final BufferedImage heatMapRenderingCntAggrZoom = + getWMSSingleTile( + minXzoom, + maxXzoom, + minYzoom, + maxYzoom, + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapCntAggrZoom = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_CNT_AGGR_ZOOM)); + + TestUtils.testTileAgainstReference( + heatMapRenderingCntAggrZoom, + refHeatMapCntAggrZoom, + 0, + 0.07); + + } + + // Test the field sum aggregation heatmap rendering (SUM_AGGR) + if (runSumAggr) { + final BufferedImage heatMapRenderingSumAggr = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapSumAggr = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_SUM_AGGR)); + + TestUtils.testTileAgainstReference(heatMapRenderingSumAggr, refHeatMapSumAggr, 0, 0.07); + + } + + if (runSumAggrZoom) { + + // Test zoomed-in version of heatmap sum aggregation + final BufferedImage heatMapRenderingSumAggrZoom = + getWMSSingleTile( + minXzoom, + maxXzoom, + minYzoom, + maxYzoom, + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapSumAggrZoom = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_SUM_AGGR_ZOOM)); + + TestUtils.testTileAgainstReference( + heatMapRenderingSumAggrZoom, + refHeatMapSumAggrZoom, + 0.0, + 0.07); + + } + + // Test the count statistics heatmap rendering + if (runCntStats) { + + final BufferedImage heatMapRenderingCntStats = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_STATS, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapCntStats = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_CNT_STATS)); + + TestUtils.testTileAgainstReference(heatMapRenderingCntStats, refHeatMapCntStats, 0, 0.07); + + } + + // Test the sum statistics heatmap rendering + if (runSumStats) { + final BufferedImage heatMapRenderingSumStats = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_STATS, + 920, + 360, + null, + false, + true); + + final BufferedImage refHeatMapSumStats = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_SUM_STATS)); + + TestUtils.testTileAgainstReference(heatMapRenderingSumStats, refHeatMapSumStats, 0, 0.07); + + } + // ------------------------------------------------------------------------- + // Test subsampling with only the spatial-temporal index ds.removeIndex(spatialIdx.getName()); ServicesTestEnvironment.getInstance().restartServices(); + // Test subsample rendering without error biSubsamplingWithoutError = getWMSSingleTile( env.getMinX(), @@ -359,11 +739,13 @@ public void testExamplesIngest() throws Exception { 920, 360, null, + true, true); Assert.assertNotNull(ref); // being a little lenient because of differences in O/S rendering TestUtils.testTileAgainstReference(biSubsamplingWithoutError, ref, 0, 0.071); + // Test subsample rendering with expected error biSubsamplingWithExpectedError = getWMSSingleTile( env.getMinX(), @@ -375,9 +757,11 @@ public void testExamplesIngest() throws Exception { 920, 360, null, + true, true); TestUtils.testTileAgainstReference(biSubsamplingWithExpectedError, ref, 0.01, 0.151); + // Test subsample rendering with lots of error biSubsamplingWithLotsOfError = getWMSSingleTile( env.getMinX(), @@ -389,10 +773,307 @@ public void testExamplesIngest() throws Exception { 920, 360, null, + true, true); TestUtils.testTileAgainstReference(biSubsamplingWithLotsOfError, ref, 0.3, 0.41); } + /** + * Run data that is unprojected (has regular GCS WGS84, but no projection) + * + * @throws Exception + */ + @SuppressWarnings("unchecked") + @Test + public void testExamplesIngestUnProjected() throws Exception { + final DataStore ds = dataStorePluginOptions.createDataStore(); + final SimpleFeatureType sft = SimpleIngest.createPointFeatureType(); + + // Set booleans for WGS84 tests + Boolean runNoSpatialBinningWGS84 = true; + Boolean runCntAggrWGS84 = false; + Boolean runCntAggrZoomedWGS84 = true; + Boolean runSumAggrZoomedWGS84 = true; + + // Use WGS84 coordinate system + final Index spatialIdx = TestUtils.createWGS84SpatialIndex(); + + // Set the spatial temporal index + final Index spatialTemporalIdx = TestUtils.createWGS84SpatialTemporalIndex(); + + @SuppressWarnings("rawtypes") + // Create data adapter + final GeotoolsFeatureDataAdapter fda = SimpleIngest.createDataAdapter(sft); + + // Create grid of temporal points + final List features = + getGriddedTemporalFeatures(new SimpleFeatureBuilder(sft), 8675309); + LOGGER.info( + String.format("Beginning to ingest a uniform grid of %d features", features.size())); + + // Initialize ingested features counter + int ingestedFeatures = 0; + + // Get a subset count + final int featuresPer5Percent = features.size() / 20; + + // Add the type to the datastore + ds.addType(fda, spatialIdx, spatialTemporalIdx); + + // Initialize a bounding box statistic + final BoundingBoxStatistic wgs84Bounds = + new BoundingBoxStatistic(fda.getTypeName(), sft.getGeometryDescriptor().getLocalName()); + + // Set the source CRS + wgs84Bounds.setSourceCrs( + fda.getFeatureType().getGeometryDescriptor().getCoordinateReferenceSystem()); + + // Set the destination CRS + wgs84Bounds.setDestinationCrs(TestUtils.CUSTOM_CRS_WGS84); + + // Set the tag + wgs84Bounds.setTag("WGS84_BOUNDS"); + + // Add the statistic to the datastore + ds.addStatistic(wgs84Bounds); + + // Write a subset of features to the datastore + try (@SuppressWarnings("rawtypes") + Writer writer = ds.createWriter(fda.getTypeName())) { + for (final SimpleFeature feat : features) { + writer.write(feat); + ingestedFeatures++; + if ((ingestedFeatures % featuresPer5Percent) == 0) { + LOGGER.info( + String.format( + "Ingested %d percent of features", + (ingestedFeatures / featuresPer5Percent) * 5)); + } + } + } + + // Get the bounding box envelope + final BoundingBoxValue env = + ds.aggregateStatistics( + StatisticQueryBuilder.newBuilder(BoundingBoxStatistic.STATS_TYPE).typeName( + fda.getTypeName()).fieldName(sft.getGeometryDescriptor().getLocalName()).tag( + "WGS84_BOUNDS").build()); + + // Check the status codes of various processes + TestUtils.assertStatusCode( + "Should Create 'testomatic2' Workspace", + 201, + geoServerServiceClient.addWorkspace("testomatic2")); + storeServiceClient.addStoreReRoute( + dataStorePluginOptions.getGeoWaveNamespace(), + dataStorePluginOptions.getType(), + dataStorePluginOptions.getGeoWaveNamespace(), + dataStorePluginOptions.getOptionsAsMap()); + + TestUtils.assertStatusCode( + "Should Add " + dataStorePluginOptions.getGeoWaveNamespace() + " Datastore", + 201, + geoServerServiceClient.addDataStore( + dataStorePluginOptions.getGeoWaveNamespace(), + "testomatic2", + dataStorePluginOptions.getGeoWaveNamespace())); + + + // ----------------HEATMAP SLD RESPONSE TESTS------------------------------------ + + // Test response code for heatmap NO SPATIAL BINNING + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP)); + + // Test response code for heatmap CNT_AGGR + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_CNT_AGGR, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR)); + + // Test response code for heatmap SUM_AGGR + TestUtils.assertStatusCode( + "Should Publish '" + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR + "' Style", + 201, + geoServerServiceClient.addStyle( + ServicesTestEnvironment.TEST_SLD_HEATMAP_FILE_SUM_AGGR, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR)); + + // ----------------------------------------------------------------------------------------- + + TestUtils.assertStatusCode( + "Should Publish '" + SimpleIngest.FEATURE_NAME + "' Layer", + 201, + geoServerServiceClient.addLayer( + dataStorePluginOptions.getGeoWaveNamespace(), + WORKSPACE2, + null, + null, + "point")); + + if (!(ds instanceof Closeable)) { + // this is kinda hacky, but its only for the integration test - the + // problem is that GeoServer and this thread have different class + // loaders so the RocksDB "singleton" instances are not shared in + // this JVM and GeoServer currently has a lock on the datastore + // after the previous addlayer - add layer tries to lookup adapters + // while it does not have the lock and therefore fails + muteLogging(); + TestUtils.assertStatusCode( + "Should return 400, that layer was already added", + 400, + geoServerServiceClient.addLayer( + dataStorePluginOptions.getGeoWaveNamespace(), + WORKSPACE2, + null, + null, + "point")); + unmuteLogging(); + } + + // ------------------------------HEATMAP WGS84 RENDERING---------------------- + + // Get coordinates for zoomed-in tests + ArrayList zoomCoords = getZoomedCoordinates(env); + + Double minXzoom = zoomCoords.get(0); + Double maxXzoom = zoomCoords.get(1); + Double minYzoom = zoomCoords.get(2); + Double maxYzoom = zoomCoords.get(3); + + // Test the count aggregation heatmap rendering (NO SPATIAL BINNING) + if (runNoSpatialBinningWGS84) { + BufferedImage heatMapRenderingNoSpatBinWGS84; + + heatMapRenderingNoSpatBinWGS84 = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP, + 920, + 360, + null, + false, + false); + + final BufferedImage refHeatMapNoSpatialBinningWGS84 = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_NO_SB_WGS84)); + TestUtils.testTileAgainstReference( + heatMapRenderingNoSpatBinWGS84, + refHeatMapNoSpatialBinningWGS84, + 0, + 0.07); + + } + + // Test the count aggregation heatmap rendering WGS84 (CNT_AGGR) + if (runCntAggrWGS84) { + // TODO: if this is run, centroid at 0, 0 cannot be projected at full extent. + final BufferedImage heatMapRenderingCntAggrWGS84 = + getWMSSingleTile( + env.getMinX(), + env.getMaxX(), + env.getMinY(), + env.getMaxY(), + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR, + 920, + 360, + null, + false, + false); + + final BufferedImage refHeatMapCntAggrWGS84 = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_CNT_AGGR_WGS84)); + TestUtils.testTileAgainstReference( + heatMapRenderingCntAggrWGS84, + refHeatMapCntAggrWGS84, + 0.0, + 0.41); + + } + + // Test the count aggregation heatmap rendering WGS84 (CNT_AGGR zoomed-in) + if (runCntAggrZoomedWGS84) { + final BufferedImage heatMapRenderingCntAggrWGS84Zoomed = + getWMSSingleTile( + minXzoom, + maxXzoom, + minYzoom, + maxYzoom, + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR, + 920, + 360, + null, + false, + false); + + final BufferedImage refHeatMapCntAggrWGS84Zoom = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_CNT_AGGR_ZOOM_WGS84)); + TestUtils.testTileAgainstReference( + heatMapRenderingCntAggrWGS84Zoomed, + refHeatMapCntAggrWGS84Zoom, + 0.0, + 0.07); + + } + + // Test the sum aggregation heatmap rendering WGS84 (SUM_AGGR zoomed-in) + if (runSumAggrZoomedWGS84) { + final BufferedImage heatMapRenderingSumAggrWGS84Zoomed = + getWMSSingleTile( + minXzoom, + maxXzoom, + minYzoom, + maxYzoom, + SimpleIngest.FEATURE_NAME, + ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR, + 920, + 360, + null, + false, + false); + + final BufferedImage refHeatMapSumAggrWGS84Zoom = + ImageIO.read(new File(REFERENCE_WMS_HEATMAP_SUM_AGGR_ZOOM_WGS84)); + TestUtils.testTileAgainstReference( + heatMapRenderingSumAggrWGS84Zoomed, + refHeatMapSumAggrWGS84Zoom, + 0.0, + 0.07); + + } + } + + + /** + * Creates a buffered image using a specified process. + * + * @param minX {double} Minimum longitude of the extent envelope. + * @param maxX {double} Maximum longitude of the extent envelope. + * @param minY {double} Minimum latitude of the extent envelope. + * @param maxY {double} Maximum latitude of the extent envelope. + * @param layer {String} The input grid. + * @param style {String} The SLD to use. + * @param width {Integer} Width (in pixels) of the extent. + * @param height {Integer} Height (in pixels) of the extent. + * @param outputFormat {String} Output format. + * @param temporalFilter {Boolean} If the data uses a temporal component. + * @param projected {Boolean} Indicates if the data is projected or just GCS (WGS84). + * @return {BufferedImage} A buffered image. + * @throws IOException + * @throws URISyntaxException + */ private static BufferedImage getWMSSingleTile( final double minX, final double maxX, @@ -403,8 +1084,16 @@ private static BufferedImage getWMSSingleTile( final int width, final int height, final String outputFormat, - final boolean temporalFilter) throws IOException, URISyntaxException { + final boolean temporalFilter, + final boolean projected) throws IOException, URISyntaxException { + + // String crsToUse = projected ? "EPSG:3857" : "EPSG:4326"; + String crsToUse = projected ? TestUtils.CUSTOM_CRSCODE : TestUtils.CUSTOM_CRSCODE_WGS84; + + // Initiate an empty Uniform Resource Identifier (URI) builder final URIBuilder builder = new URIBuilder(); + + // Build the URI builder.setScheme("http").setHost("localhost").setPort( ServicesTestEnvironment.JETTY_PORT).setPath(WMS_URL_PREFIX).setParameter( "service", @@ -412,7 +1101,7 @@ private static BufferedImage getWMSSingleTile( "request", "GetMap").setParameter("layers", layer).setParameter( "styles", - style == null ? "" : style).setParameter("crs", "EPSG:3857").setParameter( + style == null ? "" : style).setParameter("crs", crsToUse).setParameter( "bbox", String.format( "%.2f, %.2f, %.2f, %.2f", @@ -426,18 +1115,28 @@ private static BufferedImage getWMSSingleTile( String.valueOf(width)).setParameter( "height", String.valueOf(height)); + + // set a parameter if a temporal filter if (temporalFilter) { builder.setParameter( "cql_filter", "TimeStamp DURING 1997-01-01T00:00:00.000Z/1998-01-01T00:00:00.000Z"); } + // Build the http get command final HttpGet command = new HttpGet(builder.build()); + // Create the client and context final Pair clientAndContext = GeoServerIT.createClientAndContext(); + + // Get the client final CloseableHttpClient httpClient = clientAndContext.getLeft(); + + // Get the context final HttpClientContext context = clientAndContext.getRight(); + + // Execute the http command and process the response try { final HttpResponse resp = httpClient.execute(command, context); try (InputStream is = resp.getEntity().getContent()) { @@ -450,6 +1149,7 @@ private static BufferedImage getWMSSingleTile( return image; } } finally { + // Close the http connection httpClient.close(); } } @@ -463,11 +1163,20 @@ public void setUp() { public void cleanup() { geoServerServiceClient.removeFeatureLayer(SimpleIngest.FEATURE_NAME); geoServerServiceClient.removeDataStore(dataStorePluginOptions.getGeoWaveNamespace(), WORKSPACE); + geoServerServiceClient.removeDataStore( + dataStorePluginOptions.getGeoWaveNamespace(), + WORKSPACE2); geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_NO_DIFFERENCE); geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_MINOR_SUBSAMPLE); geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_MAJOR_SUBSAMPLE); geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_DISTRIBUTED_RENDER); + geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP); + geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_AGGR); + geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_AGGR); + geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_CNT_STATS); + geoServerServiceClient.removeStyle(ServicesTestEnvironment.TEST_STYLE_NAME_HEATMAP_SUM_STATS); geoServerServiceClient.removeWorkspace(WORKSPACE); + geoServerServiceClient.removeWorkspace(WORKSPACE2); } @Override diff --git a/test/src/test/java/org/locationtech/geowave/test/services/GeoWaveHeatMapProcessIT.java b/test/src/test/java/org/locationtech/geowave/test/services/GeoWaveHeatMapProcessIT.java new file mode 100644 index 00000000000..e67a0adfa23 --- /dev/null +++ b/test/src/test/java/org/locationtech/geowave/test/services/GeoWaveHeatMapProcessIT.java @@ -0,0 +1,131 @@ +/** + * Copyright (c) 2013-2020 Contributors to the Eclipse Foundation + * + *

See the NOTICE file distributed with this work for additional information regarding copyright + * ownership. All rights reserved. This program and the accompanying materials are made available + * under the terms of the Apache License, Version 2.0 which accompanies this distribution and is + * available at http://www.apache.org/licenses/LICENSE-2.0.txt + */ +package org.locationtech.geowave.test.services; + +import static org.junit.Assert.assertTrue; +import java.awt.geom.Point2D; +import org.geotools.coverage.grid.GridCoverage2D; +import org.geotools.data.simple.SimpleFeatureCollection; +import org.geotools.feature.DefaultFeatureCollection; +import org.geotools.feature.simple.SimpleFeatureBuilder; +import org.geotools.feature.simple.SimpleFeatureTypeBuilder; +import org.geotools.geometry.jts.ReferencedEnvelope; +import org.geotools.referencing.crs.DefaultGeographicCRS; +import org.junit.Test; +import org.locationtech.geowave.adapter.vector.plugin.GeoWaveHeatMapProcess; +import org.locationtech.jts.geom.Coordinate; +import org.locationtech.jts.geom.Geometry; +import org.locationtech.jts.geom.GeometryFactory; +import org.locationtech.jts.geom.MultiPoint; +import org.locationtech.jts.geom.impl.PackedCoordinateSequenceFactory; +import org.opengis.feature.simple.SimpleFeatureType; +import org.opengis.util.ProgressListener; + +public class GeoWaveHeatMapProcessIT { + + /** + * A test of a simple surface, validating that the process can be invoked and return a reasonable + * result in a simple situation. + * + *

Test includes data which lies outside the heatmap buffer area, to check that it is filtered + * correctly (i.e. does not cause out-of-range errors, and does not affect generated surface). + * + * @author M. Zagorski + * @apiNode Note: based on the GeoTools version of HeatmapProcess integration test by Martin Davis + * - OpenGeo. + * @apiNote Date: 3-25-2022
+ * + * @apiNote Changelog:
+ * + * + */ + @Test + public void testSimpleSurface() { + + ReferencedEnvelope bounds = new ReferencedEnvelope(0, 10, 0, 10, DefaultGeographicCRS.WGS84); + Coordinate[] data = + new Coordinate[] { + new Coordinate(4, 4), + new Coordinate(4, 6), + // include a coordinate outside the heatmap buffer bounds, to ensure it is + // filtered correctly + new Coordinate(100, 100)}; + SimpleFeatureCollection fc = createPoints(data, bounds); + + ProgressListener monitor = null; + + // HeatmapProcess process = new HeatmapProcess(); // changed this to the GeoWaveHeatMap + // GeoWaveHeatMap process = new GeoWaveHeatMap(); //Baseline tests pass + GeoWaveHeatMapProcess process = new GeoWaveHeatMapProcess(); // Baseline tests pass + + GridCoverage2D cov = + process.execute( + fc, // data + 20, // radius + null, // weightAttr + 1, // pixelsPerCell + bounds, // outputEnv + 100, // outputWidth + 100, // outputHeight + "CNT_AGGR", // queryType + false, // createStats + true, // useSpatialBinning + monitor // monitor) + ); + + // following tests are checking for an appropriate shape for the surface + + float center1 = coverageValue(cov, 4, 4); + float center2 = coverageValue(cov, 4, 6); + float midway = coverageValue(cov, 4, 5); + float far = coverageValue(cov, 9, 9); + + // peaks are roughly equal + float peakDiff = Math.abs(center1 - center2); + assert (peakDiff < center1 / 10); + + // dip between peaks + assertTrue(midway > center1 / 2); + + // surface is flat far away + assertTrue(far < center1 / 1000); + } + + private float coverageValue(GridCoverage2D cov, double x, double y) { + + float[] covVal = new float[1]; + Point2D worldPos = new Point2D.Double(x, y); + cov.evaluate(worldPos, covVal); + return covVal[0]; + } + + private SimpleFeatureCollection createPoints(Coordinate[] pts, ReferencedEnvelope bounds) { + + SimpleFeatureTypeBuilder tb = new SimpleFeatureTypeBuilder(); + tb.setName("data"); + tb.setCRS(bounds.getCoordinateReferenceSystem()); + tb.add("shape", MultiPoint.class); + tb.add("value", Double.class); + + SimpleFeatureType type = tb.buildFeatureType(); + SimpleFeatureBuilder fb = new SimpleFeatureBuilder(type); + DefaultFeatureCollection fc = new DefaultFeatureCollection(); + + GeometryFactory factory = new GeometryFactory(new PackedCoordinateSequenceFactory()); + + for (Coordinate p : pts) { + Geometry point = factory.createPoint(p); + fb.add(point); + fb.add(p.getZ()); + fc.add(fb.buildFeature(null)); + } + + return fc; + } +} diff --git a/test/src/test/resources/sld/HeatMap-cnt-aggr.sld b/test/src/test/resources/sld/HeatMap-cnt-aggr.sld new file mode 100644 index 00000000000..9a3c6d6f1c8 --- /dev/null +++ b/test/src/test/resources/sld/HeatMap-cnt-aggr.sld @@ -0,0 +1,86 @@ + + + + HeatMap-cnt-aggr + + HeatMap-cnt-aggr + A heatmap surface showing a specified density for count aggregations using spatial binning. + + + + + data + + + weightAttr + SIZE + + + radiusPixels + + radius + 100 + + + + pixelsPerCell + 10 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + queryType + CNT_AGGR + + + createStats + false + + + useSpatialBinning + true + + + + + + + + the_geom + 0.6 + + + + + + + + + + + + + diff --git a/test/src/test/resources/sld/HeatMap-cnt-stats.sld b/test/src/test/resources/sld/HeatMap-cnt-stats.sld new file mode 100644 index 00000000000..f5c613449bf --- /dev/null +++ b/test/src/test/resources/sld/HeatMap-cnt-stats.sld @@ -0,0 +1,86 @@ + + + + HeatMap-cnt-stats + + HeatMap-cnt-stats + A heatmap surface showing a specified density for count statistics using spatial binning. + + + + + data + + + weightAttr + SIZE + + + radiusPixels + + radius + 100 + + + + pixelsPerCell + 10 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + queryType + CNT_STATS + + + createStats + true + + + useSpatialBinning + true + + + + + + + + the_geom + 0.6 + + + + + + + + + + + + + diff --git a/test/src/test/resources/sld/HeatMap-no-spatial-binning.sld b/test/src/test/resources/sld/HeatMap-no-spatial-binning.sld new file mode 100644 index 00000000000..f8a7746ba4d --- /dev/null +++ b/test/src/test/resources/sld/HeatMap-no-spatial-binning.sld @@ -0,0 +1,86 @@ + + + + Heatmap + + Heatmap + A heatmap surface showing a specified density + + + + + data + + + weightAttr + SIZE + + + radiusPixels + + radius + 100 + + + + pixelsPerCell + 10 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + queryType + CNT_AGGR + + + createStats + false + + + useSpatialBinning + false + + + + + + + + the_geom + 0.6 + + + + + + + + + + + + + diff --git a/test/src/test/resources/sld/HeatMap-sum-aggr.sld b/test/src/test/resources/sld/HeatMap-sum-aggr.sld new file mode 100644 index 00000000000..e9d0d09208a --- /dev/null +++ b/test/src/test/resources/sld/HeatMap-sum-aggr.sld @@ -0,0 +1,86 @@ + + + + HeatMap-sum-aggr + + HeatMap-sum-aggr + A heatmap surface showing a specified density for field sum aggregations using spatial binning. + + + + + data + + + weightAttr + SIZE + + + radiusPixels + + radius + 100 + + + + pixelsPerCell + 10 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + queryType + SUM_AGGR + + + createStats + false + + + useSpatialBinning + true + + + + + + + + the_geom + 0.6 + + + + + + + + + + + + + diff --git a/test/src/test/resources/sld/HeatMap-sum-stats.sld b/test/src/test/resources/sld/HeatMap-sum-stats.sld new file mode 100644 index 00000000000..f45cd391960 --- /dev/null +++ b/test/src/test/resources/sld/HeatMap-sum-stats.sld @@ -0,0 +1,86 @@ + + + + HeatMap-sum-stats + + HeatMap-sum-stats + A heatmap surface showing a specified density for field sum statistics using spatial binning. + + + + + data + + + weightAttr + SIZE + + + radiusPixels + + radius + 100 + + + + pixelsPerCell + 10 + + + outputBBOX + + wms_bbox + + + + outputWidth + + wms_width + + + + outputHeight + + wms_height + + + + queryType + SUM_STATS + + + createStats + true + + + useSpatialBinning + true + + + + + + + + the_geom + 0.6 + + + + + + + + + + + + + diff --git a/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-oraclejdk.gif b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-oraclejdk.gif new file mode 100644 index 00000000000..20808f78df5 Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom-oraclejdk.gif b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom-oraclejdk.gif new file mode 100644 index 00000000000..5910e68819c Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom.gif b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom.gif new file mode 100644 index 00000000000..5910e68819c Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84-zoom.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84.gif b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84.gif new file mode 100644 index 00000000000..20808f78df5 Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-cnt-aggr-wgs84.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84-oraclejdk.gif b/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84-oraclejdk.gif new file mode 100644 index 00000000000..20808f78df5 Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84.gif b/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84.gif new file mode 100644 index 00000000000..20808f78df5 Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-no-spat-bin-wgs84.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom-oraclejdk.gif b/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom-oraclejdk.gif new file mode 100644 index 00000000000..5910e68819c Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom.gif b/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom.gif new file mode 100644 index 00000000000..5910e68819c Binary files /dev/null and b/test/src/test/resources/wms/W-wms-heatmap-sum-aggr-wgs84-zoom.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-oraclejdk.gif new file mode 100644 index 00000000000..9bb2739c88e Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom-oraclejdk.gif new file mode 100644 index 00000000000..e7d8cbe8d38 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom.gif new file mode 100644 index 00000000000..e7d8cbe8d38 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr-zoom.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr.gif new file mode 100644 index 00000000000..9bb2739c88e Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-aggr.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-stats-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-stats-oraclejdk.gif new file mode 100644 index 00000000000..6d5b3f8b3c7 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-stats-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-cnt-stats.gif b/test/src/test/resources/wms/X-wms-heatmap-cnt-stats.gif new file mode 100644 index 00000000000..6d5b3f8b3c7 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-cnt-stats.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin-oraclejdk.gif new file mode 100644 index 00000000000..ed52de4be31 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin.gif b/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin.gif new file mode 100644 index 00000000000..ed52de4be31 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-no-spat-bin.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-oraclejdk.gif new file mode 100644 index 00000000000..9bb2739c88e Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom-oraclejdk.gif new file mode 100644 index 00000000000..e7d8cbe8d38 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom.gif new file mode 100644 index 00000000000..e7d8cbe8d38 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr-zoom.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-aggr.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr.gif new file mode 100644 index 00000000000..9bb2739c88e Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-aggr.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-stats-oraclejdk.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-stats-oraclejdk.gif new file mode 100644 index 00000000000..6d5b3f8b3c7 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-stats-oraclejdk.gif differ diff --git a/test/src/test/resources/wms/X-wms-heatmap-sum-stats.gif b/test/src/test/resources/wms/X-wms-heatmap-sum-stats.gif new file mode 100644 index 00000000000..6d5b3f8b3c7 Binary files /dev/null and b/test/src/test/resources/wms/X-wms-heatmap-sum-stats.gif differ