Heatmap with spatial binning - aggregations and statistics

analytics/api/src/main/java/org/locationtech/geowave/analytic/kryo/GeometrySerializer.java

@@ -0,0 +1,34 @@
+package org.locationtech.geowave.analytic.kryo;
+
+import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.io.ParseException;
+import org.locationtech.jts.io.WKBReader;
+import org.locationtech.jts.io.WKBWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+import com.esotericsoftware.kryo.Kryo;
+import com.esotericsoftware.kryo.Serializer;
+import com.esotericsoftware.kryo.io.Input;
+import com.esotericsoftware.kryo.io.Output;
+
+public class GeometrySerializer extends Serializer<Geometry> {
+  static final Logger LOGGER = LoggerFactory.getLogger(GeometrySerializer.class);
+
+  @Override
+  public Geometry read(final Kryo arg0, final Input arg1, final Class<Geometry> arg2) {
+    final byte[] data = arg1.readBytes(arg1.readInt());
+    try {
+      return new WKBReader().read(data);
+    } catch (final ParseException e) {
+      LOGGER.warn("Unable to deserialize geometry", e);
+    }
+    return null;
+  }
+
+  @Override
+  public void write(final Kryo arg0, final Output arg1, final Geometry arg2) {
+    final byte[] data = new WKBWriter().write(arg2);
+    arg1.writeInt(data.length);
+    arg1.write(data);
+  }
+}

analytics/spark/src/main/java/org/locationtech/geowave/analytic/spark/GeoWaveRegistrator.java

@@ -12,12 +12,19 @@
 import org.geotools.feature.simple.SimpleFeatureImpl;
 import org.locationtech.geowave.adapter.raster.adapter.GridCoverageWritable;
 import org.locationtech.geowave.analytic.kryo.FeatureSerializer;
+import org.locationtech.geowave.analytic.kryo.GeometrySerializer;
 import org.locationtech.geowave.analytic.kryo.GridCoverageWritableSerializer;
 import org.locationtech.geowave.analytic.kryo.PersistableSerializer;
 import org.locationtech.geowave.core.index.ByteArray;
 import org.locationtech.geowave.core.index.persist.PersistableFactory;
 import org.locationtech.geowave.mapreduce.input.GeoWaveInputKey;
 import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.GeometryCollection;
+import org.locationtech.jts.geom.MultiLineString;
+import org.locationtech.jts.geom.MultiPoint;
+import org.locationtech.jts.geom.MultiPolygon;
+import org.locationtech.jts.geom.Point;
+import org.locationtech.jts.geom.Polygon;
 import org.locationtech.jts.geom.prep.PreparedGeometry;
 import com.esotericsoftware.kryo.Kryo;
 
@@ -29,13 +36,20 @@ public void registerClasses(final Kryo kryo) {
     final FeatureSerializer simpleFeatureSerializer = new FeatureSerializer();
     final GridCoverageWritableSerializer gcwSerializer = new GridCoverageWritableSerializer();
     final PersistableSerializer persistSerializer = new PersistableSerializer();
+    final GeometrySerializer geometrySerializer = new GeometrySerializer();
 
     PersistableFactory.getInstance().getClassIdMapping().entrySet().forEach(
         e -> kryo.register(e.getKey(), persistSerializer));
 
     kryo.register(GeoWaveRDD.class);
     kryo.register(GeoWaveIndexedRDD.class);
-    kryo.register(Geometry.class);
+    kryo.register(Geometry.class, geometrySerializer);
+    kryo.register(Point.class, geometrySerializer);
+    kryo.register(MultiLineString.class, geometrySerializer);
+    kryo.register(Polygon.class, geometrySerializer);
+    kryo.register(MultiPolygon.class, geometrySerializer);
+    kryo.register(MultiPoint.class, geometrySerializer);
+    kryo.register(GeometryCollection.class, geometrySerializer);
     kryo.register(PreparedGeometry.class);
     kryo.register(ByteArray.class);
     kryo.register(GeoWaveInputKey.class);

core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/ExplainCommand.java

@@ -170,9 +170,11 @@ public static StringBuilder explainMainParameter(final JCommander commander) {
       }
 
       final boolean assigned = mainParameter.isAssigned();
+      System.out.println("ASSIGNED: " + assigned);
       builder.append("Specified: ");
       final List<String> mP =
           (List<String>) mainParameter.getParameterized().get(mainParameter.getObject());
+      System.out.println("MP: " + mP);
       if (!assigned || (mP.size() == 0)) {
         builder.append("<none specified>");
       } else {

core/cli/src/main/java/org/locationtech/geowave/core/cli/operations/GeoWaveTopLevelSection.java

@@ -9,13 +9,11 @@
 package org.locationtech.geowave.core.cli.operations;
 
 import org.apache.logging.log4j.core.appender.ConsoleAppender;
-import org.apache.logging.log4j.core.config.Configuration;
 import org.apache.logging.log4j.core.config.Configurator;
 import org.apache.logging.log4j.Level;
 import org.apache.logging.log4j.core.Logger;
 import org.apache.logging.log4j.LogManager;
 import org.apache.logging.log4j.core.layout.PatternLayout;
-import org.apache.logging.log4j.core.layout.PatternLayout.Builder;
 import org.locationtech.geowave.core.cli.VersionUtils;
 import org.locationtech.geowave.core.cli.annotations.GeowaveOperation;
 import org.locationtech.geowave.core.cli.api.DefaultOperation;

core/geotime/src/main/java/org/locationtech/geowave/core/geotime/binning/SpatialBinningType.java

@@ -8,29 +8,113 @@
  */
 package org.locationtech.geowave.core.geotime.binning;
 
+import org.geotools.geometry.jts.JTS;
+import org.geotools.referencing.CRS;
 import org.locationtech.geowave.core.index.ByteArray;
 import org.locationtech.geowave.core.store.api.BinConstraints.ByteArrayConstraints;
 import org.locationtech.jts.geom.Geometry;
+import org.opengis.geometry.MismatchedDimensionException;
+import org.opengis.referencing.FactoryException;
+import org.opengis.referencing.NoSuchAuthorityCodeException;
+import org.opengis.referencing.crs.CoordinateReferenceSystem;
+import org.opengis.referencing.operation.MathTransform;
+import org.opengis.referencing.operation.TransformException;
 
 public enum SpatialBinningType implements SpatialBinningHelper {
   H3(new H3BinningHelper()), S2(new S2BinningHelper()), GEOHASH(new GeohashBinningHelper());
 
   private SpatialBinningHelper helperDelegate;
 
+  public static int WGS84_SRID = 4326;
+  public static String WGS84_SRID_EPSG = "EPSG:4326";
+  public static String WEB_MERCATOR = "EPSG:3857";
+
   private SpatialBinningType(final SpatialBinningHelper helperDelegate) {
     this.helperDelegate = helperDelegate;
   }
 
+  /**
+   * Converts a JTS geometry to WGS84 CRS.
+   * 
+   * @param geometry {Geometry} The input geometry to be processed.
+   * @return {Geometry} Returns the JTS geometry in WGS84 CRS.
+   */
+  public static Geometry convertToWGS84(Geometry geometry) {
+
+    // Get the source CRS from the user data that is set in OptimizedSimpleFeatureBuilder.java
+    CoordinateReferenceSystem sourceCRS = (CoordinateReferenceSystem) geometry.getUserData();
+    // System.out.println("SBT - SOURCE CRS is null? " + (sourceCRS == null));
+
+    // if (sourceCRS == null) {
+    // try {
+    // sourceCRS = CRS.decode(WEB_MERCATOR);
+    // } catch (NoSuchAuthorityCodeException e) {
+    // e.printStackTrace();
+    // } catch (FactoryException e) {
+    // e.printStackTrace();
+    // }
+    // }
+
+    MathTransform transform;
+    Geometry targetGeometry = null;
+
+    if (sourceCRS != null) {
+      // Only proceed if CRS is not WGS84
+      Boolean isWGS84 = sourceCRS.getName().getCode().equals("WGS 84");
+
+      if (!isWGS84) {
+        try {
+          // Decode the target CRS of "EPSG:4326"
+          CoordinateReferenceSystem targetCRS = CRS.decode(WGS84_SRID_EPSG);
+
+          // Get the transform from source CRS to target CRS with leniency
+          transform = CRS.findMathTransform(sourceCRS, targetCRS, true);
+          try {
+            // Transform the JTS geometry
+            targetGeometry = JTS.transform(geometry, transform);
+
+            // Set the SRID, although this is not necessary
+            targetGeometry.setSRID(WGS84_SRID);
+          } catch (MismatchedDimensionException | TransformException e) {
+            e.printStackTrace();
+          }
+        } catch (FactoryException e) {
+          e.printStackTrace();
+        }
+      }
+    }
+
+    return targetGeometry != null ? targetGeometry : geometry;
+  }
+
+  /**
+   * Gets the spatial bins. Note: Spatial binning aggregations call this (runs on each individual
+   * SimpleFeature).
+   * 
+   * @param geometry {Geometry} The input geometry to be processed.
+   * @param precision {Integer} The spatial binning precision.
+   * @return {ByteArray[]} Returns a ByteArray of spatial bins.
+   */
   @Override
   public ByteArray[] getSpatialBins(final Geometry geometry, final int precision) {
-    // TODO if geometry is not WGS84 we need to transform it
-    return helperDelegate.getSpatialBins(geometry, precision);
+    Geometry targetGeometry = convertToWGS84(geometry);
+
+    return helperDelegate.getSpatialBins(targetGeometry, precision);
   }
 
+  /**
+   * Gets the geometry constraints. Note: Spatial binning statistics call this - runs once on whole
+   * extent.
+   * 
+   * @param geom {Geometry} The input geometry to be processed.
+   * @param precision {Integer} The spatial binning precision.
+   * @return {ByteArrayConstraints} Returns a ByteArrayConstraints of geometry constraints.
+   */
   @Override
   public ByteArrayConstraints getGeometryConstraints(final Geometry geom, final int precision) {
-    // TODO if geometry is not WGS84 we need to transform it
-    return helperDelegate.getGeometryConstraints(geom, precision);
+    Geometry targetGeometry = convertToWGS84(geom);
+
+    return helperDelegate.getGeometryConstraints(targetGeometry, precision);
   }
 
 

core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/query/aggregate/SpatialBinningStrategy.java

@@ -101,6 +101,7 @@ public <I> ByteArray[] getBins(
     }
     if (ComplexGeometryBinningOption.USE_CENTROID_ONLY.equals(complexGeometryBinning)) {
       final Point centroid = geometry.getCentroid();
+      centroid.setUserData(geometry.getUserData());
       return type.getSpatialBins(centroid, precision);
     }
     return type.getSpatialBins(geometry, precision);

core/geotime/src/main/java/org/locationtech/geowave/core/geotime/store/statistics/binning/SpatialFieldValueBinningStrategy.java

@@ -23,6 +23,8 @@
 import org.locationtech.geowave.core.store.statistics.binning.FieldValueBinningStrategy;
 import org.locationtech.jts.geom.Envelope;
 import org.locationtech.jts.geom.Geometry;
+import org.locationtech.jts.geom.Point;
+import org.opengis.referencing.crs.CoordinateReferenceSystem;
 import com.beust.jcommander.IStringConverter;
 import com.beust.jcommander.Parameter;
 import com.beust.jcommander.ParameterException;
@@ -128,7 +130,12 @@ public Class<?>[] supportedConstraintClasses() {
   private ByteArray[] getSpatialBinsFromObj(final Object value) {
     if (value instanceof Geometry) {
       if (ComplexGeometryBinningOption.USE_CENTROID_ONLY.equals(complexGeometry)) {
-        return getSpatialBins(((Geometry) value).getCentroid());
+
+        Point centroid = ((Geometry) value).getCentroid();
+        centroid.setUserData(((Geometry) value).getUserData());
+
+        return getSpatialBins(centroid);
+        // return getSpatialBins(((Geometry) value).getCentroid());
       }
       return getSpatialBins((Geometry) value);
     }

core/store/src/main/java/org/locationtech/geowave/core/store/cli/stats/StatsCommandLineOptions.java

@@ -141,7 +141,7 @@ public List<Statistic<? extends StatisticValue<?>>> resolveMatchingStatistics(
         }
         final DataTypeAdapter<?> adapter = dataStore.getType(typeName);
         if (adapter == null) {
-          throw new ParameterException("Unable to find an type named: " + typeName);
+          throw new ParameterException("Unable to find a type named: " + typeName);
         }
         try (CloseableIterator<? extends Statistic<? extends StatisticValue<?>>> stats =
             statsStore.getDataTypeStatistics(adapter, statisticType, tag)) {
@@ -154,7 +154,7 @@ public List<Statistic<? extends StatisticValue<?>>> resolveMatchingStatistics(
         }
         final DataTypeAdapter<?> adapter = dataStore.getType(typeName);
         if (adapter == null) {
-          throw new ParameterException("Unable to find an type named: " + typeName);
+          throw new ParameterException("Unable to find a type named: " + typeName);
         }
         if (fieldName == null) {
           throw new ParameterException(

examples/java-api/src/main/java/org/locationtech/geowave/examples/ingest/SimpleIngest.java

@@ -40,7 +40,6 @@ public static void main(final String[] args) {
         DataStoreFactory.createDataStore(new MemoryRequiredOptions());
 
     si.writeExampleData(geowaveDataStore);
-    System.out.println("Finished ingesting data");
   }
 
   /** * Here we will change the ingest mechanism to use a producer/consumer pattern */
@@ -65,7 +64,7 @@ protected void writeExampleData(final DataStore geowaveDataStore) {
     try (Writer<SimpleFeature> indexWriter =
         geowaveDataStore.createWriter(dataTypeAdapter.getTypeName())) {
       // build a grid of points across the globe at each whole
-      // lattitude/longitude intersection
+      // latitude/longitude intersection
 
       for (final SimpleFeature sft : getGriddedFeatures(pointBuilder, 1000)) {
         indexWriter.write(sft);
@@ -204,6 +203,9 @@ public static SimpleFeatureType createPointFeatureType() {
     builder.add(ab.binding(String.class).nillable(true).buildDescriptor("TrajectoryID"));
     builder.add(ab.binding(String.class).nillable(true).buildDescriptor("Comment"));
 
+    // Create a SIZE field for sum aggregation and statistics tests
+    builder.add(ab.binding(Double.class).nillable(true).buildDescriptor("SIZE"));
+
     return builder.buildFeatureType();
   }
 }

extensions/adapters/vector/src/main/java/org/geotools/feature/simple/OptimizedSimpleFeatureBuilder.java

@@ -8,7 +8,9 @@
  */
 package org.geotools.feature.simple;
 
+import org.locationtech.jts.geom.Geometry;
 import org.opengis.feature.simple.SimpleFeatureType;
+import org.opengis.referencing.crs.CoordinateReferenceSystem;
 
 /**
  * Variation of SimpleFeatureBuilder that skips object conversion, since GeoWave handles that
@@ -26,6 +28,12 @@ public void set(int index, Object value) {
       throw new ArrayIndexOutOfBoundsException(
           "Can handle " + values.length + " attributes only, index is " + index);
 
+    // Add the CRS of the geometry to the user data
+    if (value instanceof Geometry) {
+      ((Geometry) value).setUserData(getFeatureType().getCoordinateReferenceSystem());
+    }
+
     values[index] = value;
+
   }
 }

extensions/adapters/vector/src/main/java/org/locationtech/geowave/adapter/vector/FeatureDataAdapter.java

@@ -325,6 +325,14 @@ public SimpleFeature fromWritable(final FeatureWritable writable) {
 
   @Override
   public Object getFieldValue(final SimpleFeature entry, final String fieldName) {
+
+    // Object fieldValue = entry.getAttribute(fieldName);
+    // if ((fieldValue instanceof Geometry)
+    // && !(((Geometry) fieldValue).getUserData() instanceof CoordinateReferenceSystem)) {
+    // ((Geometry) fieldValue).setUserData(getFeatureType().getCoordinateReferenceSystem());
+    // }
+
+
     return entry.getAttribute(fieldName);
   }