Skip to content

Commit 44a7e04

Browse files
tyrasdtyrasd
authored
Merge pull request #161 from GIScience/avoid-unneccessary-geometry-building
avoid unneccessary geometry building during count/ratio requests
2 parents fc0ec58 + 3d0d93d commit 44a7e04

File tree

4 files changed

+58
-59
lines changed

4 files changed

+58
-59
lines changed

‎CHANGELOG.md

+5
Original file line numberDiff line numberDiff line change
@@ -8,11 +8,16 @@ Changelog
88
* fix wrong thrown exceptions in case of invalid `bpolys` boundary ([#214])
99
* fix not thrown exception in case of `bpolys` and `bcircles` boundaries not lying completely within the underlying data-extract polygon ([#201])
1010

11+
### Performance Improvements
12+
13+
* avoid unneccessary geometry building for `count/ratio` requests ([#161])
14+
1115
### Other Changes
1216

1317
* upgrade to OSHDB version 0.7.0 – note that oshdb database files from the previous version are not compatible with this version anymore, you have to either [redownload](https://downloads.ohsome.org/OSHDB/v0.7/) or [recreate](https://github.com/GIScience/oshdb/blob/0.7/oshdb-etl/README.md) them from scratch ([#222])
1418
* adapt error message for contributions extration endpoint in case of wrong `time` parameter value ([#208])
1519

20+
[#161]: https://github.com/GIScience/ohsome-api/pull/161
1621
[#201]: https://github.com/GIScience/ohsome-api/issues/201
1722
[#208]: https://github.com/GIScience/ohsome-api/issues/208
1823
[#214]: https://github.com/GIScience/ohsome-api/issues/214

‎src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/AggregateRequestExecutor.java

+18-23
Original file line numberDiff line numberDiff line change
@@ -81,32 +81,28 @@ public Response aggregate() throws Exception {
8181
final SortedMap<OSHDBTimestamp, ? extends Number> result;
8282
MapReducer<OSMEntitySnapshot> mapRed = null;
8383
mapRed = inputProcessor.processParameters();
84+
var mapRedGeom = mapRed.map(OSMEntitySnapshot::getGeometry);
8485
switch (requestResource) {
8586
case COUNT:
8687
result = mapRed.aggregateByTimestamp().count();
8788
break;
88-
case AREA:
89-
result = mapRed.aggregateByTimestamp()
90-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> ExecutionUtils
91-
.cacheInUserData(snapshot.getGeometry(), () -> Geo.areaOf(snapshot.getGeometry())));
92-
break;
93-
case LENGTH:
94-
result = mapRed.aggregateByTimestamp()
95-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> ExecutionUtils
96-
.cacheInUserData(snapshot.getGeometry(),
97-
() -> Geo.lengthOf(snapshot.getGeometry())));
98-
break;
9989
case PERIMETER:
100-
result = mapRed.aggregateByTimestamp()
101-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> {
102-
if (snapshot.getGeometry() instanceof Polygonal) {
103-
return ExecutionUtils.cacheInUserData(snapshot.getGeometry(),
104-
() -> Geo.lengthOf(snapshot.getGeometry().getBoundary()));
105-
} else {
90+
result = mapRedGeom.aggregateByTimestamp()
91+
.sum(geom -> {
92+
if (!(geom instanceof Polygonal)) {
10693
return 0.0;
10794
}
95+
return ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
10896
});
10997
break;
98+
case LENGTH:
99+
result = mapRedGeom.aggregateByTimestamp()
100+
.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom)));
101+
break;
102+
case AREA:
103+
result = mapRedGeom.aggregateByTimestamp()
104+
.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom)));
105+
break;
110106
default:
111107
throw new RuntimeException("Unsupported RequestResource type for this processing. "
112108
+ "Only COUNT, LENGTH, PERIMETER, and AREA are permitted here");
@@ -345,7 +341,6 @@ private ElementsResult[] fillElementsResult(SortedMap<OSHDBTimestamp, ? extends
345341
if (boundaryType == BoundaryType.NOBOUNDARY) {
346342
throw new BadRequestException(ExceptionMessages.NO_BOUNDARY);
347343
}
348-
MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, Geometry> preResult;
349344
ArrayList<Geometry> arrGeoms = new ArrayList<>(processingData.getBoundaryList());
350345
@SuppressWarnings("unchecked") // intentionally as check for P on Polygonal is already performed
351346
Map<Integer, P> geoms = IntStream.range(0, arrGeoms.size()).boxed()
@@ -359,22 +354,22 @@ private ElementsResult[] fillElementsResult(SortedMap<OSHDBTimestamp, ? extends
359354
if (filter.isPresent()) {
360355
mapAgg = mapAgg.filter(filter.get());
361356
}
362-
preResult = mapAgg.map(OSMEntitySnapshot::getGeometry);
357+
var mapAggGeom = mapAgg.map(OSMEntitySnapshot::getGeometry);
363358
switch (requestResource) {
364359
case COUNT:
365-
return preResult.count();
360+
return mapAgg.count();
366361
case PERIMETER:
367-
return preResult.sum(geom -> {
362+
return mapAggGeom.sum(geom -> {
368363
if (!(geom instanceof Polygonal)) {
369364
return 0.0;
370365
}
371366
return ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
372367
});
373368
case LENGTH:
374-
return preResult
369+
return mapAggGeom
375370
.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom)));
376371
case AREA:
377-
return preResult.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom)));
372+
return mapAggGeom.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom)));
378373
default:
379374
return null;
380375
}

‎src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ElementsRequestExecutor.java

+15-14
Original file line numberDiff line numberDiff line change
@@ -198,10 +198,10 @@ public static <P extends Geometry & Polygonal> Response aggregateGroupByBoundary
198198
if (filter.isPresent()) {
199199
mapAgg = mapAgg.filter(filter.get());
200200
}
201-
var preResult = mapAgg.map(f -> ExecutionUtils.mapSnapshotToTags(keysInt, valuesInt, f))
202-
.aggregateBy(Pair::getKey, zeroFill).map(Pair::getValue)
203-
.aggregateByTimestamp(OSMEntitySnapshot::getTimestamp).map(OSMEntitySnapshot::getGeometry);
204-
var result = ExecutionUtils.computeNestedResult(requestResource, preResult);
201+
var result = ExecutionUtils.computeNestedResult(requestResource,
202+
mapAgg.map(f -> ExecutionUtils.mapSnapshotToTags(keysInt, valuesInt, f))
203+
.aggregateBy(Pair::getKey, zeroFill).map(Pair::getValue)
204+
.aggregateByTimestamp(OSMEntitySnapshot::getTimestamp));
205205
var groupByResult = OSHDBCombinedIndex.nest(result);
206206
GroupByResult[] resultSet = new GroupByResult[groupByResult.entrySet().size()];
207207
InputProcessingUtils utils = inputProcessor.getUtils();
@@ -683,8 +683,8 @@ public static Response aggregateRatio(RequestResource requestResource,
683683
mapRed = mapRed.filter(combinedFilter);
684684
var preResult = mapRed.aggregateByTimestamp().aggregateBy(snapshot -> {
685685
OSMEntity entity = snapshot.getEntity();
686-
boolean matches1 = filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry());
687-
boolean matches2 = filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
686+
boolean matches1 = filterExpr1.applyOSMGeometry(entity, snapshot::getGeometry);
687+
boolean matches2 = filterExpr2.applyOSMGeometry(entity, snapshot::getGeometry);
688688
if (matches1 && matches2) {
689689
return MatchType.MATCHESBOTH;
690690
} else if (matches1) {
@@ -990,11 +990,11 @@ public static <P extends Geometry & Polygonal> Response aggregateRatioGroupByBou
990990
arrGeoms.stream().collect(Collectors.toMap(arrGeoms::indexOf, geom -> (P) geom));
991991
var mapRed2 = mapRed.aggregateByTimestamp().aggregateByGeometry(geoms);
992992
mapRed2 = mapRed2.filter(combinedFilter);
993-
var preResult =
993+
var mapRed3 =
994994
mapRed2.aggregateBy((SerializableFunction<OSMEntitySnapshot, MatchType>) snapshot -> {
995995
OSMEntity entity = snapshot.getEntity();
996-
boolean matches1 = filterExpr1.applyOSMGeometry(entity, snapshot.getGeometry());
997-
boolean matches2 = filterExpr2.applyOSMGeometry(entity, snapshot.getGeometry());
996+
boolean matches1 = filterExpr1.applyOSMGeometry(entity, snapshot::getGeometry);
997+
boolean matches2 = filterExpr2.applyOSMGeometry(entity, snapshot::getGeometry);
998998
if (matches1 && matches2) {
999999
return MatchType.MATCHESBOTH;
10001000
} else if (matches1) {
@@ -1005,19 +1005,20 @@ public static <P extends Geometry & Polygonal> Response aggregateRatioGroupByBou
10051005
assert false : "MatchType matches none.";
10061006
}
10071007
return MatchType.MATCHESNONE;
1008-
}, EnumSet.allOf(MatchType.class)).map(OSMEntitySnapshot::getGeometry);
1008+
}, EnumSet.allOf(MatchType.class));
1009+
var mapRed3Geom = mapRed3.map(OSMEntitySnapshot::getGeometry);
10091010
SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<OSHDBTimestamp, Integer>, MatchType>, ? extends
10101011
Number> result = null;
10111012
switch (requestResource) {
10121013
case COUNT:
1013-
result = preResult.count();
1014+
result = mapRed3.count();
10141015
break;
10151016
case LENGTH:
10161017
result =
1017-
preResult.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom)));
1018+
mapRed3Geom.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.lengthOf(geom)));
10181019
break;
10191020
case PERIMETER:
1020-
result = preResult.sum(geom -> {
1021+
result = mapRed3Geom.sum(geom -> {
10211022
if (!(geom instanceof Polygonal)) {
10221023
return 0.0;
10231024
}
@@ -1026,7 +1027,7 @@ public static <P extends Geometry & Polygonal> Response aggregateRatioGroupByBou
10261027
break;
10271028
case AREA:
10281029
result =
1029-
preResult.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom)));
1030+
mapRed3Geom.sum(geom -> ExecutionUtils.cacheInUserData(geom, () -> Geo.areaOf(geom)));
10301031
break;
10311032
default:
10321033
break;

‎src/main/lombok/org/heigit/ohsome/ohsomeapi/executor/ExecutionUtils.java

+20-22
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,6 @@
7676
import org.heigit.ohsome.oshdb.osm.OSMType;
7777
import org.heigit.ohsome.oshdb.util.celliterator.ContributionType;
7878
import org.heigit.ohsome.oshdb.util.exceptions.OSHDBKeytablesNotFoundException;
79-
import org.heigit.ohsome.oshdb.util.function.SerializableFunction;
8079
import org.heigit.ohsome.oshdb.util.function.SerializableSupplier;
8180
import org.heigit.ohsome.oshdb.util.geometry.Geo;
8281
import org.heigit.ohsome.oshdb.util.geometry.OSHDBGeometryBuilder;
@@ -443,28 +442,26 @@ public org.wololo.geojson.Feature createOSMFeature(OSMEntity entity, Geometry ge
443442
public static <K extends Comparable<K> & Serializable, V extends Number>
444443
SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V> computeResult(
445444
RequestResource requestResource,
446-
MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, K>, OSMEntitySnapshot> preResult)
445+
MapAggregator<OSHDBCombinedIndex<OSHDBTimestamp, K>, OSMEntitySnapshot> mapAgg)
447446
throws Exception {
447+
var mapAggGeom = mapAgg.map(OSMEntitySnapshot::getGeometry);
448448
switch (requestResource) {
449449
case COUNT:
450-
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) preResult.count();
451-
case LENGTH:
452-
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) preResult
453-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> cacheInUserData(
454-
snapshot.getGeometry(), () -> Geo.lengthOf(snapshot.getGeometry())));
450+
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) mapAgg.count();
455451
case PERIMETER:
456-
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) preResult
457-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> {
458-
if (snapshot.getGeometry() instanceof Polygonal) {
459-
return cacheInUserData(snapshot.getGeometry(),
460-
() -> Geo.lengthOf(snapshot.getGeometry().getBoundary()));
452+
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) mapAggGeom
453+
.sum(geom -> {
454+
if (!(geom instanceof Polygonal)) {
455+
return 0.0;
461456
}
462-
return 0.0;
457+
return cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
463458
});
459+
case LENGTH:
460+
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) mapAggGeom
461+
.sum(geom -> cacheInUserData(geom, () -> Geo.lengthOf(geom)));
464462
case AREA:
465-
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) preResult
466-
.sum((SerializableFunction<OSMEntitySnapshot, Number>) snapshot -> cacheInUserData(
467-
snapshot.getGeometry(), () -> Geo.areaOf(snapshot.getGeometry())));
463+
return (SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V>) mapAggGeom
464+
.sum(geom -> cacheInUserData(geom, () -> Geo.areaOf(geom)));
468465
default:
469466
return null;
470467
}
@@ -479,26 +476,27 @@ SortedMap<OSHDBCombinedIndex<OSHDBTimestamp, K>, V> computeResult(
479476
SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, V>
480477
computeNestedResult(
481478
RequestResource requestResource,
482-
MapAggregator<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, Geometry>
483-
preResult) throws Exception {
479+
MapAggregator<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>,
480+
OSMEntitySnapshot> mapAgg) throws Exception {
481+
var mapAggGeom = mapAgg.map(OSMEntitySnapshot::getGeometry);
484482
switch (requestResource) {
485483
case COUNT:
486484
return (SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, V>)
487-
preResult.count();
485+
mapAgg.count();
488486
case PERIMETER:
489487
return (SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, V>)
490-
preResult.sum(geom -> {
488+
mapAggGeom.sum(geom -> {
491489
if (!(geom instanceof Polygonal)) {
492490
return 0.0;
493491
}
494492
return cacheInUserData(geom, () -> Geo.lengthOf(geom.getBoundary()));
495493
});
496494
case LENGTH:
497495
return (SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, V>)
498-
preResult.sum(geom -> cacheInUserData(geom, () -> Geo.lengthOf(geom)));
496+
mapAggGeom.sum(geom -> cacheInUserData(geom, () -> Geo.lengthOf(geom)));
499497
case AREA:
500498
return (SortedMap<OSHDBCombinedIndex<OSHDBCombinedIndex<Integer, K>, OSHDBTimestamp>, V>)
501-
preResult.sum(geom -> cacheInUserData(geom, () -> Geo.areaOf(geom)));
499+
mapAggGeom.sum(geom -> cacheInUserData(geom, () -> Geo.areaOf(geom)));
502500
default:
503501
return null;
504502
}

0 commit comments

Comments
 (0)