From 05a8fbf1c0594ef1a0846258b5d7857110e89daa Mon Sep 17 00:00:00 2001 From: Jeroen Dries Date: Wed, 29 Jun 2022 15:57:22 +0200 Subject: [PATCH 1/6] next gwc-geotrellis --- pom.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pom.xml b/pom.xml index 91b61b6a9..998eb3119 100644 --- a/pom.xml +++ b/pom.xml @@ -15,7 +15,7 @@ 2.11.12 3.5.2 1.10 - 0.15.0-SNAPSHOT + 0.16.1 UTF-8 From e2b930f53818f9d1a1b84998a46f4433eb9d1885 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 4 Nov 2022 06:37:31 +0100 Subject: [PATCH 2/6] adapt to OSCARS --- .../main/scala/org/openeo/geotrellisseeder/TileSeeder.scala | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/geotrellis-seeder/src/main/scala/org/openeo/geotrellisseeder/TileSeeder.scala b/geotrellis-seeder/src/main/scala/org/openeo/geotrellisseeder/TileSeeder.scala index 1070d9393..8075deaf0 100644 --- a/geotrellis-seeder/src/main/scala/org/openeo/geotrellisseeder/TileSeeder.scala +++ b/geotrellis-seeder/src/main/scala/org/openeo/geotrellisseeder/TileSeeder.scala @@ -71,7 +71,7 @@ case class TileSeeder(zoomLevel: Int, verbose: Boolean, partitions: Option[Int] val openSearchClient = OpenSearchClient(new URL(oscarsEndpoint.get)) val products = if (date.isDefined) { - val attributeValues = Map(Seq(("productType", productType), ("accessedFrom", "S3")) ++ oscarsSearchFilters.getOrElse(Map()).toSeq: _*) + val attributeValues = Map(Seq(("productType", productType), ("accessedFrom", "S3-private")) ++ oscarsSearchFilters.getOrElse(Map()).toSeq: _*) openSearchClient.getProducts(oscarsCollection.get, date.map(d => (d, d)).get, ProjectedExtent(LatLng.worldExtent, LatLng), attributeValues, "", "") } else { //Divide world extent to get less then 10000 products per request @@ -85,10 +85,10 @@ case class TileSeeder(zoomLevel: Int, verbose: Boolean, partitions: Option[Int] val paths = products.flatMap(_.links.filter(_.title.contains(productType)).map(_.href.toString)) - val s3Path = """(s3://)(.*:)(.*)""".r.unanchored + val s3Path = """(s3://)(.*)""".r.unanchored val s3Paths = paths.flatMap { - case s3Path(prefix, _, key) => Some(s"$prefix$key") + case s3Path(prefix, key) => Some(s"$prefix$key") case _ => None } From eb6311612ff24f1679c2699628e403c73b7cd900 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Fri, 4 Nov 2022 15:16:39 +0100 Subject: [PATCH 3/6] ported testMaskSclDilationOnS2TileEdge fix to master original commit: 418980a33bc9e310676b6b3911a2e272ea6e572c --- .../geotrellis/layers/Sentinel2FileLayerProviderTest.scala | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/Sentinel2FileLayerProviderTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/Sentinel2FileLayerProviderTest.scala index b27492e26..09f09162b 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/Sentinel2FileLayerProviderTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/layers/Sentinel2FileLayerProviderTest.scala @@ -363,7 +363,7 @@ class Sentinel2FileLayerProviderTest extends RasterMatchers { val referenceTile = GeoTiffRasterSource("https://artifactory.vgt.vito.be/testdata-public/dilation_masked.tif").read().get val actualTile = GeoTiffRasterSource("/tmp/masked.tif").read().get - assertRastersEqual(referenceTile,actualTile) + assertRastersEqual(referenceTile,actualTile,160.0) } @Test From 1bc23d20fc621898d7d94ccd56e3f407f24b2988 Mon Sep 17 00:00:00 2001 From: Jan Van den bosch Date: Wed, 24 Aug 2022 16:24:59 +0200 Subject: [PATCH 4/6] update tests after cleanup of Sentinel-2 data java.lang.IllegalArgumentException: Could not find data for your load_collection request with catalog ID urn:eop:VITO:TERRASCOPE_S2_TOC_V2. The catalog query had id and returned 0 results. at org.openeo.geotrellis.file.Sentinel2PyramidFactoryTest.createLayerForDate(Sentinel2PyramidFactoryTest.scala:179) at org.openeo.geotrellis.file.Sentinel2PyramidFactoryTest.testStatsFromNativeUTM(Sentinel2PyramidFactoryTest.scala:147) --- .../org/openeo/geotrellis/layers/FileLayerProvider.scala | 2 +- openeo-geotrellis/src/test/resources/log4j.properties | 1 + .../geotrellis/file/Sentinel2PyramidFactoryTest.scala | 6 +++--- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala index 45190fbc2..8977b2288 100644 --- a/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala +++ b/openeo-geotrellis/src/main/scala/org/openeo/geotrellis/layers/FileLayerProvider.scala @@ -566,7 +566,7 @@ class FileLayerProvider(openSearch: OpenSearchClient, openSearchCollectionId: St // TODO: these geotiffs overlap a bit so for a bbox near the edge, not one but two or even four geotiffs are taken // into account; it's more efficient to filter out the redundant ones - if (overlappingRasterSources.isEmpty) throw new IllegalArgumentException(s"Could not find data for your load_collection request with catalog ID ${openSearchCollectionId}. The catalog query had id ${correlationId} and returned ${overlappingFeatures.size} results.") + if (overlappingRasterSources.isEmpty) throw new IllegalArgumentException(s"""Could not find data for your load_collection request with catalog ID "$openSearchCollectionId". The catalog query had correlation ID "$correlationId" and returned ${overlappingFeatures.size} results.""") overlappingRasterSources diff --git a/openeo-geotrellis/src/test/resources/log4j.properties b/openeo-geotrellis/src/test/resources/log4j.properties index b11f3aaaa..624152edb 100644 --- a/openeo-geotrellis/src/test/resources/log4j.properties +++ b/openeo-geotrellis/src/test/resources/log4j.properties @@ -4,3 +4,4 @@ log4j.appender.stdout.layout=org.apache.log4j.PatternLayout log4j.appender.stdout.layout.ConversionPattern=%d{dd MMM HH:mm:ss} %p [%c{2}] - %m%n log4j.logger.org.openeo.geotrellis.ComputeStatsGeotrellisAdapter=DEBUG +log4j.logger.org.openeo.opensearch.OpenSearchClient=DEBUG diff --git a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/file/Sentinel2PyramidFactoryTest.scala b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/file/Sentinel2PyramidFactoryTest.scala index 9f4115290..d047c3828 100644 --- a/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/file/Sentinel2PyramidFactoryTest.scala +++ b/openeo-geotrellis/src/test/scala/org/openeo/geotrellis/file/Sentinel2PyramidFactoryTest.scala @@ -88,7 +88,7 @@ class Sentinel2PyramidFactoryTest { @Test def testStatsFromPyramid(): Unit = { val bbox = ProjectedExtent(Extent(373863.50, 5212258.22, 378241.73, 5216244.73), CRS.fromEpsgCode(32631)) - val localDate = LocalDate.of(2019, 10, 11) + val localDate = LocalDate.of(2022, 8, 1) val spatialLayer = createLayerForDate(bbox, localDate) checkStatsResult("testStatsFromPyramid", bbox, spatialLayer) @@ -97,7 +97,7 @@ class Sentinel2PyramidFactoryTest { @Test def testStatsFromNativeUTM(): Unit = { val bbox = ProjectedExtent(Extent(373863.50, 5212258.22, 378241.73, 5216244.73), CRS.fromEpsgCode(32631)) - val localDate = LocalDate.of(2019, 10, 11) + val localDate = LocalDate.of(2022, 8, 1) val spatialLayer = createLayerForDate(bbox, localDate,pyramid = false) checkStatsResult("testStatsFromNativeUTM", bbox, spatialLayer) @@ -112,7 +112,7 @@ class Sentinel2PyramidFactoryTest { case Summary(values) => values.head.mean } - val qgisZonalStaticsPluginResult = 4.510951226022072 + val qgisZonalStaticsPluginResult = 8.874901011878574 assertEquals(qgisZonalStaticsPluginResult, singleBandMean, 0.005) } From c0ea18938058c7b523af3fdbac15fedf5bed4241 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 24 Feb 2023 15:04:03 +0100 Subject: [PATCH 5/6] Add sc.stop() to fix local running tests. --- .../TestSameStartEndDate.scala | 23 ++++++++++--------- 1 file changed, 12 insertions(+), 11 deletions(-) diff --git a/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala b/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala index 2600f647e..6f523d3a2 100644 --- a/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala +++ b/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala @@ -31,19 +31,20 @@ class TestSameStartEndDate { .setAppName("TestSentinelHub") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryoserializer.buffer.max", "1024m")) + try { + val endpoint = "https://services.sentinel-hub.com" + val pyramid = new PyramidFactory(collectionId = null, "sentinel-1-grd", new DefaultCatalogApi(endpoint), + new DefaultProcessApi(endpoint), clientId, clientSecret, rateLimitingGuard = NoRateLimitingGuard, maxSpatialResolution = CellSize(10, 10)) + .pyramid_seq(extent, bbox_srs, from, to, bandNames, metadata_properties = Collections.emptyMap[String, Any]) - val endpoint = "https://services.sentinel-hub.com" - val pyramid = new PyramidFactory(collectionId = null, "sentinel-1-grd", new DefaultCatalogApi(endpoint), - new DefaultProcessApi(endpoint), clientId, clientSecret, rateLimitingGuard = NoRateLimitingGuard, maxSpatialResolution = CellSize(10,10)) - .pyramid_seq(extent, bbox_srs, from, to, bandNames, metadata_properties = Collections.emptyMap[String, Any]) + val (_, topLevelRdd) = pyramid.filter { case (zoom, _) => zoom == 14 }.head - val (_, topLevelRdd) = pyramid.filter { case (zoom, _) => zoom == 14 }.head + val results = topLevelRdd.collect() - val results = topLevelRdd.collect() - - for { - (_, multibandTile) <- results - tile <- multibandTile.bands - } assert(tile.isNoDataTile) + for { + (_, multibandTile) <- results + tile <- multibandTile.bands + } assert(tile.isNoDataTile) + } finally sc.stop() } } From 6c4d1c68dc888c53022212217f01c5693adbcf18 Mon Sep 17 00:00:00 2001 From: Emile Sonneveld Date: Fri, 24 Feb 2023 15:45:16 +0100 Subject: [PATCH 6/6] Revert "Add sc.stop() to fix local running tests." This reverts commit c0ea18938058c7b523af3fdbac15fedf5bed4241. --- .../TestSameStartEndDate.scala | 23 +++++++++---------- 1 file changed, 11 insertions(+), 12 deletions(-) diff --git a/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala b/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala index 6f523d3a2..2600f647e 100644 --- a/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala +++ b/geotrellis-sentinelhub/src/test/scala/org/openeo/geotrellissentinelhub/TestSameStartEndDate.scala @@ -31,20 +31,19 @@ class TestSameStartEndDate { .setAppName("TestSentinelHub") .set("spark.serializer", "org.apache.spark.serializer.KryoSerializer") .set("spark.kryoserializer.buffer.max", "1024m")) - try { - val endpoint = "https://services.sentinel-hub.com" - val pyramid = new PyramidFactory(collectionId = null, "sentinel-1-grd", new DefaultCatalogApi(endpoint), - new DefaultProcessApi(endpoint), clientId, clientSecret, rateLimitingGuard = NoRateLimitingGuard, maxSpatialResolution = CellSize(10, 10)) - .pyramid_seq(extent, bbox_srs, from, to, bandNames, metadata_properties = Collections.emptyMap[String, Any]) - val (_, topLevelRdd) = pyramid.filter { case (zoom, _) => zoom == 14 }.head + val endpoint = "https://services.sentinel-hub.com" + val pyramid = new PyramidFactory(collectionId = null, "sentinel-1-grd", new DefaultCatalogApi(endpoint), + new DefaultProcessApi(endpoint), clientId, clientSecret, rateLimitingGuard = NoRateLimitingGuard, maxSpatialResolution = CellSize(10,10)) + .pyramid_seq(extent, bbox_srs, from, to, bandNames, metadata_properties = Collections.emptyMap[String, Any]) - val results = topLevelRdd.collect() + val (_, topLevelRdd) = pyramid.filter { case (zoom, _) => zoom == 14 }.head - for { - (_, multibandTile) <- results - tile <- multibandTile.bands - } assert(tile.isNoDataTile) - } finally sc.stop() + val results = topLevelRdd.collect() + + for { + (_, multibandTile) <- results + tile <- multibandTile.bands + } assert(tile.isNoDataTile) } }