From a6fe09ebc40faec70bca993374b3156acf0f950d Mon Sep 17 00:00:00 2001 From: Andrea Aime Date: Fri, 13 Dec 2024 18:43:18 +0100 Subject: [PATCH] [GEOS-11643] WCS input read limits can be fooled by geotiff reader --- .../org/vfny/geoserver/util/WCSUtils.java | 104 +++++++++++++++++- .../org/vfny/geoserver/util/WCSUtilsTest.java | 44 ++++++++ .../wcs/DefaultWebCoverageService100.java | 29 +++-- .../org/geoserver/wcs/GetCoverageTest.java | 30 +++++ .../wcs/DefaultWebCoverageService111.java | 26 +++-- .../org/geoserver/wcs/GetCoverageTest.java | 26 +++++ .../org/geoserver/wcs2_0/GetCoverage.java | 55 +++++---- .../geoserver/wcs2_0/xml/GetCoverageTest.java | 23 ++++ .../resources/requestGetCoverageSlice.xml | 14 +++ 9 files changed, 311 insertions(+), 40 deletions(-) create mode 100644 src/wcs2_0/src/test/resources/requestGetCoverageSlice.xml diff --git a/src/wcs/src/main/java/org/vfny/geoserver/util/WCSUtils.java b/src/wcs/src/main/java/org/vfny/geoserver/util/WCSUtils.java index a6c9cd092f4..1a22800c73f 100644 --- a/src/wcs/src/main/java/org/vfny/geoserver/util/WCSUtils.java +++ b/src/wcs/src/main/java/org/vfny/geoserver/util/WCSUtils.java @@ -5,6 +5,7 @@ */ package org.vfny.geoserver.util; +import java.awt.Rectangle; import java.awt.image.RenderedImage; import java.awt.image.SampleModel; import java.text.DecimalFormat; @@ -363,6 +364,9 @@ public static void checkOutputLimits( * extra memory usage, just makes it happen sooner) */ public static void checkInputLimits(WCSInfo info, GridCoverage2D coverage) { + // null safety, and read/crop might return null + if (coverage == null) return; + // do we have to check a limit at all? long limit = info.getMaxInputMemory() * 1024; if (limit <= 0) { @@ -370,10 +374,7 @@ public static void checkInputLimits(WCSInfo info, GridCoverage2D coverage) { } // compute the coverage memory usage and compare with limit - long actual = - getCoverageSize( - coverage.getGridGeometry().getGridRange2D(), - coverage.getRenderedImage().getSampleModel()); + long actual = getReadCoverageSize(coverage); if (actual > limit) { throw new WcsException( "This request is trying to read too much data, " @@ -385,6 +386,69 @@ public static void checkInputLimits(WCSInfo info, GridCoverage2D coverage) { } } + /** + * If the image is deferred loaded, and cropped, consider that full tiles need to be read before + * cropping can happen. This is important for images that are tiled but whose form factor is + * very much streatched, so that the tiles are read in full, even if only a small portion of + * them is actually used. This is adopting a simplistic approach, assuming the crop is the last + * operation in the chain, while in general other operations might be involved (but if they are, + * and rescaling/warping is in the mix, then computing the actual read size is going to be + * pretty complicated). + */ + static long getReadCoverageSize(GridCoverage2D coverage) { + RenderedImage ri = coverage.getRenderedImage(); + + GridEnvelope2D gridEnvelope = coverage.getGridGeometry().getGridRange2D(); + // deferred crop? we are just going to read the tiles covering the crop area + if (isDeferredLoaded(ri)) { + RenderedOp op = (RenderedOp) ri; + String operationName = op.getOperationName(); + // "crop" can be implemented both as actual crop, or mosaic + if ("Crop".equals(operationName) + || ("Mosaic".equals(operationName) && op.getNumSources() == 1)) { + gridEnvelope = getCropTilesEnvelope(op); + } + } + return getCoverageSize(gridEnvelope, ri.getSampleModel()); + } + + /** + * Returns the envelope of the tiles covering the image. This helps computing the actual size of + * the image that will be read from the disk, in case the image is tiled and the crop operation + * is the last one in the chain. + */ + private static GridEnvelope2D getCropTilesEnvelope(RenderedOp crop) { + RenderedImage source = (RenderedImage) crop.getSources().get(0); + Rectangle bounds = crop.getBounds(); + int tileXOffset = source.getTileGridXOffset(); + int tileYOffset = source.getTileGridYOffset(); + int tileWidth = source.getTileWidth(); + int tileHeight = source.getTileHeight(); + int tileMinX = snapToTileGrid(bounds.x, tileXOffset, tileWidth); + int tileMinY = snapToTileGrid(bounds.y, tileYOffset, tileHeight); + int tileMaxX = snapToTileGrid(bounds.x + bounds.width, tileXOffset, tileWidth); + int tileMaxY = snapToTileGrid(bounds.y + bounds.height, tileYOffset, tileHeight); + // account the size of the source image, could be smaller than the suggested tile size + int minReadX = Math.max(tileXOffset + tileMinX * tileWidth, source.getMinX()); + int minReadY = Math.max(tileYOffset + tileMinY * tileHeight, source.getMinY()); + int maxReadX = + Math.min( + tileXOffset + (tileMaxX + 1) * tileWidth, + source.getMinX() + source.getWidth()); + int maxReadY = + Math.min( + tileYOffset + (tileMaxY + 1) * tileHeight, + source.getMinY() + source.getHeight()); + GridEnvelope2D gridEnvelope = + new GridEnvelope2D( + minReadX, minReadY, (maxReadX - minReadX), (maxReadY - minReadY)); + return gridEnvelope; + } + + private static int snapToTileGrid(int position, int offset, int tileSize) { + return (position - offset) / tileSize; + } + /** * Computes the size of a grid coverage in bytes given its grid envelope and the target sample * model @@ -879,4 +943,36 @@ private static GridGeometry2D simpleGridGeometryFit( throw new RuntimeException("Failed to invert grid to world", e); } } + + /** + * Checks if the coverage rendered image is deferred loaded, that is, if it's a JAI chain + * originating in a ImageRead operation + */ + public static boolean isDeferredLoaded(GridCoverage2D coverage) { + RenderedImage ri = coverage.getRenderedImage(); + return isDeferredLoaded(ri); + } + + /** + * Checks if the rendered image is based on a ImageRead operation, or if the potential JAI chain + * backing it results in a deferred loading. The method recursively calls itself and check the + * sources of the rendered image. Naively assumes that if one source is using ImageRead, then + * the whole chain is deferred loaded (which is true in all existing GeoTools readers) + */ + private static boolean isDeferredLoaded(RenderedImage ri) { + if (ri instanceof RenderedOp) { + RenderedOp rop = (RenderedOp) ri; + if ("ImageRead".equals(rop.getOperationName())) { + return true; + } + for (Object source : rop.getSources()) { + if (source instanceof RenderedImage) { + if (isDeferredLoaded((RenderedImage) source)) { + return true; + } + } + } + } + return false; + } } diff --git a/src/wcs/src/test/java/org/vfny/geoserver/util/WCSUtilsTest.java b/src/wcs/src/test/java/org/vfny/geoserver/util/WCSUtilsTest.java index ba6be4e7888..4dd9c3229aa 100644 --- a/src/wcs/src/test/java/org/vfny/geoserver/util/WCSUtilsTest.java +++ b/src/wcs/src/test/java/org/vfny/geoserver/util/WCSUtilsTest.java @@ -11,8 +11,10 @@ import org.geoserver.catalog.ProjectionPolicy; import org.geoserver.data.test.SystemTestData; import org.geoserver.test.GeoServerSystemTestSupport; +import org.geoserver.wcs.CoverageCleanerCallback; import org.geotools.api.referencing.datum.PixelInCell; import org.geotools.api.referencing.operation.MathTransform; +import org.geotools.coverage.grid.GridCoverage2D; import org.geotools.coverage.grid.GridEnvelope2D; import org.geotools.coverage.grid.GridGeometry2D; import org.geotools.coverage.grid.io.GridCoverage2DReader; @@ -180,4 +182,46 @@ public void fitReprojected() throws Exception { assertEquals(points[2] - points[0], fg2w.getScaleX(), 20d); assertEquals(points[3] - points[1], -fg2w.getScaleY(), 20d); } + + @Test + public void testGridSize() throws IOException { + CoverageInfo ci = getCatalog().getCoverageByName(getLayerId(SystemTestData.TASMANIA_DEM)); + GridCoverage2DReader reader = (GridCoverage2DReader) ci.getGridCoverageReader(null, null); + GridCoverage2D coverage = reader.read(null); + try { + long size = WCSUtils.getReadCoverageSize(coverage); + long expected = 120 * 240 * 2; // w * h * pixel size + assertEquals(expected, size); + } finally { + CoverageCleanerCallback.disposeCoverage(coverage); + } + } + + @Test + public void testGridSizeCropped() throws IOException { + CoverageInfo ci = getCatalog().getCoverageByName(getLayerId(SystemTestData.TASMANIA_DEM)); + GridCoverage2DReader reader = (GridCoverage2DReader) ci.getGridCoverageReader(null, null); + GridCoverage2D coverage = reader.read(null); + ReferencedEnvelope envelope = coverage.getEnvelope2D(); + double cropWidth = envelope.getWidth() / 4; + // image is 120x240, with tiles that are 120x34. Crop so that we get part of the tile width + // and one full tile height, plus a residual that forces loading a second tile + GridCoverage2D cropped = + WCSUtils.crop( + coverage, + new ReferencedEnvelope( + envelope.getCenterX() - cropWidth, + envelope.getCenterX() + cropWidth, + envelope.getMaxY(), // from the top, raster start there + envelope.getMaxY() - envelope.getHeight() * 50d / 240d, + ci.getCRS())); + try { + long size = WCSUtils.getReadCoverageSize(cropped); + long expected = 120 * (34 * 2) * 2; // w * h * pixel size + assertEquals(expected, size); + } finally { + CoverageCleanerCallback.disposeCoverage(cropped); + CoverageCleanerCallback.disposeCoverage(coverage); + } + } } diff --git a/src/wcs1_0/src/main/java/org/geoserver/wcs/DefaultWebCoverageService100.java b/src/wcs1_0/src/main/java/org/geoserver/wcs/DefaultWebCoverageService100.java index feb4dcbeeaa..7ebe7f6aa7f 100644 --- a/src/wcs1_0/src/main/java/org/geoserver/wcs/DefaultWebCoverageService100.java +++ b/src/wcs1_0/src/main/java/org/geoserver/wcs/DefaultWebCoverageService100.java @@ -5,6 +5,7 @@ */ package org.geoserver.wcs; +import static org.vfny.geoserver.util.WCSUtils.checkInputLimits; import static org.vfny.geoserver.wcs.WcsException.WcsExceptionCode.InvalidParameterValue; import java.awt.Rectangle; @@ -432,6 +433,27 @@ public GridCoverage[] getCoverage(final GetCoverageType request) { + "match no portions of it."); } + // compute intersection envelope to be used + GeneralBounds destinationEnvelope = + getDestinationEnvelope(requestedEnvelope, nativeEnvelope, targetCRS); + GeneralBounds destinationEnvelopeNativeCRS = destinationEnvelope; + if (!CRS.isEquivalent(nativeCRS, targetCRS)) { + destinationEnvelopeNativeCRS = CRS.transform(destinationEnvelope, nativeCRS); + } + + // do we have more than requested? Some readers return more than requested, + // but they do so with deferred loading. We need to understand if deferred loading + // is used, and if so, crop before checking the input limits, otherwise, + // check the input limits before cropping + if (WCSUtils.isDeferredLoaded(coverage)) { + // crop to the requested area before checking limits + coverage = WCSUtils.crop(coverage, destinationEnvelopeNativeCRS); + checkInputLimits(wcs, coverage); + } else { + checkInputLimits(wcs, coverage); + coverage = WCSUtils.crop(coverage, destinationEnvelopeNativeCRS); + } + // double check what we have loaded WCSUtils.checkInputLimits(wcs, coverage); @@ -443,13 +465,6 @@ public GridCoverage[] getCoverage(final GetCoverageType request) { bandSelectedCoverage = bandSelection(request, coverage); } - // - // final step for the requested coverage - // - // compute intersection envelope to be used - GeneralBounds destinationEnvelope = - getDestinationEnvelope(requestedEnvelope, nativeEnvelope, targetCRS); - final GridGeometry2D destinationGridGeometry = getGridGeometry(destinationSize, destinationG2W, destinationEnvelope); diff --git a/src/wcs1_0/src/test/java/org/geoserver/wcs/GetCoverageTest.java b/src/wcs1_0/src/test/java/org/geoserver/wcs/GetCoverageTest.java index e21e2fb2448..8f1cda6ed20 100644 --- a/src/wcs1_0/src/test/java/org/geoserver/wcs/GetCoverageTest.java +++ b/src/wcs1_0/src/test/java/org/geoserver/wcs/GetCoverageTest.java @@ -63,6 +63,7 @@ import org.geotools.referencing.operation.transform.AffineTransform2D; import org.geotools.util.PreventLocalEntityResolver; import org.geotools.wcs.WCSConfiguration; +import org.hamcrest.CoreMatchers; import org.junit.Before; import org.junit.Test; import org.springframework.mock.web.MockHttpServletResponse; @@ -375,6 +376,35 @@ public void testInputLimits() throws Exception { } } + @Test + public void testInputLimitsBounds() throws Exception { + try { + // request at roughly the native resolution + String url = + "wcs/BlueMarble/wcs?sourcecoverage=" + + getLayerId(TASMANIA_BM) + + "&request=getcoverage&service=wcs&version=1.0.0&format=image/geotiff&bbox=0,-43.3,180,-43.29" + + "&crs=EPSG:4326&resx=0.00417&resy=0.00417"; + + // the suggested tile size is 512x512. This makes the reader get the whole file in a + // single tile, even if the cropped image is around 25KB. The request should thus fail + setInputLimit(30); + MockHttpServletResponse response = getAsServletResponse(url); + assertThat( + response.getContentType(), + CoreMatchers.startsWith("application/vnd.ogc.se_xml")); + + // now set it to a larger amount, 400kb is enough to read 360x360x3 bytes + // (but not to read 512x512x3, the limit machinery accounts for actual file size) + setInputLimit(400); + response = getAsServletResponse(url); + assertThat(response.getContentType(), containsString("image/tiff")); + } finally { + // reset imits + setInputLimit(-1); + } + } + @Test public void testOutputLimits() throws Exception { try { diff --git a/src/wcs1_1/src/main/java/org/geoserver/wcs/DefaultWebCoverageService111.java b/src/wcs1_1/src/main/java/org/geoserver/wcs/DefaultWebCoverageService111.java index e9735d6f304..e9489813a09 100644 --- a/src/wcs1_1/src/main/java/org/geoserver/wcs/DefaultWebCoverageService111.java +++ b/src/wcs1_1/src/main/java/org/geoserver/wcs/DefaultWebCoverageService111.java @@ -5,6 +5,7 @@ */ package org.geoserver.wcs; +import static org.vfny.geoserver.util.WCSUtils.checkInputLimits; import static org.vfny.geoserver.wcs.WcsException.WcsExceptionCode.InvalidParameterValue; import java.awt.geom.AffineTransform; @@ -328,7 +329,7 @@ public GridCoverage[] getCoverage(GetCoverageType request) { // Check we're not being requested to read too much data from input (first check, // guesses the grid size using the information contained in CoverageInfo) - WCSUtils.checkInputLimits(wcs, meta, reader, requestedGridGeometry); + checkInputLimits(wcs, meta, reader, requestedGridGeometry); // // Check if we have a filter among the params @@ -356,18 +357,23 @@ public GridCoverage[] getCoverage(GetCoverageType request) { throw new IOException("The requested coverage could not be found."); } - // now that we have read the coverage double check the input size - WCSUtils.checkInputLimits(wcs, coverage); - - // some raster sources do not really read less data (arcgrid for example), we may need - // to crop - if (!intersectionEnvelopeInSourceCRS.contains(coverage.getEnvelope2D(), true)) { + // do we have more than requested? Some readers return more than requested, + // but they do so with deferred loading. We need to understand if deferred loading + // is used, and if so, crop before checking the input limits, otherwise, + // check the input limits before cropping + if (WCSUtils.isDeferredLoaded(coverage)) { + // crop to the requested area before checking limits + coverage = WCSUtils.crop(coverage, intersectionEnvelopeInSourceCRS); + checkInputLimits(wcs, coverage); + } else { + checkInputLimits(wcs, coverage); coverage = WCSUtils.crop(coverage, intersectionEnvelopeInSourceCRS); - if (coverage == null) - throw new ServiceException( - "Requested area incompatible with raster space, less than a pixel would be read"); } + if (coverage == null) + throw new ServiceException( + "Requested area incompatible with raster space, less than a pixel would be read"); + /** Band Select (works on just one field) */ GridCoverage2D bandSelectedCoverage = coverage; String interpolationType = null; diff --git a/src/wcs1_1/src/test/java/org/geoserver/wcs/GetCoverageTest.java b/src/wcs1_1/src/test/java/org/geoserver/wcs/GetCoverageTest.java index 8b3a1523b66..8fea6b7df0a 100644 --- a/src/wcs1_1/src/test/java/org/geoserver/wcs/GetCoverageTest.java +++ b/src/wcs1_1/src/test/java/org/geoserver/wcs/GetCoverageTest.java @@ -419,6 +419,32 @@ public void testInputLimits() throws Exception { } } + @Test + public void testInputLimitsBounds() throws Exception { + try { + String url = + "wcs/BlueMarble/wcs?identifier=" + + getLayerId(TASMANIA_BM) + + "&request=getcoverage&service=wcs&version=1.1.1&&format=image/geotiff" + + "&BoundingBox=-43.3,0,-43.29,180,urn:ogc:def:crs:EPSG:6.6:4326"; + + // the suggested tile size is 512x512. This makes the reader get the whole file in a + // single tile, even if the cropped image is around 25KB. The request should thus fail + setInputLimit(30); + MockHttpServletResponse response = getAsServletResponse(url); + assertEquals("application/xml", response.getContentType()); + + // now set it to a larger amount, 400kb is enough to read 360x360x3 bytes + // (but not to read 512x512x3, the limit machinery accounts for actual file size) + setInputLimit(400); + response = getAsServletResponse(url); + assertThat(response.getContentType(), containsString("multipart/related")); + } finally { + // reset imits + setInputLimit(-1); + } + } + @Test public void testTimeInputLimitsDefault() throws Exception { String queryString = diff --git a/src/wcs2_0/src/main/java/org/geoserver/wcs2_0/GetCoverage.java b/src/wcs2_0/src/main/java/org/geoserver/wcs2_0/GetCoverage.java index 7af02bb53b5..132bceed1a8 100644 --- a/src/wcs2_0/src/main/java/org/geoserver/wcs2_0/GetCoverage.java +++ b/src/wcs2_0/src/main/java/org/geoserver/wcs2_0/GetCoverage.java @@ -1086,21 +1086,12 @@ private List readCoverage( } readCoverages.add(cov); } - // do we have more than requested? - ReferencedEnvelope covEnvelope = cov.getEnvelope2D(); - GridCoverage2D cropped = cov; - if (covEnvelope.contains(readBoundingBox) - && (covEnvelope.getWidth() > readBoundingBox.getWidth() - || covEnvelope.getHeight() > readBoundingBox.getHeight())) { - cropped = cropOnEnvelope(cov, readEnvelope); - if (cropped == null) continue; - } // do we have less than expected? - GridCoverage2D padded = cropped; - Bounds croppedEnvelope = cropped.getEnvelope(); + GridCoverage2D padded = cov; + Bounds croppedEnvelope = cov.getEnvelope(); if (!new GeneralBounds(croppedEnvelope).contains(padEnvelope, true)) { - padded = padOnEnvelope(cropped, padEnvelope); + padded = padOnEnvelope(cov, padEnvelope); } if (padded != null) result.add(padded); @@ -1350,14 +1341,17 @@ private GridCoverage2D readCoverage( request.getOverviewPolicy(), readHints); if (coverage != null) { - // check limits again - if (incrementalInputSize == null) { - WCSUtils.checkInputLimits(wcs, coverage); + // do we have more than requested? Some readers return more than requested, + // but they do so with deferred loading. We need to understand if deferred loading + // is used, and if so, crop before checking the input limits, otherwise, + // check the input limits before cropping + if (WCSUtils.isDeferredLoaded(coverage)) { + // crop to the requested area before checking limits + coverage = cropCoverage(coverage, subset); + checkInputLimits(incrementalInputSize, coverage); } else { - // Check for each coverage added if the total coverage dimension exceeds the maximum - // limit - // If the size is exceeded an exception is thrown - incrementalInputSize.addSize(coverage); + checkInputLimits(incrementalInputSize, coverage); + coverage = cropCoverage(coverage, subset); } // see what scaling factors the reader actually applied @@ -1383,6 +1377,29 @@ private GridCoverage2D readCoverage( return coverage; } + private void checkInputLimits(ImageSizeRecorder incrementalInputSize, GridCoverage2D coverage) { + // check limits again + if (incrementalInputSize == null) { + WCSUtils.checkInputLimits(wcs, coverage); + } else { + // Check for each coverage added if the total coverage dimension exceeds the maximum + // limit if the size is exceeded an exception is thrown + incrementalInputSize.addSize(coverage); + } + } + + private GridCoverage2D cropCoverage(GridCoverage2D coverage, Bounds subset) { + ReferencedEnvelope readBoundingBox = ReferencedEnvelope.reference(subset); + ReferencedEnvelope covEnvelope = coverage.getEnvelope2D(); + GridCoverage2D cropped = coverage; + if (covEnvelope.contains(readBoundingBox) + && (covEnvelope.getWidth() > readBoundingBox.getWidth() + || covEnvelope.getHeight() > readBoundingBox.getHeight())) { + cropped = cropOnEnvelope(coverage, subset); + } + return cropped; + } + MathTransform getMathTransform( CoverageInfo ci, GridCoverage2DReader reader, diff --git a/src/wcs2_0/src/test/java/org/geoserver/wcs2_0/xml/GetCoverageTest.java b/src/wcs2_0/src/test/java/org/geoserver/wcs2_0/xml/GetCoverageTest.java index 960fbde8cc0..02694189cbd 100644 --- a/src/wcs2_0/src/test/java/org/geoserver/wcs2_0/xml/GetCoverageTest.java +++ b/src/wcs2_0/src/test/java/org/geoserver/wcs2_0/xml/GetCoverageTest.java @@ -383,6 +383,29 @@ public void testInputLimits() throws Exception { setInputLimit(-1); } + @Test + public void testInputLimitsBounds() throws Exception { + try { + // single slice + final File xml = new File("./src/test/resources/requestGetCoverageSlice.xml"); + final String request = FileUtils.readFileToString(xml, "UTF-8"); + // the suggested tile size is 512x512. This makes the reader get the whole file in a + // single tile, even if the cropped image is around 25KB. The request should thus fail + setInputLimit(30); + MockHttpServletResponse response = postAsServletResponse("wcs", request); + assertEquals("application/xml", response.getContentType()); + + // now set it to a larger amount, 400kb is enough to read 360x360x3 bytes + // (but not to read 512x512x3, the limit machinery accounts for actual file size) + setInputLimit(400); + response = postAsServletResponse("wcs", request); + assertEquals("image/tiff", response.getContentType()); + } finally { + // reset imits + setInputLimit(-1); + } + } + @Test public void testOutputLimits() throws Exception { final File xml = new File("./src/test/resources/requestGetFullCoverage.xml"); diff --git a/src/wcs2_0/src/test/resources/requestGetCoverageSlice.xml b/src/wcs2_0/src/test/resources/requestGetCoverageSlice.xml new file mode 100644 index 00000000000..8f153a18b49 --- /dev/null +++ b/src/wcs2_0/src/test/resources/requestGetCoverageSlice.xml @@ -0,0 +1,14 @@ + + + wcs__BlueMarble + + Lat + -43.3 + -43.29 + + image/tiff + \ No newline at end of file