[GEOS-11643] WCS input read limits can be fooled by geotiff reader

src/wcs/src/main/java/org/vfny/geoserver/util/WCSUtils.java

@@ -5,6 +5,7 @@
  */
 package org.vfny.geoserver.util;
 
+import java.awt.Rectangle;
 import java.awt.image.RenderedImage;
 import java.awt.image.SampleModel;
 import java.text.DecimalFormat;
@@ -363,17 +364,17 @@ public static void checkOutputLimits(
      * extra memory usage, just makes it happen sooner)
      */
     public static void checkInputLimits(WCSInfo info, GridCoverage2D coverage) {
+        // null safety, and read/crop might return null
+        if (coverage == null) return;
+
         // do we have to check a limit at all?
         long limit = info.getMaxInputMemory() * 1024;
         if (limit <= 0) {
             return;
         }
 
         // compute the coverage memory usage and compare with limit
-        long actual =
-                getCoverageSize(
-                        coverage.getGridGeometry().getGridRange2D(),
-                        coverage.getRenderedImage().getSampleModel());
+        long actual = getReadCoverageSize(coverage);
         if (actual > limit) {
             throw new WcsException(
                     "This request is trying to read too much data, "
@@ -385,6 +386,69 @@ public static void checkInputLimits(WCSInfo info, GridCoverage2D coverage) {
         }
     }
 
+    /**
+     * If the image is deferred loaded, and cropped, consider that full tiles need to be read before
+     * cropping can happen. This is important for images that are tiled but whose form factor is
+     * very much streatched, so that the tiles are read in full, even if only a small portion of
+     * them is actually used. This is adopting a simplistic approach, assuming the crop is the last
+     * operation in the chain, while in general other operations might be involved (but if they are,
+     * and rescaling/warping is in the mix, then computing the actual read size is going to be
+     * pretty complicated).
+     */
+    static long getReadCoverageSize(GridCoverage2D coverage) {
+        RenderedImage ri = coverage.getRenderedImage();
+
+        GridEnvelope2D gridEnvelope = coverage.getGridGeometry().getGridRange2D();
+        // deferred crop? we are just going to read the tiles covering the crop area
+        if (isDeferredLoaded(ri)) {
+            RenderedOp op = (RenderedOp) ri;
+            String operationName = op.getOperationName();
+            // "crop" can be implemented both as actual crop, or mosaic
+            if ("Crop".equals(operationName)
+                    || ("Mosaic".equals(operationName) && op.getNumSources() == 1)) {
+                gridEnvelope = getCropTilesEnvelope(op);
+            }
+        }
+        return getCoverageSize(gridEnvelope, ri.getSampleModel());
+    }
+
+    /**
+     * Returns the envelope of the tiles covering the image. This helps computing the actual size of
+     * the image that will be read from the disk, in case the image is tiled and the crop operation
+     * is the last one in the chain.
+     */
+    private static GridEnvelope2D getCropTilesEnvelope(RenderedOp crop) {
+        RenderedImage source = (RenderedImage) crop.getSources().get(0);
+        Rectangle bounds = crop.getBounds();
+        int tileXOffset = source.getTileGridXOffset();
+        int tileYOffset = source.getTileGridYOffset();
+        int tileWidth = source.getTileWidth();
+        int tileHeight = source.getTileHeight();
+        int tileMinX = snapToTileGrid(bounds.x, tileXOffset, tileWidth);
+        int tileMinY = snapToTileGrid(bounds.y, tileYOffset, tileHeight);
+        int tileMaxX = snapToTileGrid(bounds.x + bounds.width, tileXOffset, tileWidth);
+        int tileMaxY = snapToTileGrid(bounds.y + bounds.height, tileYOffset, tileHeight);
+        // account the size of the source image, could be smaller than the suggested tile size
+        int minReadX = Math.max(tileXOffset + tileMinX * tileWidth, source.getMinX());
+        int minReadY = Math.max(tileYOffset + tileMinY * tileHeight, source.getMinY());
+        int maxReadX =
+                Math.min(
+                        tileXOffset + (tileMaxX + 1) * tileWidth,
+                        source.getMinX() + source.getWidth());
+        int maxReadY =
+                Math.min(
+                        tileYOffset + (tileMaxY + 1) * tileHeight,
+                        source.getMinY() + source.getHeight());
+        GridEnvelope2D gridEnvelope =
+                new GridEnvelope2D(
+                        minReadX, minReadY, (maxReadX - minReadX), (maxReadY - minReadY));
+        return gridEnvelope;
+    }
+
+    private static int snapToTileGrid(int position, int offset, int tileSize) {
+        return (position - offset) / tileSize;
+    }
+
     /**
      * Computes the size of a grid coverage in bytes given its grid envelope and the target sample
      * model
@@ -879,4 +943,36 @@ private static GridGeometry2D simpleGridGeometryFit(
             throw new RuntimeException("Failed to invert grid to world", e);
         }
     }
+
+    /**
+     * Checks if the coverage rendered image is deferred loaded, that is, if it's a JAI chain
+     * originating in a ImageRead operation
+     */
+    public static boolean isDeferredLoaded(GridCoverage2D coverage) {
+        RenderedImage ri = coverage.getRenderedImage();
+        return isDeferredLoaded(ri);
+    }
+
+    /**
+     * Checks if the rendered image is based on a ImageRead operation, or if the potential JAI chain
+     * backing it results in a deferred loading. The method recursively calls itself and check the
+     * sources of the rendered image. Naively assumes that if one source is using ImageRead, then
+     * the whole chain is deferred loaded (which is true in all existing GeoTools readers)
+     */
+    private static boolean isDeferredLoaded(RenderedImage ri) {
+        if (ri instanceof RenderedOp) {
+            RenderedOp rop = (RenderedOp) ri;
+            if ("ImageRead".equals(rop.getOperationName())) {
+                return true;
+            }
+            for (Object source : rop.getSources()) {
+                if (source instanceof RenderedImage) {
+                    if (isDeferredLoaded((RenderedImage) source)) {
+                        return true;
+                    }
+                }
+            }
+        }
+        return false;
+    }
 }

src/wcs/src/test/java/org/vfny/geoserver/util/WCSUtilsTest.java

@@ -11,8 +11,10 @@
 import org.geoserver.catalog.ProjectionPolicy;
 import org.geoserver.data.test.SystemTestData;
 import org.geoserver.test.GeoServerSystemTestSupport;
+import org.geoserver.wcs.CoverageCleanerCallback;
 import org.geotools.api.referencing.datum.PixelInCell;
 import org.geotools.api.referencing.operation.MathTransform;
+import org.geotools.coverage.grid.GridCoverage2D;
 import org.geotools.coverage.grid.GridEnvelope2D;
 import org.geotools.coverage.grid.GridGeometry2D;
 import org.geotools.coverage.grid.io.GridCoverage2DReader;
@@ -180,4 +182,46 @@ public void fitReprojected() throws Exception {
         assertEquals(points[2] - points[0], fg2w.getScaleX(), 20d);
         assertEquals(points[3] - points[1], -fg2w.getScaleY(), 20d);
     }
+
+    @Test
+    public void testGridSize() throws IOException {
+        CoverageInfo ci = getCatalog().getCoverageByName(getLayerId(SystemTestData.TASMANIA_DEM));
+        GridCoverage2DReader reader = (GridCoverage2DReader) ci.getGridCoverageReader(null, null);
+        GridCoverage2D coverage = reader.read(null);
+        try {
+            long size = WCSUtils.getReadCoverageSize(coverage);
+            long expected = 120 * 240 * 2; // w * h * pixel size
+            assertEquals(expected, size);
+        } finally {
+            CoverageCleanerCallback.disposeCoverage(coverage);
+        }
+    }
+
+    @Test
+    public void testGridSizeCropped() throws IOException {
+        CoverageInfo ci = getCatalog().getCoverageByName(getLayerId(SystemTestData.TASMANIA_DEM));
+        GridCoverage2DReader reader = (GridCoverage2DReader) ci.getGridCoverageReader(null, null);
+        GridCoverage2D coverage = reader.read(null);
+        ReferencedEnvelope envelope = coverage.getEnvelope2D();
+        double cropWidth = envelope.getWidth() / 4;
+        // image is 120x240, with tiles that are 120x34. Crop so that we get part of the tile width
+        // and one full tile height, plus a residual that forces loading a second tile
+        GridCoverage2D cropped =
+                WCSUtils.crop(
+                        coverage,
+                        new ReferencedEnvelope(
+                                envelope.getCenterX() - cropWidth,
+                                envelope.getCenterX() + cropWidth,
+                                envelope.getMaxY(), // from the top, raster start there
+                                envelope.getMaxY() - envelope.getHeight() * 50d / 240d,
+                                ci.getCRS()));
+        try {
+            long size = WCSUtils.getReadCoverageSize(cropped);
+            long expected = 120 * (34 * 2) * 2; // w * h * pixel size
+            assertEquals(expected, size);
+        } finally {
+            CoverageCleanerCallback.disposeCoverage(cropped);
+            CoverageCleanerCallback.disposeCoverage(coverage);
+        }
+    }
 }

src/wcs1_0/src/main/java/org/geoserver/wcs/DefaultWebCoverageService100.java

@@ -5,6 +5,7 @@
  */
 package org.geoserver.wcs;
 
+import static org.vfny.geoserver.util.WCSUtils.checkInputLimits;
 import static org.vfny.geoserver.wcs.WcsException.WcsExceptionCode.InvalidParameterValue;
 
 import java.awt.Rectangle;
@@ -432,6 +433,27 @@ public GridCoverage[] getCoverage(final GetCoverageType request) {
                                 + "match no portions of it.");
             }
 
+            // compute intersection envelope to be used
+            GeneralBounds destinationEnvelope =
+                    getDestinationEnvelope(requestedEnvelope, nativeEnvelope, targetCRS);
+            GeneralBounds destinationEnvelopeNativeCRS = destinationEnvelope;
+            if (!CRS.isEquivalent(nativeCRS, targetCRS)) {
+                destinationEnvelopeNativeCRS = CRS.transform(destinationEnvelope, nativeCRS);
+            }
+
+            // do we have more than requested? Some readers return more than requested,
+            // but they do so with deferred loading. We need to understand if deferred loading
+            // is used, and if so, crop before checking the input limits, otherwise,
+            // check the input limits before cropping
+            if (WCSUtils.isDeferredLoaded(coverage)) {
+                // crop to the requested area before checking limits
+                coverage = WCSUtils.crop(coverage, destinationEnvelopeNativeCRS);
+                checkInputLimits(wcs, coverage);
+            } else {
+                checkInputLimits(wcs, coverage);
+                coverage = WCSUtils.crop(coverage, destinationEnvelopeNativeCRS);
+            }
+
             // double check what we have loaded
             WCSUtils.checkInputLimits(wcs, coverage);
 
@@ -443,13 +465,6 @@ public GridCoverage[] getCoverage(final GetCoverageType request) {
                 bandSelectedCoverage = bandSelection(request, coverage);
             }
 
-            //
-            // final step for the requested coverage
-            //
-            // compute intersection envelope to be used
-            GeneralBounds destinationEnvelope =
-                    getDestinationEnvelope(requestedEnvelope, nativeEnvelope, targetCRS);
-
             final GridGeometry2D destinationGridGeometry =
                     getGridGeometry(destinationSize, destinationG2W, destinationEnvelope);
 

src/wcs1_0/src/test/java/org/geoserver/wcs/GetCoverageTest.java

@@ -63,6 +63,7 @@
 import org.geotools.referencing.operation.transform.AffineTransform2D;
 import org.geotools.util.PreventLocalEntityResolver;
 import org.geotools.wcs.WCSConfiguration;
+import org.hamcrest.CoreMatchers;
 import org.junit.Before;
 import org.junit.Test;
 import org.springframework.mock.web.MockHttpServletResponse;
@@ -375,6 +376,35 @@ public void testInputLimits() throws Exception {
         }
     }
 
+    @Test
+    public void testInputLimitsBounds() throws Exception {
+        try {
+            // request at roughly the native resolution
+            String url =
+                    "wcs/BlueMarble/wcs?sourcecoverage="
+                            + getLayerId(TASMANIA_BM)
+                            + "&request=getcoverage&service=wcs&version=1.0.0&format=image/geotiff&bbox=0,-43.3,180,-43.29"
+                            + "&crs=EPSG:4326&resx=0.00417&resy=0.00417";
+
+            // the suggested tile size is 512x512. This makes the reader get the whole file in a
+            // single tile, even if the cropped image is around 25KB. The request should thus fail
+            setInputLimit(30);
+            MockHttpServletResponse response = getAsServletResponse(url);
+            assertThat(
+                    response.getContentType(),
+                    CoreMatchers.startsWith("application/vnd.ogc.se_xml"));
+
+            // now set it to a larger amount, 400kb is enough to read 360x360x3 bytes
+            // (but not to read 512x512x3, the limit machinery accounts for actual file size)
+            setInputLimit(400);
+            response = getAsServletResponse(url);
+            assertThat(response.getContentType(), containsString("image/tiff"));
+        } finally {
+            // reset imits
+            setInputLimit(-1);
+        }
+    }
+
     @Test
     public void testOutputLimits() throws Exception {
         try {

src/wcs1_1/src/main/java/org/geoserver/wcs/DefaultWebCoverageService111.java

@@ -5,6 +5,7 @@
  */
 package org.geoserver.wcs;
 
+import static org.vfny.geoserver.util.WCSUtils.checkInputLimits;
 import static org.vfny.geoserver.wcs.WcsException.WcsExceptionCode.InvalidParameterValue;
 
 import java.awt.geom.AffineTransform;
@@ -328,7 +329,7 @@ public GridCoverage[] getCoverage(GetCoverageType request) {
 
             // Check we're not being requested to read too much data from input (first check,
             // guesses the grid size using the information contained in CoverageInfo)
-            WCSUtils.checkInputLimits(wcs, meta, reader, requestedGridGeometry);
+            checkInputLimits(wcs, meta, reader, requestedGridGeometry);
 
             //
             // Check if we have a filter among the params
@@ -356,18 +357,23 @@ public GridCoverage[] getCoverage(GetCoverageType request) {
                 throw new IOException("The requested coverage could not be found.");
             }
 
-            // now that we have read the coverage double check the input size
-            WCSUtils.checkInputLimits(wcs, coverage);
-
-            // some raster sources do not really read less data (arcgrid for example), we may need
-            // to crop
-            if (!intersectionEnvelopeInSourceCRS.contains(coverage.getEnvelope2D(), true)) {
+            // do we have more than requested? Some readers return more than requested,
+            // but they do so with deferred loading. We need to understand if deferred loading
+            // is used, and if so, crop before checking the input limits, otherwise,
+            // check the input limits before cropping
+            if (WCSUtils.isDeferredLoaded(coverage)) {
+                // crop to the requested area before checking limits
+                coverage = WCSUtils.crop(coverage, intersectionEnvelopeInSourceCRS);
+                checkInputLimits(wcs, coverage);
+            } else {
+                checkInputLimits(wcs, coverage);
                 coverage = WCSUtils.crop(coverage, intersectionEnvelopeInSourceCRS);
-                if (coverage == null)
-                    throw new ServiceException(
-                            "Requested area incompatible with raster space, less than a pixel would be read");
             }
 
+            if (coverage == null)
+                throw new ServiceException(
+                        "Requested area incompatible with raster space, less than a pixel would be read");
+
             /** Band Select (works on just one field) */
             GridCoverage2D bandSelectedCoverage = coverage;
             String interpolationType = null;

src/wcs1_1/src/test/java/org/geoserver/wcs/GetCoverageTest.java

@@ -419,6 +419,32 @@ public void testInputLimits() throws Exception {
         }
     }
 
+    @Test
+    public void testInputLimitsBounds() throws Exception {
+        try {
+            String url =
+                    "wcs/BlueMarble/wcs?identifier="
+                            + getLayerId(TASMANIA_BM)
+                            + "&request=getcoverage&service=wcs&version=1.1.1&&format=image/geotiff"
+                            + "&BoundingBox=-43.3,0,-43.29,180,urn:ogc:def:crs:EPSG:6.6:4326";
+
+            // the suggested tile size is 512x512. This makes the reader get the whole file in a
+            // single tile, even if the cropped image is around 25KB. The request should thus fail
+            setInputLimit(30);
+            MockHttpServletResponse response = getAsServletResponse(url);
+            assertEquals("application/xml", response.getContentType());
+
+            // now set it to a larger amount, 400kb is enough to read 360x360x3 bytes
+            // (but not to read 512x512x3, the limit machinery accounts for actual file size)
+            setInputLimit(400);
+            response = getAsServletResponse(url);
+            assertThat(response.getContentType(), containsString("multipart/related"));
+        } finally {
+            // reset imits
+            setInputLimit(-1);
+        }
+    }
+
     @Test
     public void testTimeInputLimitsDefault() throws Exception {
         String queryString =