Merge pull request #1084 from gritGmbH/enhancement/raster-cache

Fix cache handling of raster files

deegree-core/deegree-core-coverage/src/main/java/org/deegree/coverage/raster/cache/RasterCache.java

@@ -43,13 +43,15 @@
 import java.io.File;
 import java.io.Serializable;
 import java.util.Comparator;
+import java.util.HashMap;
 import java.util.Iterator;
 import java.util.Map;
 import java.util.SortedSet;
 import java.util.UUID;
 import java.util.concurrent.ConcurrentHashMap;
 import java.util.concurrent.ConcurrentSkipListSet;
 
+import org.deegree.commons.utils.FileUtils;
 import org.deegree.commons.utils.StringUtils;
 import org.deegree.coverage.raster.SimpleRaster;
 import org.deegree.coverage.raster.data.RasterDataFactory;
@@ -110,6 +112,9 @@ public class RasterCache {
 
     private final static ConcurrentSkipListSet<CacheRasterReader> cache = new ConcurrentSkipListSet<CacheRasterReader>(
                                                                                                                         new CacheComparator() );
+
+    private final static Map<String, String> uniqueRasterCacheIds = new HashMap<String, String>();
+
     static {
         evaluateProperties();
     }
@@ -481,6 +486,7 @@ public RasterReader addReader( RasterReader reader ) {
                 boolean createCache = reader.shouldCreateCacheFile();
                 File cacheFile = null;
                 if ( createCache ) {
+                    LOG.trace( "create cachefile for location {}", reader.getDataLocationId() );
                     cacheFile = createCacheFile( reader.getDataLocationId() );
                 }
                 result = new CacheRasterReader( reader, cacheFile, this );
@@ -594,4 +600,47 @@ public static void disableAllCaches() {
         maxCacheDisk = 0;
     }
 
+    /**
+     * Generate unique (in the scope of RasterCache) short filename for raster reader identification
+     * 
+     * @param file
+     *            file reference to start with
+     * @return unique filename (may contains prefix to make it unique)
+     */
+    public static synchronized String getUniqueCacheIdentifier( File file ) {
+        String fname = FileUtils.getFilename( file );
+        String apath = file.getAbsolutePath();
+
+        int idx = 0;
+        String key = fname;
+
+        while ( !apath.equals( uniqueRasterCacheIds.getOrDefault( key, apath ) ) ) {
+            idx++;
+            key = idx + "_" + fname;
+        }
+        uniqueRasterCacheIds.put( key, apath );
+        return key;
+    }
+
+    /**
+     * Helper function to check if a .no-cache or .no-cache-[level] file exists
+     */
+    public static boolean hasNoCacheFile( File file, int level ) {
+        try {
+            if ( file == null || !file.exists() )
+                return false;
+
+            File all = new File( file.getParentFile(), file.getName() + ".no-cache" );
+            if ( all.exists() ) {
+                return true;
+            }
+
+            File lvl = new File( file.getParentFile(), file.getName() + ".no-cache-" + level );
+            return lvl.exists();
+        } catch ( Exception ex ) {
+            LOG.debug( "Failed to check for .no-cache files for {} level {}", file, level );
+            LOG.debug( "Got exception", ex );
+            return false;
+        }
+    }
 }

deegree-core/deegree-core-coverage/src/main/java/org/deegree/coverage/raster/io/imageio/IIORasterDataReader.java

@@ -56,6 +56,7 @@
 import javax.imageio.stream.ImageInputStream;
 
 import org.deegree.coverage.raster.cache.ByteBufferPool;
+import org.deegree.coverage.raster.cache.RasterCache;
 import org.deegree.coverage.raster.data.container.BufferResult;
 import org.deegree.coverage.raster.data.info.BandType;
 import org.deegree.coverage.raster.data.info.DataType;
@@ -80,6 +81,15 @@
  */
 public class IIORasterDataReader implements RasterDataReader {
 
+    private static final boolean CACHE_DISABLED;
+
+    static {
+        CACHE_DISABLED = "false".equalsIgnoreCase( System.getProperty( "deegree.raster.cache.iioreader", "true" ));
+        if ( !CACHE_DISABLED ) {
+            LoggerFactory.getLogger( IIORasterDataReader.class ).info( "IIORaster cache is disabled for file based resources via system property!" );
+        }
+    }
+
     // io handles
     private File file;
 
@@ -120,6 +130,8 @@ public class IIORasterDataReader implements RasterDataReader {
     private RasterDataInfo rdi;
 
     private final int imageIndex;
+
+    private final boolean forceNoCache;
 
     /**
      * Create a IIORasterDataReader for given file
@@ -130,7 +142,7 @@ public class IIORasterDataReader implements RasterDataReader {
      *            with values.
      */
     public IIORasterDataReader( File file, RasterIOOptions options, int imageIndex ) {
-        this( options, false, imageIndex );
+        this( options, false, imageIndex, CACHE_DISABLED || RasterCache.hasNoCacheFile( file, imageIndex ) );
         this.file = file;
     }
 
@@ -143,15 +155,16 @@ public IIORasterDataReader( File file, RasterIOOptions options, int imageIndex )
      *            with values
      */
     public IIORasterDataReader( InputStream stream, RasterIOOptions options, int imageIndex ) {
-        this( options, ( stream != null && stream.markSupported() ), imageIndex );
+        this( options, ( stream != null && stream.markSupported() ), imageIndex, false );
         this.inputStream = stream;
     }
 
-    private IIORasterDataReader( RasterIOOptions options, boolean resetableStream, int imageIndex ) {
+    private IIORasterDataReader( RasterIOOptions options, boolean resetableStream, int imageIndex, boolean forceNoCache ) {
         this.imageIndex = imageIndex;
         this.format = options.get( RasterIOOptions.OPT_FORMAT );
         this.options = options;
         this.resetableStream = resetableStream;
+        this.forceNoCache = forceNoCache;
     }
 
     /**
@@ -370,6 +383,11 @@ protected File file() {
      * @return true if the imageio thinks the file can be accessed easily
      */
     boolean shouldCreateCacheFile() {
+        if ( forceNoCache ) {
+            LOG.debug( "cache for reader with file {} and index {} disabbled by no-cache-file/system-property", file, imageIndex );
+            return false;
+        }
+
         boolean result = true;
         try {
             synchronized ( LOCK ) {
@@ -469,5 +487,4 @@ public void dispose() {
             }
         }
     }
-
 }

deegree-core/deegree-core-coverage/src/main/java/org/deegree/coverage/raster/io/imageio/IIORasterReader.java

@@ -44,24 +44,21 @@
 
 import javax.imageio.ImageIO;
 
-import org.deegree.commons.utils.FileUtils;
 import org.deegree.coverage.raster.AbstractRaster;
 import org.deegree.coverage.raster.SimpleRaster;
 import org.deegree.coverage.raster.cache.RasterCache;
 import org.deegree.coverage.raster.data.container.BufferResult;
 import org.deegree.coverage.raster.data.info.RasterDataInfo;
 import org.deegree.coverage.raster.geom.RasterGeoReference;
-import org.deegree.coverage.raster.geom.RasterRect;
 import org.deegree.coverage.raster.geom.RasterGeoReference.OriginLocation;
+import org.deegree.coverage.raster.geom.RasterRect;
 import org.deegree.coverage.raster.io.RasterIOOptions;
 import org.deegree.coverage.raster.io.RasterReader;
 import org.deegree.coverage.raster.io.WorldFileAccess;
 import org.deegree.coverage.raster.utils.RasterFactory;
 import org.deegree.cs.coordinatesystems.ICRS;
 import org.deegree.cs.persistence.CRSManager;
 import org.deegree.geometry.Envelope;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
 
 /**
  * 
@@ -76,8 +73,6 @@ public class IIORasterReader implements RasterReader {
 
     private static final Set<String> SUPPORTED_TYPES;
 
-    private static Logger LOG = LoggerFactory.getLogger( IIORasterReader.class );
-
     static {
         SUPPORTED_TYPES = new HashSet<String>();
 
@@ -126,7 +121,6 @@ public boolean canLoad( File filename ) {
     public AbstractRaster load( File file, RasterIOOptions options )
                             throws IOException {
         String imageIndex = options.get( RasterIOOptions.IMAGE_INDEX );
-        LOG.debug( "reading " + file + " with ImageIO" );
         reader = new IIORasterDataReader( file, options, imageIndex == null ? 0 : Integer.parseInt( imageIndex ) );
         AbstractRaster r = loadFromReader( reader, options );
         return r;
@@ -193,9 +187,14 @@ private AbstractRaster loadFromReader( IIORasterDataReader reader, RasterIOOptio
         RasterDataInfo rdi = reader.getRasterDataInfo();
 
         RasterCache cache = RasterCache.getInstance( opts );
-        SimpleRaster result = cache.createFromCache( this, this.dataLocationId );
+        SimpleRaster result = null;
+
+        boolean useCache = shouldCreateCacheFile();
+        if ( useCache ) {
+            result = cache.createFromCache( this, this.dataLocationId );
+        }
         if ( result == null ) {
-            result = RasterFactory.createEmptyRaster( rdi, envelope, rasterReference, this, true, opts );
+            result = RasterFactory.createEmptyRaster( rdi, envelope, rasterReference, this, useCache, opts );
         }
         this.reader.dispose();
         return result;
@@ -207,9 +206,14 @@ private void setID( RasterIOOptions options ) {
         this.dataLocationId = options != null ? options.get( RasterIOOptions.ORIGIN_OF_RASTER ) : null;
         if ( dataLocationId == null ) {
             if ( reader != null && reader.file() != null ) {
-                this.dataLocationId = FileUtils.getFilename( reader.file() );
+                this.dataLocationId = RasterCache.getUniqueCacheIdentifier( reader.file() );
             }
         }
+
+        String imageIndex = options.get( RasterIOOptions.IMAGE_INDEX );
+        if (imageIndex != null) {
+            this.dataLocationId += "__" + imageIndex;
+        }
     }
 
     @Override

deegree-core/deegree-core-coverage/src/main/java/org/deegree/coverage/raster/io/jai/JAIRasterReader.java

@@ -43,6 +43,7 @@
 
 import org.deegree.commons.utils.FileUtils;
 import org.deegree.coverage.raster.AbstractRaster;
+import org.deegree.coverage.raster.cache.RasterCache;
 import org.deegree.coverage.raster.data.container.BufferResult;
 import org.deegree.coverage.raster.data.info.RasterDataInfo;
 import org.deegree.coverage.raster.geom.RasterGeoReference;
@@ -132,7 +133,7 @@ private void setID( RasterIOOptions options ) {
         this.dataLocationId = options != null ? options.get( RasterIOOptions.ORIGIN_OF_RASTER ) : null;
         if ( dataLocationId == null ) {
             if ( this.file != null ) {
-                this.dataLocationId = FileUtils.getFilename( this.file );
+                this.dataLocationId = RasterCache.getUniqueCacheIdentifier( this.file );
             }
         }
     }

deegree-core/deegree-core-rendering-2d/src/main/java/org/deegree/rendering/r2d/legends/LegendBuilder.java

@@ -124,6 +124,11 @@ Pair<Integer, Integer> getLegendSize( Style style ) {
             res.first = max( res.first, item.getMaxWidth( opts ) );
         }
 
+        if ( res.second == 0 ) {
+            // prevent >0 * 0 sized images
+            res.second = 2 * opts.spacing + opts.baseWidth;
+        }
+
         return res;
     }
 

deegree-services/deegree-services-wms/src/main/java/org/deegree/services/wms/GetLegendHandler.java

@@ -45,18 +45,21 @@ Occam Labs UG (haftungsbeschränkt)
 import static java.awt.RenderingHints.KEY_TEXT_ANTIALIASING;
 import static java.awt.RenderingHints.VALUE_ANTIALIAS_ON;
 import static java.awt.RenderingHints.VALUE_TEXT_ANTIALIAS_ON;
+import static org.deegree.cs.i18n.Messages.get;
 import static org.deegree.style.utils.ImageUtils.postprocessPng8bit;
 
 import java.awt.Graphics2D;
 import java.awt.image.BufferedImage;
 import java.util.HashMap;
 
+import org.deegree.commons.ows.exception.OWSException;
 import org.deegree.commons.utils.Pair;
 import org.deegree.layer.LayerRef;
 import org.deegree.protocol.wms.ops.GetLegendGraphic;
 import org.deegree.rendering.r2d.legends.Legends;
 import org.deegree.style.StyleRef;
 import org.deegree.style.se.unevaluated.Style;
+import org.deegree.theme.Theme;
 
 /**
  * Produces legends for the map service.
@@ -78,7 +81,7 @@ class GetLegendHandler {
         this.service = service;
     }
 
-    BufferedImage getLegend( GetLegendGraphic req ) {
+    BufferedImage getLegend( GetLegendGraphic req ) throws OWSException {
         Legends renderer = new Legends( req.getLegendOptions() );
 
         Style style = findLegendStyle( req.getLayer(), req.getStyle() );
@@ -122,12 +125,24 @@ Pair<Integer, Integer> getLegendSize( Style style ) {
         return res;
     }
 
-    private Style findLegendStyle( LayerRef layer, StyleRef styleRef ) {
+    private Style findLegendStyle( LayerRef layer, StyleRef styleRef )
+                            throws OWSException {
         Style style;
-        style = service.themeMap.get( layer.getName() ).getLayerMetadata().getLegendStyles().get( styleRef.getName() );
+        Theme theme = service.themeMap.get( layer.getName() );
+        if ( theme == null ) {
+            throw new OWSException( get( "WMS.LAYER_NOT_KNOWN", layer.getName() ), OWSException.LAYER_NOT_DEFINED );
+        }
+
+        style = theme.getLayerMetadata().getLegendStyles().get( styleRef.getName() );
+        if ( style == null ) {
+            style = theme.getLayerMetadata().getStyles().get( styleRef.getName() );
+        }
+
         if ( style == null ) {
-            style = service.themeMap.get( layer.getName() ).getLayerMetadata().getStyles().get( styleRef.getName() );
+            throw new OWSException( get( "WMS.UNDEFINED_STYLE", styleRef.getName(), layer.getName() ),
+                                    OWSException.STYLE_NOT_DEFINED );
         }
+
         return style;
     }
 

deegree-services/deegree-webservices-handbook/src/main/asciidoc/coveragestores.adoc

@@ -55,6 +55,13 @@ coverage sources. The RasterDirectory paramter can additionally have the
 recursive attribute with true and false as value to declare
 subdirectories to be included.
 
+WARNING: When using raster files, deegree creates on demand cache files. 
+Depending on the raster data used, the size of the cache files may vary.
+In individual cases, the use of cache files can be prevented by creating a 
+file _<filename>.no-cache_ or _<filename>.no-cache-<level>_ for whole files 
+or individual levels. Disabling the cache files can have a negative effect 
+on memory consumption. It is recommend to leave the cache enabled if possible.
+
 === MultiResolutionRaster
 
 A <MultiResolutionRaster> wraps single raster elements and adds a