Merge pull request #94 from GIScience/feature/update-gh-dataaccess

feat(gh update dataaccess): Update DataAccess to latest graphhoper version

core/pom.xml

@@ -189,8 +189,8 @@
                 <groupId>org.apache.maven.plugins</groupId>
                 <artifactId>maven-compiler-plugin</artifactId>
                 <configuration>
-                    <source>8</source>
-                    <target>8</target>
+                    <source>17</source>
+                    <target>17</target>
                 </configuration>
             </plugin>
         </plugins>

core/src/main/java/com/graphhopper/GraphHopper.java

@@ -96,7 +96,8 @@ public class GraphHopper {
     private EncodingManager encodingManager;
     private int defaultSegmentSize = -1;
     private String ghLocation = "";
-    private DAType dataAccessType = DAType.RAM_STORE;
+    private DAType dataAccessDefaultType = DAType.RAM_STORE;
+    private final LinkedHashMap<String, String> dataAccessConfig = new LinkedHashMap<>();
     private boolean sortGraph = false;
     private boolean elevation = false;
     private LockFactory lockFactory = new NativeFSLockFactory();
@@ -227,9 +228,9 @@ public GraphHopper setMinNetworkSize(int minNetworkSize) {
     public GraphHopper setStoreOnFlush(boolean storeOnFlush) {
         ensureNotLoaded();
         if (storeOnFlush)
-            dataAccessType = DAType.RAM_STORE;
+            dataAccessDefaultType = DAType.RAM_STORE;
         else
-            dataAccessType = DAType.RAM;
+            dataAccessDefaultType = DAType.RAM;
         return this;
     }
 
@@ -502,8 +503,14 @@ public GraphHopper init(GraphHopperConfig ghConfig) {
         setGraphHopperLocation(graphHopperFolder);
         defaultSegmentSize = ghConfig.getInt("graph.dataaccess.segment_size", defaultSegmentSize);
 
-        String graphDATypeStr = ghConfig.getString("graph.dataaccess", "RAM_STORE");
-        dataAccessType = DAType.fromString(graphDATypeStr);
+        String daTypeString = ghConfig.getString("graph.dataaccess.default_type", ghConfig.getString("graph.dataaccess", "RAM_STORE"));
+        dataAccessDefaultType = DAType.fromString(daTypeString);
+        for (Map.Entry<String, Object> entry : ghConfig.asPMap().toMap().entrySet()) {
+            if (entry.getKey().startsWith("graph.dataaccess.type."))
+                dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.type.".length()), entry.getValue().toString());
+            if (entry.getKey().startsWith("graph.dataaccess.mmap.preload."))
+                dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.mmap.".length()), entry.getValue().toString());
+        }
 
         sortGraph = ghConfig.getBool("graph.do_sort", sortGraph);
         removeZipped = ghConfig.getBool("graph.remove_zipped", removeZipped);
@@ -811,22 +818,24 @@ public boolean load(String graphHopperFolder) {
 
         setGraphHopperLocation(graphHopperFolder);
 
-        if (!allowWrites && dataAccessType.isMMap())
-            dataAccessType = DAType.MMAP_RO;
+        if (!allowWrites && dataAccessDefaultType.isMMap())
+            dataAccessDefaultType = DAType.MMAP_RO;
         if (encodingManager == null) {
-            StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessType));
+            StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessDefaultType));
             encodingManager = properties.loadExisting()
                     ? EncodingManager.create(emBuilder, encodedValueFactory, flagEncoderFactory, properties)
                     : buildEncodingManager(new GraphHopperConfig());
         }
 
-        GHDirectory dir = new GHDirectory(ghLocation, dataAccessType);
+        GHDirectory directory = new GHDirectory(ghLocation, dataAccessDefaultType);
+        directory.configure(dataAccessConfig);
 
         // ORS-GH MOD START use storage factory in ORSGraphHopper
         if (graphStorageFactory != null) {
-            ghStorage = graphStorageFactory.createStorage(dir, this);
+            ghStorage = graphStorageFactory.createStorage(directory, this);
         } else {
-            ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
+
+            ghStorage = new GraphHopperStorage(directory, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize);
         }
         // ORS-GH MOD END
         checkProfilesConsistency();
@@ -866,6 +875,7 @@ public boolean load(String graphHopperFolder) {
                 return false;
 
             postProcessing(false);
+            directory.loadMMap();
             setFullyLoaded();
             return true;
         } finally {

core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java

@@ -119,7 +119,7 @@ public double getEle(double lat, double lon) {
                     demProvider.setHeights(heights);
                     demProvider.setSeaLevel(true);
                     // use small size on disc and in-memory
-                    heights.setSegmentSize(100).create(10).
+                    heights.create(10).
                             flush();
                     return 0;
                 }

core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java

@@ -135,7 +135,7 @@ public double getEle(double lat, double lon) {
                 } catch (IOException e) {
                     demProvider.setSeaLevel(true);
                     // use small size on disc and in-memory
-                    heights.setSegmentSize(100).create(10).
+                    heights.create(10).
                             flush();
                     return 0;
                 }

core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java

@@ -141,7 +141,7 @@ public String toString() {
         // In this sense its even 'better' to use node-based.
         this.traversalMode = TraversalMode.NODE_BASED;
 // ORS-GH MOD START
-        this.landmarkWeightDA = dir.find(getLandmarksFileName() + lmConfig.getName());
+        this.landmarkWeightDA = dir.create(getLandmarksFileName() + lmConfig.getName());
 // ORS-GH MOD END
 
         this.landmarks = landmarks;
@@ -837,6 +837,14 @@ public interface LandmarkExplorer extends RoutingAlgorithm{
 
         void initLandmarkWeights(final int lmIdx, int lmNodeId, final long rowSize, final int offset);
     }
+
+    /**
+     * For testing only
+     */
+    DataAccess _getInternalDA() {
+        return landmarkWeightDA;
+    }
+
     /**
      * This class is used to calculate landmark location (equally distributed).
      * It derives from DijkstraBidirectionRef, but is only used as forward or backward search.

core/src/main/java/com/graphhopper/search/NameIndex.java

@@ -137,10 +137,6 @@ public boolean isClosed() {
         return names.isClosed();
     }
 
-    public void setSegmentSize(int segments) {
-        names.setSegmentSize(segments);
-    }
-
     public long getCapacity() {
         return names.getCapacity();
     }

core/src/main/java/com/graphhopper/search/StringIndex.java

@@ -50,17 +50,12 @@ public class StringIndex {
     private long lastEntryPointer = -1;
     private Map<String, String> lastEntryMap;
 
-    public StringIndex(Directory dir) {
-        this(dir, 1000);
-    }
-
     /**
      * Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object.
      */
-    public StringIndex(Directory dir, final int cacheSize) {
-        keys = dir.find("string_index_keys");
-        keys.setSegmentSize(10 * 1024);
-        vals = dir.find("string_index_vals");
+    public StringIndex(Directory dir, final int cacheSize, final int segmentSize) {
+        keys = dir.create("string_index_keys", segmentSize);
+        vals = dir.create("string_index_vals", segmentSize);
         smallCache = new LinkedHashMap<String, Long>(cacheSize, 0.75f, true) {
             @Override
             protected boolean removeEldestEntry(Map.Entry<String, Long> entry) {
@@ -196,7 +191,7 @@ public Map<String, String> getAll(final long entryPointer) {
         if (keyCount == 0)
             return Collections.emptyMap();
 
-        Map<String, String> map = new HashMap<>(keyCount);
+        Map<String, String> map = new LinkedHashMap<>(keyCount);
         long tmpPointer = entryPointer + 1;
         for (int i = 0; i < keyCount; i++) {
             int currentKeyIndex = vals.getShort(tmpPointer);
@@ -282,7 +277,7 @@ public String get(final long entryPointer, String key) {
             tmpPointer += 1 + valueLength;
         }
 
-        // value for specified key does not existing for the specified pointer
+        // value for specified key does not exist for the specified pointer
         return null;
     }
 
@@ -327,11 +322,6 @@ public boolean isClosed() {
         return vals.isClosed() && keys.isClosed();
     }
 
-    public void setSegmentSize(int segments) {
-        keys.setSegmentSize(segments);
-        vals.setSegmentSize(segments);
-    }
-
     public long getCapacity() {
         return vals.getCapacity() + keys.getCapacity();
     }

core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java

@@ -20,7 +20,6 @@
 import com.graphhopper.util.BitUtil;
 import com.graphhopper.util.Helper;
 
-import java.io.File;
 import java.io.IOException;
 import java.io.RandomAccessFile;
 import java.nio.ByteOrder;
@@ -34,24 +33,25 @@ public abstract class AbstractDataAccess implements DataAccess {
     protected static final int HEADER_OFFSET = 20 * 4 + 20;
     protected static final byte[] EMPTY = new byte[1024];
     private static final int SEGMENT_SIZE_DEFAULT = 1 << 20;
-    protected final ByteOrder byteOrder;
-    protected final BitUtil bitUtil;
+    protected final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN;
+    protected final BitUtil bitUtil = BitUtil.LITTLE;
     private final String location;
-    protected int header[] = new int[(HEADER_OFFSET - 20) / 4];
+    protected int[] header = new int[(HEADER_OFFSET - 20) / 4];
     protected String name;
-    protected int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT;
+    protected int segmentSizeInBytes;
     protected int segmentSizePower;
     protected int indexDivisor;
     protected boolean closed = false;
 
-    public AbstractDataAccess(String name, String location, ByteOrder order) {
-        byteOrder = order;
-        bitUtil = BitUtil.get(order);
+    public AbstractDataAccess(String name, String location, int segmentSize) {
         this.name = name;
         if (!Helper.isEmpty(location) && !location.endsWith("/"))
             throw new IllegalArgumentException("Create DataAccess object via its corresponding Directory!");
 
         this.location = location;
+        if (segmentSize < 0)
+            segmentSize = SEGMENT_SIZE_DEFAULT;
+        setSegmentSize(segmentSize);
     }
 
     @Override
@@ -105,7 +105,7 @@ protected long readHeader(RandomAccessFile raFile) throws IOException {
 
         String versionHint = raFile.readUTF();
         if (!"GH".equals(versionHint))
-            throw new IllegalArgumentException("Not a GraphHopper file! Expected 'GH' as file marker but was " + versionHint);
+            throw new IllegalArgumentException("Not a GraphHopper file " + getFullName() + "! Expected 'GH' as file marker but was " + versionHint);
 
         long bytes = raFile.readLong();
         setSegmentSize(raFile.readInt());
@@ -121,8 +121,7 @@ protected void copyHeader(DataAccess da) {
         }
     }
 
-    @Override
-    public DataAccess setSegmentSize(int bytes) {
+    DataAccess setSegmentSize(int bytes) {
         if (bytes > 0) {
             // segment size should be a power of 2
             int tmp = (int) (Math.log(bytes) / Math.log(2));

core/src/main/java/com/graphhopper/storage/BaseGraph.java

@@ -90,20 +90,16 @@ class BaseGraph implements Graph {
     public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) {
         this.dir = dir;
         this.intsForFlags = intsForFlags;
-        this.bitUtil = BitUtil.get(dir.getByteOrder());
-        this.wayGeometry = dir.find("geometry");
-        this.stringIndex = new StringIndex(dir);
-        this.nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType()));
-        this.edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType()));
+        this.bitUtil = BitUtil.LITTLE;
+        this.wayGeometry = dir.create("geometry", segmentSize);
+        this.stringIndex = new StringIndex(dir, 1000, segmentSize);
+        this.nodes = dir.create("nodes", dir.getDefaultType("nodes", true), segmentSize);
+        this.edges = dir.create("edges", dir.getDefaultType("edges", true), segmentSize);
         this.bounds = BBox.createInverse(withElevation);
         this.nodeAccess = new GHNodeAccess(this, withElevation);
-        if (withTurnCosts) {
-            turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs"));
-        } else {
-            turnCostStorage = null;
-        }
+        this.turnCostStorage = withTurnCosts ? new TurnCostStorage(this, dir.create("turn_costs", dir.getDefaultType("turn_costs", true), segmentSize)) : null;
         if (segmentSize >= 0) {
-            setSegmentSize(segmentSize);
+            checkNotInitialized();
         }
     }
 
@@ -330,17 +326,6 @@ public BBox getBounds() {
         return bounds;
     }
 
-    private void setSegmentSize(int bytes) {
-        checkNotInitialized();
-        nodes.setSegmentSize(bytes);
-        edges.setSegmentSize(bytes);
-        wayGeometry.setSegmentSize(bytes);
-        stringIndex.setSegmentSize(bytes);
-        if (supportsTurnCosts()) {
-            turnCostStorage.setSegmentSize(bytes);
-        }
-    }
-
     synchronized void freeze() {
         if (isFrozen())
             throw new IllegalStateException("base graph already frozen");

core/src/main/java/com/graphhopper/storage/CHStorage.java

@@ -75,14 +75,10 @@ public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased)
 
     public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased, String type) {
         this.isTypeCore =  CHConfig.TYPE_CORE.equals(type);
-        this.nodesCH = dir.find("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()));
-        this.shortcuts = dir.find("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()));
+        this.nodesCH = dir.create("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
+        this.shortcuts = dir.create("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize);
 // ORS-GH MOD END
         this.edgeBased = edgeBased;
-        if (segmentSize >= 0) {
-            nodesCH.setSegmentSize(segmentSize);
-            shortcuts.setSegmentSize(segmentSize);
-        }
         // shortcuts are stored consecutively using this layout (the last two entries only exist for edge-based):
         // NODEA | NODEB | WEIGHT | SKIP_EDGE1 | SKIP_EDGE2 | S_ORIG_FIRST | S_ORIG_LAST
         S_NODEA = 0;
@@ -124,6 +120,25 @@ public void create() {
         shortcuts.create(0);
     }
 
+    /**
+     * Creates a new storage. Alternatively we could load an existing one using {@link #loadExisting()}}.
+     * The number of nodes must be given here while the expected number of shortcuts can
+     * be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value
+     * so the resulting files/byte arrays won't be unnecessarily large.
+     * todo: we could also trim down the shortcuts DataAccess when we are done adding shortcuts
+     */
+    public void create(int nodes, int expectedShortcuts) {
+        if (nodeCount >= 0)
+            throw new IllegalStateException("CHStorage can only be created once");
+        if (nodes < 0)
+            throw new IllegalStateException("CHStorage must be created with a positive number of nodes");
+        nodesCH.create((long) nodes * nodeCHEntryBytes);
+        nodeCount = nodes;
+        for (int node = 0; node < nodes; node++)
+            setLastShortcut(toNodePointer(node), -1);
+        shortcuts.create((long) expectedShortcuts * shortcutEntryBytes);
+    }
+
     /**
      * Initializes the storage. The number of nodes must be given here while the expected number of shortcuts can
      * be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value

core/src/main/java/com/graphhopper/storage/DataAccess.java

@@ -124,12 +124,6 @@ public interface DataAccess extends Closeable {
      */
     int getSegmentSize();
 
-    /**
-     * In order to increase allocated space one needs to layout the underlying storage in segments.
-     * This is how you can customize the size.
-     */
-    DataAccess setSegmentSize(int bytes);
-
     /**
      * @return the number of segments.
      */