From f278a8ae7611edc79dade4c307b337bd26b7c96f Mon Sep 17 00:00:00 2001 From: Rafael Troilo Date: Tue, 19 Dec 2023 20:34:02 +0100 Subject: [PATCH] feat(gh-update): Update DataAccess to latest graphhoper version --- core/pom.xml | 4 +- .../java/com/graphhopper/GraphHopper.java | 32 ++- .../dem/AbstractSRTMElevationProvider.java | 2 +- .../dem/AbstractTiffElevationProvider.java | 2 +- .../routing/lm/LandmarkStorage.java | 10 +- .../com/graphhopper/search/NameIndex.java | 4 - .../com/graphhopper/search/StringIndex.java | 20 +- .../storage/AbstractDataAccess.java | 21 +- .../com/graphhopper/storage/BaseGraph.java | 29 +-- .../com/graphhopper/storage/CHStorage.java | 27 ++- .../com/graphhopper/storage/DataAccess.java | 6 - .../com/graphhopper/storage/Directory.java | 71 +++++- .../com/graphhopper/storage/GHDirectory.java | 145 ++++++++---- .../graphhopper/storage/MMapDataAccess.java | 219 +++++------------- .../graphhopper/storage/RAMDataAccess.java | 40 ++-- .../graphhopper/storage/RAMIntDataAccess.java | 36 ++- .../storage/StorableProperties.java | 21 +- .../graphhopper/storage/TurnCostStorage.java | 4 - .../storage/index/LocationIndexTree.java | 28 +-- .../java/com/graphhopper/util/GHUtility.java | 11 +- .../reader/dem/CGIARProviderTest.java | 2 +- .../reader/dem/GMTEDProviderTest.java | 2 +- .../routing/lm/LandmarkStorageTest.java | 11 +- .../graphhopper/search/StringIndexTest.java | 10 +- .../storage/AbstractDirectoryTester.java | 8 +- .../storage/AbstractGraphStorageTester.java | 5 +- .../graphhopper/storage/CHStorageTest.java | 21 +- .../graphhopper/storage/DataAccessTest.java | 34 +-- .../storage/MMapDataAccessTest.java | 8 +- .../storage/RAMDataAccessTest.java | 4 +- .../storage/RAMIntDataAccessTest.java | 4 +- 31 files changed, 417 insertions(+), 424 deletions(-) diff --git a/core/pom.xml b/core/pom.xml index df496d29cc2..ea03f52b6b1 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -189,8 +189,8 @@ org.apache.maven.plugins maven-compiler-plugin - 8 - 8 + 17 + 17 diff --git a/core/src/main/java/com/graphhopper/GraphHopper.java b/core/src/main/java/com/graphhopper/GraphHopper.java index a20230b1057..69f9efeabd5 100644 --- a/core/src/main/java/com/graphhopper/GraphHopper.java +++ b/core/src/main/java/com/graphhopper/GraphHopper.java @@ -96,7 +96,8 @@ public class GraphHopper { private EncodingManager encodingManager; private int defaultSegmentSize = -1; private String ghLocation = ""; - private DAType dataAccessType = DAType.RAM_STORE; + private DAType dataAccessDefaultType = DAType.RAM_STORE; + private final LinkedHashMap dataAccessConfig = new LinkedHashMap<>(); private boolean sortGraph = false; private boolean elevation = false; private LockFactory lockFactory = new NativeFSLockFactory(); @@ -227,9 +228,9 @@ public GraphHopper setMinNetworkSize(int minNetworkSize) { public GraphHopper setStoreOnFlush(boolean storeOnFlush) { ensureNotLoaded(); if (storeOnFlush) - dataAccessType = DAType.RAM_STORE; + dataAccessDefaultType = DAType.RAM_STORE; else - dataAccessType = DAType.RAM; + dataAccessDefaultType = DAType.RAM; return this; } @@ -502,8 +503,14 @@ public GraphHopper init(GraphHopperConfig ghConfig) { setGraphHopperLocation(graphHopperFolder); defaultSegmentSize = ghConfig.getInt("graph.dataaccess.segment_size", defaultSegmentSize); - String graphDATypeStr = ghConfig.getString("graph.dataaccess", "RAM_STORE"); - dataAccessType = DAType.fromString(graphDATypeStr); + String daTypeString = ghConfig.getString("graph.dataaccess.default_type", ghConfig.getString("graph.dataaccess", "RAM_STORE")); + dataAccessDefaultType = DAType.fromString(daTypeString); + for (Map.Entry entry : ghConfig.asPMap().toMap().entrySet()) { + if (entry.getKey().startsWith("graph.dataaccess.type.")) + dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.type.".length()), entry.getValue().toString()); + if (entry.getKey().startsWith("graph.dataaccess.mmap.preload.")) + dataAccessConfig.put(entry.getKey().substring("graph.dataaccess.mmap.".length()), entry.getValue().toString()); + } sortGraph = ghConfig.getBool("graph.do_sort", sortGraph); removeZipped = ghConfig.getBool("graph.remove_zipped", removeZipped); @@ -811,22 +818,24 @@ public boolean load(String graphHopperFolder) { setGraphHopperLocation(graphHopperFolder); - if (!allowWrites && dataAccessType.isMMap()) - dataAccessType = DAType.MMAP_RO; + if (!allowWrites && dataAccessDefaultType.isMMap()) + dataAccessDefaultType = DAType.MMAP_RO; if (encodingManager == null) { - StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessType)); + StorableProperties properties = new StorableProperties(new GHDirectory(ghLocation, dataAccessDefaultType)); encodingManager = properties.loadExisting() ? EncodingManager.create(emBuilder, encodedValueFactory, flagEncoderFactory, properties) : buildEncodingManager(new GraphHopperConfig()); } - GHDirectory dir = new GHDirectory(ghLocation, dataAccessType); + GHDirectory directory = new GHDirectory(ghLocation, dataAccessDefaultType); + directory.configure(dataAccessConfig); // ORS-GH MOD START use storage factory in ORSGraphHopper if (graphStorageFactory != null) { - ghStorage = graphStorageFactory.createStorage(dir, this); + ghStorage = graphStorageFactory.createStorage(directory, this); } else { - ghStorage = new GraphHopperStorage(dir, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize); + + ghStorage = new GraphHopperStorage(directory, encodingManager, hasElevation(), encodingManager.needsTurnCostsSupport(), defaultSegmentSize); } // ORS-GH MOD END checkProfilesConsistency(); @@ -866,6 +875,7 @@ public boolean load(String graphHopperFolder) { return false; postProcessing(false); + directory.loadMMap(); setFullyLoaded(); return true; } finally { diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java index 3a5e1075755..d6b599a2d8f 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractSRTMElevationProvider.java @@ -119,7 +119,7 @@ public double getEle(double lat, double lon) { demProvider.setHeights(heights); demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). + heights.create(10). flush(); return 0; } diff --git a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java index bf0a491a71b..0383d2af57b 100644 --- a/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java +++ b/core/src/main/java/com/graphhopper/reader/dem/AbstractTiffElevationProvider.java @@ -135,7 +135,7 @@ public double getEle(double lat, double lon) { } catch (IOException e) { demProvider.setSeaLevel(true); // use small size on disc and in-memory - heights.setSegmentSize(100).create(10). + heights.create(10). flush(); return 0; } diff --git a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java index 9c7dfc908ea..b9789cdc149 100644 --- a/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java +++ b/core/src/main/java/com/graphhopper/routing/lm/LandmarkStorage.java @@ -141,7 +141,7 @@ public String toString() { // In this sense its even 'better' to use node-based. this.traversalMode = TraversalMode.NODE_BASED; // ORS-GH MOD START - this.landmarkWeightDA = dir.find(getLandmarksFileName() + lmConfig.getName()); + this.landmarkWeightDA = dir.create(getLandmarksFileName() + lmConfig.getName()); // ORS-GH MOD END this.landmarks = landmarks; @@ -837,6 +837,14 @@ public interface LandmarkExplorer extends RoutingAlgorithm{ void initLandmarkWeights(final int lmIdx, int lmNodeId, final long rowSize, final int offset); } + + /** + * For testing only + */ + DataAccess _getInternalDA() { + return landmarkWeightDA; + } + /** * This class is used to calculate landmark location (equally distributed). * It derives from DijkstraBidirectionRef, but is only used as forward or backward search. diff --git a/core/src/main/java/com/graphhopper/search/NameIndex.java b/core/src/main/java/com/graphhopper/search/NameIndex.java index 67e9f113673..165afdf42f8 100644 --- a/core/src/main/java/com/graphhopper/search/NameIndex.java +++ b/core/src/main/java/com/graphhopper/search/NameIndex.java @@ -137,10 +137,6 @@ public boolean isClosed() { return names.isClosed(); } - public void setSegmentSize(int segments) { - names.setSegmentSize(segments); - } - public long getCapacity() { return names.getCapacity(); } diff --git a/core/src/main/java/com/graphhopper/search/StringIndex.java b/core/src/main/java/com/graphhopper/search/StringIndex.java index 50fab782462..6e9c71263b4 100644 --- a/core/src/main/java/com/graphhopper/search/StringIndex.java +++ b/core/src/main/java/com/graphhopper/search/StringIndex.java @@ -50,17 +50,12 @@ public class StringIndex { private long lastEntryPointer = -1; private Map lastEntryMap; - public StringIndex(Directory dir) { - this(dir, 1000); - } - /** * Specify a larger cacheSize to reduce disk usage. Note that this increases the memory usage of this object. */ - public StringIndex(Directory dir, final int cacheSize) { - keys = dir.find("string_index_keys"); - keys.setSegmentSize(10 * 1024); - vals = dir.find("string_index_vals"); + public StringIndex(Directory dir, final int cacheSize, final int segmentSize) { + keys = dir.create("string_index_keys", segmentSize); + vals = dir.create("string_index_vals", segmentSize); smallCache = new LinkedHashMap(cacheSize, 0.75f, true) { @Override protected boolean removeEldestEntry(Map.Entry entry) { @@ -196,7 +191,7 @@ public Map getAll(final long entryPointer) { if (keyCount == 0) return Collections.emptyMap(); - Map map = new HashMap<>(keyCount); + Map map = new LinkedHashMap<>(keyCount); long tmpPointer = entryPointer + 1; for (int i = 0; i < keyCount; i++) { int currentKeyIndex = vals.getShort(tmpPointer); @@ -282,7 +277,7 @@ public String get(final long entryPointer, String key) { tmpPointer += 1 + valueLength; } - // value for specified key does not existing for the specified pointer + // value for specified key does not exist for the specified pointer return null; } @@ -327,11 +322,6 @@ public boolean isClosed() { return vals.isClosed() && keys.isClosed(); } - public void setSegmentSize(int segments) { - keys.setSegmentSize(segments); - vals.setSegmentSize(segments); - } - public long getCapacity() { return vals.getCapacity() + keys.getCapacity(); } diff --git a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java index 081302f2d33..12f6ec6b1b9 100644 --- a/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/AbstractDataAccess.java @@ -20,7 +20,6 @@ import com.graphhopper.util.BitUtil; import com.graphhopper.util.Helper; -import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; import java.nio.ByteOrder; @@ -34,24 +33,25 @@ public abstract class AbstractDataAccess implements DataAccess { protected static final int HEADER_OFFSET = 20 * 4 + 20; protected static final byte[] EMPTY = new byte[1024]; private static final int SEGMENT_SIZE_DEFAULT = 1 << 20; - protected final ByteOrder byteOrder; - protected final BitUtil bitUtil; + protected final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; + protected final BitUtil bitUtil = BitUtil.LITTLE; private final String location; - protected int header[] = new int[(HEADER_OFFSET - 20) / 4]; + protected int[] header = new int[(HEADER_OFFSET - 20) / 4]; protected String name; - protected int segmentSizeInBytes = SEGMENT_SIZE_DEFAULT; + protected int segmentSizeInBytes; protected int segmentSizePower; protected int indexDivisor; protected boolean closed = false; - public AbstractDataAccess(String name, String location, ByteOrder order) { - byteOrder = order; - bitUtil = BitUtil.get(order); + public AbstractDataAccess(String name, String location, int segmentSize) { this.name = name; if (!Helper.isEmpty(location) && !location.endsWith("/")) throw new IllegalArgumentException("Create DataAccess object via its corresponding Directory!"); this.location = location; + if (segmentSize < 0) + segmentSize = SEGMENT_SIZE_DEFAULT; + setSegmentSize(segmentSize); } @Override @@ -105,7 +105,7 @@ protected long readHeader(RandomAccessFile raFile) throws IOException { String versionHint = raFile.readUTF(); if (!"GH".equals(versionHint)) - throw new IllegalArgumentException("Not a GraphHopper file! Expected 'GH' as file marker but was " + versionHint); + throw new IllegalArgumentException("Not a GraphHopper file " + getFullName() + "! Expected 'GH' as file marker but was " + versionHint); long bytes = raFile.readLong(); setSegmentSize(raFile.readInt()); @@ -121,8 +121,7 @@ protected void copyHeader(DataAccess da) { } } - @Override - public DataAccess setSegmentSize(int bytes) { + DataAccess setSegmentSize(int bytes) { if (bytes > 0) { // segment size should be a power of 2 int tmp = (int) (Math.log(bytes) / Math.log(2)); diff --git a/core/src/main/java/com/graphhopper/storage/BaseGraph.java b/core/src/main/java/com/graphhopper/storage/BaseGraph.java index 97273a63757..9658aaefe7e 100644 --- a/core/src/main/java/com/graphhopper/storage/BaseGraph.java +++ b/core/src/main/java/com/graphhopper/storage/BaseGraph.java @@ -90,20 +90,16 @@ class BaseGraph implements Graph { public BaseGraph(Directory dir, int intsForFlags, boolean withElevation, boolean withTurnCosts, int segmentSize) { this.dir = dir; this.intsForFlags = intsForFlags; - this.bitUtil = BitUtil.get(dir.getByteOrder()); - this.wayGeometry = dir.find("geometry"); - this.stringIndex = new StringIndex(dir); - this.nodes = dir.find("nodes", DAType.getPreferredInt(dir.getDefaultType())); - this.edges = dir.find("edges", DAType.getPreferredInt(dir.getDefaultType())); + this.bitUtil = BitUtil.LITTLE; + this.wayGeometry = dir.create("geometry", segmentSize); + this.stringIndex = new StringIndex(dir, 1000, segmentSize); + this.nodes = dir.create("nodes", dir.getDefaultType("nodes", true), segmentSize); + this.edges = dir.create("edges", dir.getDefaultType("edges", true), segmentSize); this.bounds = BBox.createInverse(withElevation); this.nodeAccess = new GHNodeAccess(this, withElevation); - if (withTurnCosts) { - turnCostStorage = new TurnCostStorage(this, dir.find("turn_costs")); - } else { - turnCostStorage = null; - } + this.turnCostStorage = withTurnCosts ? new TurnCostStorage(this, dir.create("turn_costs", dir.getDefaultType("turn_costs", true), segmentSize)) : null; if (segmentSize >= 0) { - setSegmentSize(segmentSize); + checkNotInitialized(); } } @@ -330,17 +326,6 @@ public BBox getBounds() { return bounds; } - private void setSegmentSize(int bytes) { - checkNotInitialized(); - nodes.setSegmentSize(bytes); - edges.setSegmentSize(bytes); - wayGeometry.setSegmentSize(bytes); - stringIndex.setSegmentSize(bytes); - if (supportsTurnCosts()) { - turnCostStorage.setSegmentSize(bytes); - } - } - synchronized void freeze() { if (isFrozen()) throw new IllegalStateException("base graph already frozen"); diff --git a/core/src/main/java/com/graphhopper/storage/CHStorage.java b/core/src/main/java/com/graphhopper/storage/CHStorage.java index 3f55107f9b4..90d97205fc6 100644 --- a/core/src/main/java/com/graphhopper/storage/CHStorage.java +++ b/core/src/main/java/com/graphhopper/storage/CHStorage.java @@ -75,14 +75,10 @@ public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased) public CHStorage(Directory dir, String name, int segmentSize, boolean edgeBased, String type) { this.isTypeCore = CHConfig.TYPE_CORE.equals(type); - this.nodesCH = dir.find("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType())); - this.shortcuts = dir.find("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType())); + this.nodesCH = dir.create("nodes_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize); + this.shortcuts = dir.create("shortcuts_" + type + "_" + name, DAType.getPreferredInt(dir.getDefaultType()), segmentSize); // ORS-GH MOD END this.edgeBased = edgeBased; - if (segmentSize >= 0) { - nodesCH.setSegmentSize(segmentSize); - shortcuts.setSegmentSize(segmentSize); - } // shortcuts are stored consecutively using this layout (the last two entries only exist for edge-based): // NODEA | NODEB | WEIGHT | SKIP_EDGE1 | SKIP_EDGE2 | S_ORIG_FIRST | S_ORIG_LAST S_NODEA = 0; @@ -124,6 +120,25 @@ public void create() { shortcuts.create(0); } + /** + * Creates a new storage. Alternatively we could load an existing one using {@link #loadExisting()}}. + * The number of nodes must be given here while the expected number of shortcuts can + * be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value + * so the resulting files/byte arrays won't be unnecessarily large. + * todo: we could also trim down the shortcuts DataAccess when we are done adding shortcuts + */ + public void create(int nodes, int expectedShortcuts) { + if (nodeCount >= 0) + throw new IllegalStateException("CHStorage can only be created once"); + if (nodes < 0) + throw new IllegalStateException("CHStorage must be created with a positive number of nodes"); + nodesCH.create((long) nodes * nodeCHEntryBytes); + nodeCount = nodes; + for (int node = 0; node < nodes; node++) + setLastShortcut(toNodePointer(node), -1); + shortcuts.create((long) expectedShortcuts * shortcutEntryBytes); + } + /** * Initializes the storage. The number of nodes must be given here while the expected number of shortcuts can * be given to prevent some memory allocations, but is not a requirement. When in doubt rather use a small value diff --git a/core/src/main/java/com/graphhopper/storage/DataAccess.java b/core/src/main/java/com/graphhopper/storage/DataAccess.java index 7b1f7962124..5bb8158ac07 100644 --- a/core/src/main/java/com/graphhopper/storage/DataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/DataAccess.java @@ -124,12 +124,6 @@ public interface DataAccess extends Closeable { */ int getSegmentSize(); - /** - * In order to increase allocated space one needs to layout the underlying storage in segments. - * This is how you can customize the size. - */ - DataAccess setSegmentSize(int bytes); - /** * @return the number of segments. */ diff --git a/core/src/main/java/com/graphhopper/storage/Directory.java b/core/src/main/java/com/graphhopper/storage/Directory.java index 6ae6d633755..cd1588f32a3 100644 --- a/core/src/main/java/com/graphhopper/storage/Directory.java +++ b/core/src/main/java/com/graphhopper/storage/Directory.java @@ -18,7 +18,7 @@ package com.graphhopper.storage; import java.nio.ByteOrder; -import java.util.Collection; +import java.util.Map; /** * Maintains a collection of DataAccess objects stored at the same location. One GraphStorage per @@ -36,27 +36,79 @@ public interface Directory { /** * @return the order in which the data is stored + * @deprecated */ - ByteOrder getByteOrder(); + @Deprecated + default ByteOrder getByteOrder() { + return ByteOrder.LITTLE_ENDIAN; + } /** - * Tries to find the object with that name if not existent it creates one and associates the - * location with it. A name is unique in one Directory. + * Creates a new DataAccess object with the given name in the location of this Directory. Each name can only + * be used once. */ - DataAccess find(String name); + DataAccess create(String name); - DataAccess find(String name, DAType type); + /** + * @deprecated use {@link #create(String)} instead. + */ + @Deprecated + default DataAccess find(String name) { + return create(name); + } + + /** + * @param segmentSize segment size in bytes or -1 to use the default of the corresponding DataAccess implementation + */ + DataAccess create(String name, int segmentSize); + + /** + * @deprecated use {@link #create(String, int)} instead. + */ + @Deprecated + default DataAccess find(String name, int segmentSize) { + return create(name, segmentSize); + } + + DataAccess create(String name, DAType type); + + /** + * @deprecated use {@link #create(String, DAType)} instead. + */ + @Deprecated + default DataAccess find(String name, DAType type) { + return create(name, type); + } + + DataAccess create(String name, DAType type, int segmentSize); + + /** + * @deprecated use {@link #create(String, DAType, int)} instead. + */ + @Deprecated + default DataAccess find(String name, DAType type, int segmentSize) { + return create(name, type, segmentSize); + } /** * Removes the specified object from the directory. */ - void remove(DataAccess da); + void remove(String name); + /** + * @deprecated use {@link #remove(String)} instead. + */ + @Deprecated + default void remove(DataAccess da) { + remove(da.getName()); + } /** * @return the default type of a newly created DataAccess object */ DAType getDefaultType(); + DAType getDefaultType(String dataAccess, boolean preferInts); + /** * Removes all contained objects from the directory and releases its resources. */ @@ -67,10 +119,5 @@ public interface Directory { */ void close(); - /** - * Returns all created directories. - */ - Collection getAll(); - Directory create(); } diff --git a/core/src/main/java/com/graphhopper/storage/GHDirectory.java b/core/src/main/java/com/graphhopper/storage/GHDirectory.java index 7934a629bfe..1024d10fc3d 100644 --- a/core/src/main/java/com/graphhopper/storage/GHDirectory.java +++ b/core/src/main/java/com/graphhopper/storage/GHDirectory.java @@ -18,11 +18,10 @@ package com.graphhopper.storage; import java.io.File; -import java.nio.ByteOrder; -import java.util.Collection; -import java.util.HashMap; -import java.util.Map; +import java.util.*; +import static com.graphhopper.storage.DAType.RAM_INT; +import static com.graphhopper.storage.DAType.RAM_INT_STORE; import static com.graphhopper.util.Helper.*; /** @@ -32,13 +31,14 @@ */ public class GHDirectory implements Directory { protected final String location; - private final DAType defaultType; - private final ByteOrder byteOrder = ByteOrder.LITTLE_ENDIAN; - protected Map map = new HashMap<>(); - protected Map types = new HashMap<>(); + private final DAType typeFallback; + // first rule matches => LinkedHashMap + private final Map defaultTypes = new LinkedHashMap<>(); + private final Map mmapPreloads = new LinkedHashMap<>(); + private final Map map = Collections.synchronizedMap(new HashMap<>()); public GHDirectory(String _location, DAType defaultType) { - this.defaultType = defaultType; + this.typeFallback = defaultType; if (isEmpty(_location)) _location = new File("").getAbsolutePath(); @@ -51,53 +51,98 @@ public GHDirectory(String _location, DAType defaultType) { throw new RuntimeException("file '" + dir + "' exists but is not a directory"); } - @Override - public ByteOrder getByteOrder() { - return byteOrder; + /** + * Configure the DAType (specified by the value) of a single DataAccess object (specified by the key). For "MMAP" you + * can prepend "preload." to the name and specify a percentage which preloads the DataAccess into physical memory of + * the specified percentage (only applied for load, not for import). + * As keys can be patterns the order is important and the LinkedHashMap is forced as type. + */ + public Directory configure(LinkedHashMap config) { + for (Map.Entry kv : config.entrySet()) { + String value = kv.getValue().trim(); + if (kv.getKey().startsWith("preload.")) + try { + String pattern = kv.getKey().substring("preload.".length()); + mmapPreloads.put(pattern, Integer.parseInt(value)); + } catch (NumberFormatException ex) { + throw new IllegalArgumentException("DataAccess " + kv.getKey() + " has an incorrect preload value: " + value); + } + else { + String pattern = kv.getKey(); + defaultTypes.put(pattern, DAType.fromString(value)); + } + } + return this; } - public Directory put(String name, DAType type) { - if (!name.equals(toLowerCase(name))) - throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); + /** + * Returns the preload value or 0 if no patterns match. + * See {@link #configure(LinkedHashMap)} + */ + int getPreload(String name) { + for (Map.Entry entry : mmapPreloads.entrySet()) + if (name.matches(entry.getKey())) return entry.getValue(); + return 0; + } - types.put(name, type); - return this; + public void loadMMap() { + for (DataAccess da : map.values()) { + if (!(da instanceof MMapDataAccess)) + continue; + int preload = getPreload(da.getName()); + if (preload > 0) + ((MMapDataAccess) da).load(preload); + } + } + + @Override + public DataAccess create(String name) { + return create(name, getDefault(name, typeFallback)); } @Override - public DataAccess find(String name) { - DAType type = types.get(name); - if (type == null) - type = defaultType; + public DataAccess create(String name, int segmentSize) { + return create(name, getDefault(name, typeFallback), segmentSize); + } + + private DAType getDefault(String name, DAType typeFallback) { + for (Map.Entry entry : defaultTypes.entrySet()) + if (name.matches(entry.getKey())) return entry.getValue(); + return typeFallback; + } - return find(name, type); + @Override + public DataAccess create(String name, DAType type) { + return create(name, type, -1); } @Override - public DataAccess find(String name, DAType type) { + public DataAccess create(String name, DAType type, int segmentSize) { if (!name.equals(toLowerCase(name))) throw new IllegalArgumentException("Since 0.7 DataAccess objects does no longer accept upper case names"); - DataAccess da = map.get(name); - if (da != null) { - if (!type.equals(da.getType())) - throw new IllegalStateException("Found existing DataAccess object '" + name - + "' but types did not match. Requested:" + type + ", was:" + da.getType()); - return da; - } + if (map.containsKey(name)) + // we do not allow creating two DataAccess with the same name, because on disk there can only be one DA + // per file name + try { + throw new IllegalStateException("DataAccess " + name + " has already been created"); + } catch (Exception e){ + throw e; + } + DataAccess da; if (type.isInMemory()) { if (type.isInteg()) { if (type.isStoring()) - da = new RAMIntDataAccess(name, location, true, byteOrder); + da = new RAMIntDataAccess(name, location, true, segmentSize); else - da = new RAMIntDataAccess(name, location, false, byteOrder); + da = new RAMIntDataAccess(name, location, false, segmentSize); } else if (type.isStoring()) - da = new RAMDataAccess(name, location, true, byteOrder); + da = new RAMDataAccess(name, location, true, segmentSize); else - da = new RAMDataAccess(name, location, false, byteOrder); + da = new RAMDataAccess(name, location, false, segmentSize); } else if (type.isMMap()) { - da = new MMapDataAccess(name, location, byteOrder, type.isAllowWrites()); + da = new MMapDataAccess(name, location, type.isAllowWrites(), segmentSize); } else { throw new IllegalArgumentException("DAType not supported " + type); } @@ -124,13 +169,13 @@ public void clear() { } @Override - public void remove(DataAccess da) { - DataAccess old = map.remove(da.getName()); + public void remove(String name) { + DataAccess old = map.remove(name); if (old == null) - throw new IllegalStateException("Couldn't remove DataAccess: " + da.getName()); + throw new IllegalStateException("Couldn't remove DataAccess: " + name); - da.close(); - removeBackingFile(da, da.getName()); + old.close(); + removeBackingFile(old, name); } private void removeBackingFile(DataAccess da, String name) { @@ -140,11 +185,22 @@ private void removeBackingFile(DataAccess da, String name) { @Override public DAType getDefaultType() { - return defaultType; + return typeFallback; + } + + /** + * This method returns the default DAType of the specified DataAccess (as string). If preferInts is true then this + * method returns e.g. RAM_INT if the type of the specified DataAccess is RAM. + */ + public DAType getDefaultType(String dataAccess, boolean preferInts) { + DAType type = getDefault(dataAccess, typeFallback); + if (preferInts && type.isInMemory()) + return type.isStoring() ? RAM_INT_STORE : RAM_INT; + return type; } public boolean isStoring() { - return defaultType.isStoring(); + return typeFallback.isStoring(); } @Override @@ -154,11 +210,6 @@ public Directory create() { return this; } - @Override - public Collection getAll() { - return map.values(); - } - @Override public String toString() { return getLocation(); diff --git a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java index 25eadfd2235..3b681f6badf 100644 --- a/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/MMapDataAccess.java @@ -17,7 +17,6 @@ */ package com.graphhopper.storage; -import com.graphhopper.util.Constants; import com.graphhopper.util.Helper; import org.slf4j.Logger; import org.slf4j.LoggerFactory; @@ -26,18 +25,12 @@ import java.io.IOException; import java.io.RandomAccessFile; import java.lang.reflect.Field; -import java.lang.reflect.InvocationTargetException; import java.lang.reflect.Method; import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.MappedByteBuffer; import java.nio.channels.FileChannel; -import java.security.AccessController; -import java.security.PrivilegedActionException; -import java.security.PrivilegedExceptionAction; import java.util.ArrayList; import java.util.List; -import java.util.StringTokenizer; /** * A DataAccess implementation using a memory-mapped file, i.e. a facility of the @@ -61,103 +54,36 @@ public final class MMapDataAccess extends AbstractDataAccess { private final boolean allowWrites; private RandomAccessFile raFile; - private List segments = new ArrayList<>(); + private final List segments = new ArrayList<>(); - MMapDataAccess(String name, String location, ByteOrder order, boolean allowWrites) { - super(name, location, order); + MMapDataAccess(String name, String location, boolean allowWrites, int segmentSize) { + super(name, location, segmentSize); this.allowWrites = allowWrites; } - public static boolean jreIsMinimumJava9() { - final StringTokenizer st = new StringTokenizer(System.getProperty("java.specification.version"), "."); - int JVM_MAJOR_VERSION = Integer.parseInt(st.nextToken()); - int JVM_MINOR_VERSION; - if (st.hasMoreTokens()) { - JVM_MINOR_VERSION = Integer.parseInt(st.nextToken()); - } else { - JVM_MINOR_VERSION = 0; - } - return JVM_MAJOR_VERSION > 1 || (JVM_MAJOR_VERSION == 1 && JVM_MINOR_VERSION >= 9); - } - public static void cleanMappedByteBuffer(final ByteBuffer buffer) { // TODO avoid reflection on every call try { - AccessController.doPrivileged(new PrivilegedExceptionAction() { - @Override - public Object run() throws Exception { - if (jreIsMinimumJava9()) { - // >=JDK9 class sun.misc.Unsafe { void invokeCleaner(ByteBuffer buf) } - final Class unsafeClass = Class.forName("sun.misc.Unsafe"); - // fetch the unsafe instance and bind it to the virtual MethodHandle - final Field f = unsafeClass.getDeclaredField("theUnsafe"); - f.setAccessible(true); - final Object theUnsafe = f.get(null); - final Method method = unsafeClass.getDeclaredMethod("invokeCleaner", ByteBuffer.class); - try { - method.invoke(theUnsafe, buffer); - return null; - } catch (Throwable t) { - throw new RuntimeException(t); - } - } - - if (buffer.getClass().getSimpleName().equals("MappedByteBufferAdapter")) { - if (!Constants.ANDROID) - throw new RuntimeException("MappedByteBufferAdapter only supported for Android at the moment"); - - // For Android 4.1 call ((MappedByteBufferAdapter)buffer).free() see #914 - Class directByteBufferClass = Class.forName("java.nio.MappedByteBufferAdapter"); - callBufferFree(buffer, directByteBufferClass); - } else { - // <=JDK8 class DirectByteBuffer { sun.misc.Cleaner cleaner(Buffer buf) } - // then call sun.misc.Cleaner.clean - final Class directByteBufferClass = Class.forName("java.nio.DirectByteBuffer"); - try { - final Method dbbCleanerMethod = directByteBufferClass.getMethod("cleaner"); - dbbCleanerMethod.setAccessible(true); - // call: cleaner = ((DirectByteBuffer)buffer).cleaner() - final Object cleaner = dbbCleanerMethod.invoke(buffer); - if (cleaner != null) { - final Class cleanerMethodReturnType = dbbCleanerMethod.getReturnType(); - final Method cleanMethod = cleanerMethodReturnType.getDeclaredMethod("clean"); - cleanMethod.setAccessible(true); - // call: ((sun.misc.Cleaner)cleaner).clean() - cleanMethod.invoke(cleaner); - } - } catch (NoSuchMethodException ex2) { - if (Constants.ANDROID) - // For Android 5.1.1 call ((DirectByteBuffer)buffer).free() see #933 - callBufferFree(buffer, directByteBufferClass); - else - // ignore if method cleaner or clean is not available - LOGGER.warn("NoSuchMethodException | " + System.getProperty("java.version"), ex2); - } - } - - return null; - } - }); - } catch (PrivilegedActionException e) { - throw new RuntimeException("Unable to unmap the mapped buffer", e); - } - } - - private static void callBufferFree(ByteBuffer buffer, Class directByteBufferClass) - throws InvocationTargetException, IllegalAccessException { - try { - final Method dbbFreeMethod = directByteBufferClass.getMethod("free"); - dbbFreeMethod.setAccessible(true); - dbbFreeMethod.invoke(buffer); - } catch (NoSuchMethodException ex2) { - LOGGER.warn("NoSuchMethodException | " + System.getProperty("java.version"), ex2); + // >=JDK9 class sun.misc.Unsafe { void invokeCleaner(ByteBuffer buf) } + final Class unsafeClass = Class.forName("sun.misc.Unsafe"); + // fetch the unsafe instance and bind it to the virtual MethodHandle + final Field f = unsafeClass.getDeclaredField("theUnsafe"); + f.setAccessible(true); + final Object theUnsafe = f.get(null); + final Method method = unsafeClass.getDeclaredMethod("invokeCleaner", ByteBuffer.class); + try { + method.invoke(theUnsafe, buffer); + } catch (Throwable t) { + throw new RuntimeException(t); + } + } catch (Exception ex) { + throw new RuntimeException("Unable to unmap the mapped buffer", ex); } } private void initRandomAccessFile() { - if (raFile != null) { + if (raFile != null) return; - } try { // raFile necessary for loadExisting and create @@ -174,7 +100,6 @@ public MMapDataAccess create(long bytes) { } initRandomAccessFile(); bytes = Math.max(10 * 4, bytes); - setSegmentSize(segmentSizeInBytes); ensureCapacity(bytes); return this; } @@ -309,6 +234,18 @@ public void flush() { } } + /** + * Load memory mapped files into physical memory. + */ + public void load(int percentage) { + if (percentage < 0 || percentage > 100) + throw new IllegalArgumentException("Percentage for MMapDataAccess.load for " + getName() + " must be in [0,100] but was " + percentage); + int max = Math.round(segments.size() * percentage / 100f); + for (int i = 0; i < max; i++) { + segments.get(i).load(); + } + } + @Override public void close() { super.close(); @@ -318,65 +255,50 @@ public void close() { } @Override - public final void setInt(long bytePos, int value) { + public void setInt(long bytePos, int value) { int bufferIndex = (int) (bytePos >> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 4 > segmentSizeInBytes) throw new IllegalStateException("Padding required. Currently an int cannot be distributed over two segments. " + bytePos); ByteBuffer byteBuffer = segments.get(bufferIndex); - synchronized (byteBuffer) { - byteBuffer.putInt(index, value); - } + byteBuffer.putInt(index, value); } @Override - public final int getInt(long bytePos) { + public int getInt(long bytePos) { int bufferIndex = (int) (bytePos >> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 4 > segmentSizeInBytes) throw new IllegalStateException("Padding required. Currently an int cannot be distributed over two segments. " + bytePos); ByteBuffer byteBuffer = segments.get(bufferIndex); - synchronized (byteBuffer) { - return byteBuffer.getInt(index); - } + return byteBuffer.getInt(index); } @Override - public final void setShort(long bytePos, short value) { + public void setShort(long bytePos, short value) { int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); ByteBuffer byteBuffer = segments.get(bufferIndex); - synchronized (byteBuffer) { - if (index + 2 > segmentSizeInBytes) { - ByteBuffer byteBufferNext = segments.get(bufferIndex + 1); - synchronized (byteBufferNext) { - // special case if short has to be written into two separate segments - byteBuffer.put(index, (byte) value); - byteBufferNext.put(0, (byte) (value >>> 8)); - } - } else { - byteBuffer.putShort(index, value); - } + if (index + 2 > segmentSizeInBytes) { + ByteBuffer byteBufferNext = segments.get(bufferIndex + 1); + // special case if short has to be written into two separate segments + byteBuffer.put(index, (byte) value); + byteBufferNext.put(0, (byte) (value >>> 8)); + } else { + byteBuffer.putShort(index, value); } } @Override - public final short getShort(long bytePos) { + public short getShort(long bytePos) { int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); ByteBuffer byteBuffer = segments.get(bufferIndex); if (index + 2 > segmentSizeInBytes) { ByteBuffer byteBufferNext = segments.get(bufferIndex + 1); - // never lock byteBuffer and byteBufferNext in a different order to avoid deadlocks (shouldn't happen) - synchronized (byteBuffer) { - synchronized (byteBufferNext) { - return (short) ((byteBufferNext.get(0) & 0xFF) << 8 | byteBuffer.get(index) & 0xFF); - } - } - } - synchronized (byteBuffer) { - return byteBuffer.getShort(index); + return (short) ((byteBufferNext.get(0) & 0xFF) << 8 | byteBuffer.get(index) & 0xFF); } + return byteBuffer.getShort(index); } @Override @@ -386,21 +308,15 @@ public void setBytes(long bytePos, byte[] values, int length) { final int index = (int) (bytePos & indexDivisor); final int delta = index + length - segmentSizeInBytes; final ByteBuffer bb1 = segments.get(bufferIndex); - synchronized (bb1) { - bb1.position(index); - if (delta > 0) { - length -= delta; - bb1.put(values, 0, length); - } else { - bb1.put(values, 0, length); - } + if (delta > 0) { + length -= delta; + bb1.put(index, values, 0, length); + } else { + bb1.put(index, values, 0, length); } if (delta > 0) { final ByteBuffer bb2 = segments.get(bufferIndex + 1); - synchronized (bb2) { - bb2.position(0); - bb2.put(values, length, delta); - } + bb2.put(0, values, length, delta); } } @@ -411,21 +327,14 @@ public void getBytes(long bytePos, byte[] values, int length) { int index = (int) (bytePos & indexDivisor); int delta = index + length - segmentSizeInBytes; final ByteBuffer bb1 = segments.get(bufferIndex); - synchronized (bb1) { - bb1.position(index); - if (delta > 0) { - length -= delta; - bb1.get(values, 0, length); - } else { - bb1.get(values, 0, length); - } - } if (delta > 0) { + length -= delta; + bb1.get(index, values, 0, length); + final ByteBuffer bb2 = segments.get(bufferIndex + 1); - synchronized (bb2) { - bb2.position(0); - bb2.get(values, length, delta); - } + bb2.get(0, values, length, delta); + } else { + bb1.get(index, values, 0, length); } } @@ -434,10 +343,7 @@ public void setByte(long bytePos, byte value) { int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); final ByteBuffer bb1 = segments.get(bufferIndex); - synchronized (bb1) { - bb1.position(index); - bb1.put(value); - } + bb1.put(index, value); } @Override @@ -445,19 +351,14 @@ public byte getByte(long bytePos) { int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); final ByteBuffer bb1 = segments.get(bufferIndex); - synchronized (bb1) { - bb1.position(index); - return bb1.get(); - } + return bb1.get(index); } @Override public long getCapacity() { long cap = 0; for (ByteBuffer bb : segments) { - synchronized (bb) { - cap += bb.capacity(); - } + cap += bb.capacity(); } return cap; } diff --git a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java index 124aaedeebb..409b9a8d5fe 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMDataAccess.java @@ -20,12 +20,11 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteOrder; import java.util.Arrays; /** * This is an in-memory byte-based data structure with the possibility to be stored on flush(). - * Thread safe. + * Read thread-safe. *

* * @author Peter Karich @@ -34,8 +33,8 @@ public class RAMDataAccess extends AbstractDataAccess { private byte[][] segments = new byte[0][]; private boolean store; - RAMDataAccess(String name, String location, boolean store, ByteOrder order) { - super(name, location, order); + RAMDataAccess(String name, String location, boolean store, int segmentSize) { + super(name, location, segmentSize); this.store = store; } @@ -57,7 +56,6 @@ public RAMDataAccess create(long bytes) { if (segments.length > 0) throw new IllegalThreadStateException("already created"); - setSegmentSize(segmentSizeInBytes); ensureCapacity(Math.max(10 * 4, bytes)); return this; } @@ -84,8 +82,8 @@ public boolean ensureCapacity(long bytes) { segments = newSegs; } catch (OutOfMemoryError err) { throw new OutOfMemoryError(err.getMessage() + " - problem when allocating new memory. Old capacity: " - + cap + ", new bytes:" + newBytes + ", segmentSizeIntsPower:" + segmentSizePower - + ", new segments:" + segmentsToCreate + ", existing:" + segments.length); + + cap + ", new bytes:" + newBytes + ", segmentSizeIntsPower:" + segmentSizePower + + ", new segments:" + segmentsToCreate + ", existing:" + segments.length); } return true; } @@ -106,14 +104,13 @@ public boolean loadExisting() { return false; try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r")) { long byteCount = readHeader(raFile) - HEADER_OFFSET; if (byteCount < 0) return false; raFile.seek(HEADER_OFFSET); - // raFile.readInt() <- too slow + // raFile.readInt() <- too slow int segmentCount = (int) (byteCount / segmentSizeInBytes); if (byteCount % segmentSizeInBytes != 0) segmentCount++; @@ -128,8 +125,6 @@ public boolean loadExisting() { segments[s] = bytes; } return true; - } finally { - raFile.close(); } } catch (IOException ex) { throw new RuntimeException("Problem while loading " + getFullName(), ex); @@ -145,18 +140,15 @@ public void flush() { return; try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw")) { long len = getCapacity(); writeHeader(raFile, len, segmentSizeInBytes); raFile.seek(HEADER_OFFSET); // raFile.writeInt() <- too slow, so copy into byte array for (int s = 0; s < segments.length; s++) { - byte area[] = segments[s]; + byte[] area = segments[s]; raFile.write(area); } - } finally { - raFile.close(); } } catch (Exception ex) { throw new RuntimeException("Couldn't store bytes to " + toString(), ex); @@ -175,7 +167,7 @@ public final void setInt(long bytePos, int value) { @Override public final int getInt(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 4 > segmentSizeInBytes) @@ -185,7 +177,7 @@ public final int getInt(long bytePos) { @Override public final void setShort(long bytePos, short value) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 2 > segmentSizeInBytes) { @@ -199,7 +191,7 @@ public final void setShort(long bytePos, short value) { @Override public final short getShort(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); if (index + 2 > segmentSizeInBytes) @@ -211,7 +203,7 @@ public final short getShort(long bytePos) { @Override public void setBytes(long bytePos, byte[] values, int length) { assert length <= segmentSizeInBytes : "the length has to be smaller or equal to the segment size: " + length + " vs. " + segmentSizeInBytes; - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); byte[] seg = segments[bufferIndex]; @@ -229,7 +221,7 @@ public void setBytes(long bytePos, byte[] values, int length) { @Override public void getBytes(long bytePos, byte[] values, int length) { assert length <= segmentSizeInBytes : "the length has to be smaller or equal to the segment size: " + length + " vs. " + segmentSizeInBytes; - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); byte[] seg = segments[bufferIndex]; @@ -246,7 +238,7 @@ public void getBytes(long bytePos, byte[] values, int length) { @Override public final void setByte(long bytePos, byte value) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); segments[bufferIndex][index] = value; @@ -254,7 +246,7 @@ public final void setByte(long bytePos, byte value) { @Override public final byte getByte(long bytePos) { - assert segmentSizePower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; int bufferIndex = (int) (bytePos >>> segmentSizePower); int index = (int) (bytePos & indexDivisor); return segments[bufferIndex][index]; diff --git a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java index 5e8bfa1b36a..5f3f7e743df 100644 --- a/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java +++ b/core/src/main/java/com/graphhopper/storage/RAMIntDataAccess.java @@ -20,7 +20,6 @@ import java.io.File; import java.io.IOException; import java.io.RandomAccessFile; -import java.nio.ByteOrder; import java.util.Arrays; /** @@ -36,8 +35,8 @@ class RAMIntDataAccess extends AbstractDataAccess { private boolean store; private int segmentSizeIntsPower; - RAMIntDataAccess(String name, String location, boolean store, ByteOrder order) { - super(name, location, order); + RAMIntDataAccess(String name, String location, boolean store, int segmentSize) { + super(name, location, segmentSize); this.store = store; } @@ -59,7 +58,6 @@ public RAMIntDataAccess create(long bytes) { if (segments.length > 0) throw new IllegalThreadStateException("already created"); - setSegmentSize(segmentSizeInBytes); ensureCapacity(Math.max(10 * 4, bytes)); return this; } @@ -87,8 +85,8 @@ public boolean ensureCapacity(long bytes) { return true; } catch (OutOfMemoryError err) { throw new OutOfMemoryError(err.getMessage() + " - problem when allocating new memory. Old capacity: " - + cap + ", new bytes:" + newBytes + ", segmentSizeIntsPower:" + segmentSizeIntsPower - + ", new segments:" + segmentsToCreate + ", existing:" + segments.length); + + cap + ", new bytes:" + newBytes + ", segmentSizeIntsPower:" + segmentSizeIntsPower + + ", new segments:" + segmentsToCreate + ", existing:" + segments.length); } } @@ -108,15 +106,14 @@ public boolean loadExisting() { return false; } try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "r")) { long byteCount = readHeader(raFile) - HEADER_OFFSET; if (byteCount < 0) { return false; } byte[] bytes = new byte[segmentSizeInBytes]; raFile.seek(HEADER_OFFSET); - // raFile.readInt() <- too slow + // raFile.readInt() <- too slow int segmentCount = (int) (byteCount / segmentSizeInBytes); if (byteCount % segmentSizeInBytes != 0) segmentCount++; @@ -124,15 +121,13 @@ public boolean loadExisting() { segments = new int[segmentCount][]; for (int s = 0; s < segmentCount; s++) { int read = raFile.read(bytes) / 4; - int area[] = new int[read]; + int[] area = new int[read]; for (int j = 0; j < read; j++) { area[j] = bitUtil.toInt(bytes, j * 4); } segments[s] = area; } return true; - } finally { - raFile.close(); } } catch (IOException ex) { throw new RuntimeException("Problem while loading " + getFullName(), ex); @@ -148,14 +143,13 @@ public void flush() { return; } try { - RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw"); - try { + try (RandomAccessFile raFile = new RandomAccessFile(getFullName(), "rw")) { long len = getCapacity(); writeHeader(raFile, len, segmentSizeInBytes); raFile.seek(HEADER_OFFSET); // raFile.writeInt() <- too slow, so copy into byte array for (int s = 0; s < segments.length; s++) { - int area[] = segments[s]; + int[] area = segments[s]; int intLen = area.length; byte[] byteArea = new byte[intLen * 4]; for (int i = 0; i < intLen; i++) { @@ -163,8 +157,6 @@ public void flush() { } raFile.write(byteArea); } - } finally { - raFile.close(); } } catch (Exception ex) { throw new RuntimeException("Couldn't store integers to " + toString(), ex); @@ -173,7 +165,7 @@ public void flush() { @Override public final void setInt(long bytePos, int value) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; bytePos >>>= 2; int bufferIndex = (int) (bytePos >>> segmentSizeIntsPower); int index = (int) (bytePos & indexDivisor); @@ -182,7 +174,7 @@ public final void setInt(long bytePos, int value) { @Override public final int getInt(long bytePos) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; bytePos >>>= 2; int bufferIndex = (int) (bytePos >>> segmentSizeIntsPower); int index = (int) (bytePos & indexDivisor); @@ -191,7 +183,7 @@ public final int getInt(long bytePos) { @Override public final void setShort(long bytePos, short value) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; if (bytePos % 4 != 0 && bytePos % 4 != 2) throw new IllegalMonitorStateException("bytePos of wrong multiple for RAMInt " + bytePos); @@ -207,7 +199,7 @@ public final void setShort(long bytePos, short value) { @Override public final short getShort(long bytePos) { - assert segmentSizeIntsPower > 0 : "call create or loadExisting before usage!"; + assert segments.length > 0 : "call create or loadExisting before usage!"; if (bytePos % 4 != 0 && bytePos % 4 != 2) throw new IllegalMonitorStateException("bytePos of wrong multiple for RAMInt " + bytePos); @@ -258,7 +250,7 @@ public int getSegments() { } @Override - public DataAccess setSegmentSize(int bytes) { + DataAccess setSegmentSize(int bytes) { super.setSegmentSize(bytes); segmentSizeIntsPower = (int) (Math.log(segmentSizeInBytes / 4) / Math.log(2)); indexDivisor = segmentSizeInBytes / 4 - 1; diff --git a/core/src/main/java/com/graphhopper/storage/StorableProperties.java b/core/src/main/java/com/graphhopper/storage/StorableProperties.java index e798ca75b49..9af52d7299f 100644 --- a/core/src/main/java/com/graphhopper/storage/StorableProperties.java +++ b/core/src/main/java/com/graphhopper/storage/StorableProperties.java @@ -41,9 +41,9 @@ public class StorableProperties { private final DataAccess da; public StorableProperties(Directory dir) { - this.da = dir.find("properties"); // reduce size - da.setSegmentSize(1 << 15); + int segmentSize = 1 << 15; + this.da = dir.create("properties", segmentSize); } public synchronized boolean loadExisting() { @@ -103,12 +103,11 @@ public synchronized StorableProperties put(String key, Object val) { public synchronized String get(String key) { if (!key.equals(toLowerCase(key))) throw new IllegalArgumentException("Do not use upper case keys (" + key + ") for StorableProperties since 0.7"); + return map.getOrDefault(key, ""); + } - String ret = map.get(key); - if (ret == null) - return ""; - - return ret; + public synchronized Map getAll() { + return map; } public synchronized void close() { @@ -179,6 +178,14 @@ boolean check(String key, int vers, boolean silent) { return true; } + public synchronized boolean containsVersion() { + return map.containsKey("nodes.version") || + map.containsKey("edges.version") || + map.containsKey("geometry.version") || + map.containsKey("location_index.version") || + map.containsKey("string_index.version"); + } + @Override public synchronized String toString() { return da.toString(); diff --git a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java index a362a02f120..44ff63b77d9 100644 --- a/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java +++ b/core/src/main/java/com/graphhopper/storage/TurnCostStorage.java @@ -54,10 +54,6 @@ public TurnCostStorage(BaseGraph baseGraph, DataAccess turnCosts) { this.turnCosts = turnCosts; } - public void setSegmentSize(int bytes) { - turnCosts.setSegmentSize(bytes); - } - public TurnCostStorage create(long initBytes) { turnCosts.create(initBytes); return this; diff --git a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java index 0c69c8ca65e..41c14f11919 100644 --- a/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java +++ b/core/src/main/java/com/graphhopper/storage/index/LocationIndexTree.java @@ -80,6 +80,18 @@ public LocationIndexTree(Graph g, Directory dir) { this.graph = g; this.nodeAccess = g.getNodeAccess(); this.directory = dir; + + // Clone this defensively -- In case something funny happens and things get added to the Graph after + // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we + // need it to be immutable. + BBox bounds = graph.getBounds().clone(); + + // I want to be able to create a location index for the empty graph without error, but for that + // I need valid bounds so that the initialization logic works. + if (!bounds.isValid()) + bounds = new BBox(-10.0, 10.0, -10.0, 10.0); + + lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); } public int getMinResolutionInMeter() { @@ -120,18 +132,6 @@ public LocationIndex setResolution(int minResolutionInMeter) { } public boolean loadExisting() { - // Clone this defensively -- In case something funny happens and things get added to the Graph after - // this index is built. Reason is that the expected structure of the index is a function of the bbox, so we - // need it to be immutable. - BBox bounds = graph.getBounds().clone(); - - // I want to be able to create a location index for the empty graph without error, but for that - // I need valid bounds so that the initialization logic works. - if (!bounds.isValid()) - bounds = new BBox(-10.0,10.0,-10.0,10.0); - - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); - if (!lineIntIndex.loadExisting()) return false; @@ -166,11 +166,10 @@ public LocationIndex prepareIndex(EdgeFilter edgeFilter) { // I want to be able to create a location index for the empty graph without error, but for that // I need valid bounds so that the initialization logic works. if (!bounds.isValid()) - bounds = new BBox(-10.0,10.0,-10.0,10.0); + bounds = new BBox(-10.0, 10.0, -10.0, 10.0); InMemConstructionIndex inMemConstructionIndex = prepareInMemConstructionIndex(bounds, edgeFilter); - lineIntIndex = new LineIntIndex(bounds, directory, "location_index"); lineIntIndex.setMinResolutionInMeter(minResolutionInMeter); lineIntIndex.store(inMemConstructionIndex); lineIntIndex.setChecksum(checksum()); @@ -309,6 +308,7 @@ public Snap findClosest(final double queryLat, final double queryLon, final Edge if (closestMatch.isValid()) { closestMatch.setQueryDistance(DIST_PLANE.calcDenormalizedDist(closestMatch.getQueryDistance())); closestMatch.calcSnappedPoint(DIST_PLANE); + closestMatch.setQueryDistance(DIST_PLANE.calcDist(closestMatch.getSnappedPoint().lat, closestMatch.getSnappedPoint().lon, queryLat, queryLon)); } return closestMatch; } diff --git a/core/src/main/java/com/graphhopper/util/GHUtility.java b/core/src/main/java/com/graphhopper/util/GHUtility.java index b3b41c1d174..25aa861d120 100644 --- a/core/src/main/java/com/graphhopper/util/GHUtility.java +++ b/core/src/main/java/com/graphhopper/util/GHUtility.java @@ -24,9 +24,7 @@ import com.graphhopper.coll.GHBitSet; import com.graphhopper.coll.GHBitSetImpl; import com.graphhopper.routing.ev.*; -import com.graphhopper.routing.querygraph.EdgeIteratorStateHelper; import com.graphhopper.routing.util.*; -import com.graphhopper.routing.weighting.QueryGraphWeighting; import com.graphhopper.routing.weighting.Weighting; import com.graphhopper.storage.*; import com.graphhopper.storage.index.LocationIndex; @@ -516,6 +514,15 @@ public int getOrigEdgeLast() { }; } + public static void checkDAVersion(String name, int expectedVersion, int version) { + if (version != expectedVersion) { + throw new IllegalStateException("Unexpected version for '" + name + "'. Got: " + version + ", " + + "expected: " + expectedVersion + ". " + + "Make sure you are using the same GraphHopper version for reading the files that was used for creating them. " + + "See https://discuss.graphhopper.com/t/722"); + } + } + /** * @return the the edge between base and adj, or null if there is no such edge * @throws IllegalArgumentException when there are multiple edges diff --git a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java index 6e161418b56..1907b9e5428 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/CGIARProviderTest.java @@ -91,7 +91,7 @@ public void downloadFile(String url, String toFile) throws IOException { // file not found => small! assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override diff --git a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java index 1bc732a0d07..ab3c9abf9a4 100644 --- a/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java +++ b/core/src/test/java/com/graphhopper/reader/dem/GMTEDProviderTest.java @@ -98,7 +98,7 @@ public void downloadFile(String url, String toFile) throws IOException { // file not found => small! assertTrue(file.exists()); - assertEquals(228, file.length()); + assertEquals(1048676, file.length()); instance.setDownloader(new Downloader("test GH") { @Override diff --git a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java index bcdf8fb544c..55f268c287a 100644 --- a/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java +++ b/core/src/test/java/com/graphhopper/routing/lm/LandmarkStorageTest.java @@ -95,11 +95,10 @@ public double calcEdgeWeight(EdgeIteratorState edgeState, boolean reverse) { public void testSetGetWeight() { GHUtility.setSpeed(60, true, true, encoder, graph.edge(0, 1).setDistance(40.1)); Directory dir = new RAMDirectory(); - DataAccess da = dir.find("landmarks_c1"); - da.create(2000); LandmarkStorage lms = new LandmarkStorage(graph, dir, new LMConfig("c1", new FastestWeighting(encoder)), 4). setMaximumWeight(LandmarkStorage.PRECISION); + lms._getInternalDA().create(2000); // 2^16=65536, use -1 for infinity and -2 for maximum lms.setWeight(0, 65536); // reached maximum value but do not reset to 0 instead use 2^16-2 @@ -108,14 +107,6 @@ public void testSetGetWeight() { assertEquals(65534, lms.getFromWeight(0, 0)); lms.setWeight(0, 79999); assertEquals(65534, lms.getFromWeight(0, 0)); - - da.setInt(0, Integer.MAX_VALUE); - assertTrue(lms.isInfinity(0)); - // for infinity return much bigger value - // assertEquals(Integer.MAX_VALUE, lms.getFromWeight(0, 0)); - - lms.setWeight(0, 79999); - assertFalse(lms.isInfinity(0)); } @Test diff --git a/core/src/test/java/com/graphhopper/search/StringIndexTest.java b/core/src/test/java/com/graphhopper/search/StringIndexTest.java index c13adbce2ed..7c983a436e5 100644 --- a/core/src/test/java/com/graphhopper/search/StringIndexTest.java +++ b/core/src/test/java/com/graphhopper/search/StringIndexTest.java @@ -15,7 +15,7 @@ public class StringIndexTest { private StringIndex create() { - return new StringIndex(new RAMDirectory()).create(1000); + return new StringIndex(new RAMDirectory(), 1000, -1).create(1000); } Map createMap(String... strings) { @@ -164,12 +164,12 @@ public void testFlush() { String location = "./target/stringindex-store"; Helper.removeDir(new File(location)); - StringIndex index = new StringIndex(new RAMDirectory(location, true).create()).create(1000); + StringIndex index = new StringIndex(new RAMDirectory(location, true).create(), 1000, -1).create(1000); long pointer = index.add(createMap("", "test")); index.flush(); index.close(); - index = new StringIndex(new RAMDirectory(location, true)); + index = new StringIndex(new RAMDirectory(location, true), 1000, -1); assertTrue(index.loadExisting()); assertEquals("test", index.get(pointer, "")); // make sure bytePointer is correctly set after loadExisting @@ -185,7 +185,7 @@ public void testLoadKeys() { String location = "./target/stringindex-store"; Helper.removeDir(new File(location)); - StringIndex index = new StringIndex(new RAMDirectory(location, true).create()).create(1000); + StringIndex index = new StringIndex(new RAMDirectory(location, true).create(), 1000, -1).create(1000); long pointerA = index.add(createMap("c", "test value")); assertEquals(2, index.getKeys().size()); long pointerB = index.add(createMap("a", "value", "b", "another value")); @@ -194,7 +194,7 @@ public void testLoadKeys() { index.flush(); index.close(); - index = new StringIndex(new RAMDirectory(location, true)); + index = new StringIndex(new RAMDirectory(location, true), 1000, -1); assertTrue(index.loadExisting()); assertEquals("[, c, a, b]", index.getKeys().toString()); assertEquals("test value", index.get(pointerA, "c")); diff --git a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java index 345225df6b9..65b202a44e6 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractDirectoryTester.java @@ -24,7 +24,7 @@ import java.io.File; -import static org.junit.jupiter.api.Assertions.assertSame; +import static org.junit.jupiter.api.Assertions.assertThrows; /** * @author Peter Karich @@ -50,11 +50,9 @@ public void setUp() { @Test public void testNoDuplicates() { Directory dir = createDir(); - DataAccess da1 = dir.find("testing"); - DataAccess da2 = dir.find("testing"); - assertSame(da1, da2); + DataAccess da1 = dir.create("testing"); + assertThrows(IllegalStateException.class, () -> dir.create("testing")); da1.close(); - da2.close(); } @Test diff --git a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java index ab039dedd96..9e0d3b9cc46 100644 --- a/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java +++ b/core/src/test/java/com/graphhopper/storage/AbstractGraphStorageTester.java @@ -658,7 +658,6 @@ public void testStringIndex() { @Test public void test8AndMoreBytesForEdgeFlags() { - Directory dir = new RAMDirectory(); List list = new ArrayList<>(); list.add(new CarFlagEncoder(29, 0.001, 0) { @Override @@ -668,7 +667,7 @@ public String toString() { }); list.add(new CarFlagEncoder(29, 0.001, 0)); EncodingManager manager = EncodingManager.create(list); - graph = new GraphHopperStorage(dir, manager, false).create(defaultSize); + graph = new GraphHopperStorage(new RAMDirectory(), manager, false).create(defaultSize); EdgeIteratorState edge = graph.edge(0, 1); IntsRef intsRef = manager.createEdgeFlags(); @@ -678,7 +677,7 @@ public String toString() { assertEquals(Integer.MAX_VALUE / 3, edge.getFlags().ints[0]); graph.close(); - graph = new GraphHopperStorage(dir, manager, false).create(defaultSize); + graph = new GraphHopperStorage(new RAMDirectory(), manager, false).create(defaultSize); DecimalEncodedValue avSpeed0Enc = manager.getDecimalEncodedValue(getKey("car0", "average_speed")); BooleanEncodedValue access0Enc = manager.getBooleanEncodedValue(getKey("car0", "access")); diff --git a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java index e14d8d65790..4524f8b7f3c 100644 --- a/core/src/test/java/com/graphhopper/storage/CHStorageTest.java +++ b/core/src/test/java/com/graphhopper/storage/CHStorageTest.java @@ -4,7 +4,6 @@ import org.junit.jupiter.api.Test; import org.junit.jupiter.api.io.TempDir; -import java.nio.ByteOrder; import java.nio.file.Path; import static org.junit.jupiter.api.Assertions.assertEquals; @@ -12,17 +11,25 @@ class CHStorageTest { + @Test + void setAndGetLevels() { + RAMDirectory dir = new RAMDirectory(); + CHStorage store = new CHStorage(dir, "ch1", -1, false); + store.create(30, 5); + assertEquals(0, store.getLevel(store.toNodePointer(10))); + store.setLevel(store.toNodePointer(10), 100); + assertEquals(100, store.getLevel(store.toNodePointer(10))); + store.setLevel(store.toNodePointer(29), 300); + assertEquals(300, store.getLevel(store.toNodePointer(29))); + } + @Test void createAndLoad(@TempDir Path path) { { GHDirectory dir = new GHDirectory(path.toAbsolutePath().toString(), DAType.RAM_INT_STORE); CHStorage chStorage = new CHStorage(dir, "car", -1, false); // we have to call create, because we want to create a new storage not load an existing one - chStorage.create(); - // init is needed as well, because we have to set the nodes capacity and we cannot do this in create() yet. - // we can also not use init instead of create, because currently GraphHopperStorage needs to 'create' all - // its data objects. if we want to change this lifecycle we need to change this in GraphHopperStorage first - chStorage.init(5, 3); + chStorage.create(5, 3); assertEquals(0, chStorage.shortcutNodeBased(0, 1, PrepareEncoder.getScFwdDir(), 10, 3, 5)); assertEquals(1, chStorage.shortcutNodeBased(1, 2, PrepareEncoder.getScFwdDir(), 11, 4, 6)); assertEquals(2, chStorage.shortcutNodeBased(2, 3, PrepareEncoder.getScFwdDir(), 12, 5, 7)); @@ -71,7 +78,7 @@ public void testBigWeight() { @Test public void testLargeNodeA() { int nodeA = Integer.MAX_VALUE; - RAMIntDataAccess access = new RAMIntDataAccess("", "", false, ByteOrder.LITTLE_ENDIAN); + RAMIntDataAccess access = new RAMIntDataAccess("", "", false, -1); access.create(1000); access.setInt(0, nodeA << 1 | 1 & PrepareEncoder.getScFwdDir()); assertTrue(access.getInt(0) < 0); diff --git a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java index f82b79e1689..caee43a03b0 100644 --- a/core/src/test/java/com/graphhopper/storage/DataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/DataAccessTest.java @@ -24,7 +24,6 @@ import org.junit.jupiter.api.Test; import java.io.File; -import java.nio.ByteOrder; import static org.junit.jupiter.api.Assertions.*; @@ -33,11 +32,14 @@ */ public abstract class DataAccessTest { private final File folder = new File("./target/tmp/da"); - protected ByteOrder defaultOrder = ByteOrder.LITTLE_ENDIAN; protected String directory; protected String name = "dataacess"; - public abstract DataAccess createDataAccess(String location); + public DataAccess createDataAccess(String location) { + return createDataAccess(location, 128); + } + + public abstract DataAccess createDataAccess(String location, int segmentSize); @BeforeEach public void setUp() { @@ -89,11 +91,10 @@ public void testLoadFlush() { public void testExceptionIfNoEnsureCapacityWasCalled() { DataAccess da = createDataAccess(name); assertFalse(da.loadExisting()); - // throw some undefined exception if no ensureCapacity was called try { da.setInt(2 * 4, 321); - assertTrue(false); - } catch (Exception ex) { + fail(); + } catch (Throwable t) { } } @@ -154,8 +155,7 @@ public void testEnsureCapacity() { @Test public void testSegments() { - DataAccess da = createDataAccess(name); - da.setSegmentSize(128); + DataAccess da = createDataAccess(name, 128); da.create(10); assertEquals(1, da.getSegments()); da.ensureCapacity(500); @@ -175,8 +175,16 @@ public void testSegments() { @Test public void testSegmentSize() { - DataAccess da = createDataAccess(name); - da.setSegmentSize(20); + DataAccess da = createDataAccess(name, 20); + da.create(10); + // a minimum segment size is applied + assertEquals(128, da.getSegmentSize()); + da.flush(); + da.close(); + + da = createDataAccess(name, 256); + da.loadExisting(); + // we chose a different segment size, but it is ignored assertEquals(128, da.getSegmentSize()); da.close(); } @@ -186,15 +194,15 @@ public void testSet_GetBytes() { DataAccess da = createDataAccess(name); da.create(300); assertEquals(128, da.getSegmentSize()); - byte[] bytes = BitUtil.BIG.fromInt(Integer.MAX_VALUE / 3); + byte[] bytes = BitUtil.LITTLE.fromInt(Integer.MAX_VALUE / 3); da.setBytes(8, bytes, bytes.length); bytes = new byte[4]; da.getBytes(8, bytes, bytes.length); - assertEquals(Integer.MAX_VALUE / 3, BitUtil.BIG.toInt(bytes)); + assertEquals(Integer.MAX_VALUE / 3, BitUtil.LITTLE.toInt(bytes)); da.setBytes(127, bytes, bytes.length); da.getBytes(127, bytes, bytes.length); - assertEquals(Integer.MAX_VALUE / 3, BitUtil.BIG.toInt(bytes)); + assertEquals(Integer.MAX_VALUE / 3, BitUtil.LITTLE.toInt(bytes)); da.close(); diff --git a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java index 2559a84cffd..1082e3d9a98 100644 --- a/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/MMapDataAccessTest.java @@ -26,13 +26,13 @@ */ public class MMapDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new MMapDataAccess(name, directory, defaultOrder, true).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new MMapDataAccess(name, directory, true, segmentSize); } @Test public void textMixRAM2MMAP() { - DataAccess da = new RAMDataAccess(name, directory, true, defaultOrder); + DataAccess da = new RAMDataAccess(name, directory, true, -1); assertFalse(da.loadExisting()); da.create(100); da.setInt(7 * 4, 123); @@ -54,7 +54,7 @@ public void textMixMMAP2RAM() { // TODO "memory mapped flush" is expensive and not required. only writing the header is required. da.flush(); da.close(); - da = new RAMDataAccess(name, directory, true, defaultOrder); + da = new RAMDataAccess(name, directory, true, -1); assertTrue(da.loadExisting()); assertEquals(123, da.getInt(7 * 4)); da.close(); diff --git a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java index 897901f855e..4406f57053a 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMDataAccessTest.java @@ -22,7 +22,7 @@ */ public class RAMDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new RAMDataAccess(name, directory, true, defaultOrder).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new RAMDataAccess(name, directory, true, segmentSize); } } diff --git a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java index 4f72fac4c79..bfdc069836e 100644 --- a/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java +++ b/core/src/test/java/com/graphhopper/storage/RAMIntDataAccessTest.java @@ -22,8 +22,8 @@ */ public class RAMIntDataAccessTest extends DataAccessTest { @Override - public DataAccess createDataAccess(String name) { - return new RAMIntDataAccess(name, directory, true, defaultOrder).setSegmentSize(128); + public DataAccess createDataAccess(String name, int segmentSize) { + return new RAMIntDataAccess(name, directory, true, segmentSize); } @Override