remove FAT_TILE_INDEX (#700)

CMakeLists.txt

@@ -100,6 +100,7 @@ file(GLOB tilemaker_src_files
 	src/sorted_node_store.cpp
 	src/sorted_way_store.cpp
 	src/tag_map.cpp
+	src/tile_coordinates_set.cpp
 	src/tile_data.cpp
 	src/tilemaker.cpp
 	src/tile_worker.cpp

Makefile

@@ -114,6 +114,7 @@ tilemaker: \
 	src/sorted_node_store.o \
 	src/sorted_way_store.o \
 	src/tag_map.o \
+	src/tile_coordinates_set.o \
 	src/tile_data.o \
 	src/tilemaker.o \
 	src/tile_worker.o \
@@ -130,7 +131,8 @@ test: \
 	test_relation_roles \
 	test_significant_tags \
 	test_sorted_node_store \
-	test_sorted_way_store
+	test_sorted_way_store \
+	test_tile_coordinates_set
 
 test_append_vector: \
 	src/mmap_allocator.o \
@@ -193,6 +195,11 @@ test_sorted_way_store: \
 	test/sorted_way_store.test.o
 	$(CXX) $(CXXFLAGS) -o test.sorted_way_store $^ $(INC) $(LIB) $(LDFLAGS) && ./test.sorted_way_store
 
+test_tile_coordinates_set: \
+	src/tile_coordinates_set.o \
+	test/tile_coordinates_set.test.o
+	$(CXX) $(CXXFLAGS) -o test.tile_coordinates_set $^ $(INC) $(LIB) $(LDFLAGS) && ./test.tile_coordinates_set
+
 test_pbf_reader: \
 	src/helpers.o \
 	src/pbf_reader.o \

docs/INSTALL.md

@@ -65,11 +65,3 @@ The docker container can be run like this:
     docker run -v /Users/Local/Downloads/:/srv -i -t --rm tilemaker /srv/germany-latest.osm.pbf --output=/srv/germany.mbtiles
 
 Keep in mind to map the volume your .osm.pbf files are in to a path within your docker container, as seen in the example above. 
-
-### Compile-time options
-
-tilemaker has a compile-time option that increases memory usage but may be useful in certain circumstances. You can include it when building like this:
-
-    make "CONFIG=-DFAT_TILE_INDEX"
-
-FAT_TILE_INDEX allows you to generate vector tiles at zoom level 17 or greater. You almost certainly don't need to do this. Vector tiles are usually generated up to zoom 14 (sometimes 15), and then the browser/app client uses the vector data to scale up at subsequent zoom levels.

include/coordinates.h

@@ -24,17 +24,8 @@ typedef uint64_t RelationID;
 typedef std::vector<WayID> WayVec;
 
 
-#ifdef FAT_TILE_INDEX
 // Supports up to z22
 typedef uint32_t TileCoordinate;
-typedef uint16_t Z6Offset;
-#define TILE_COORDINATE_MAX UINT32_MAX
-#else
-// Supports up to z14
-typedef uint16_t TileCoordinate;
-typedef uint8_t Z6Offset;
-#define TILE_COORDINATE_MAX UINT16_MAX
-#endif
 
 class TileCoordinates_ {
 

include/osm_mem_tiles.h

@@ -31,7 +31,7 @@ class OsmMemTiles : public TileDataSource {
 public:
 	OsmMemTiles(
 		size_t threadNum,
-		uint baseZoom,
+		uint indexZoom,
 		bool includeID,
 		const NodeStore& nodeStore,
 		const WayStore& wayStore

include/shp_mem_tiles.h

@@ -9,7 +9,7 @@ extern bool verbose;
 class ShpMemTiles : public TileDataSource
 {
 public:
-	ShpMemTiles(size_t threadNum, uint baseZoom);
+	ShpMemTiles(size_t threadNum, uint indexZoom);
 
 	std::string name() const override { return "shp"; }
 

include/tile_coordinates_set.h

@@ -0,0 +1,47 @@
+#ifndef TILE_COORDINATES_SET_H
+#define TILE_COORDINATES_SET_H
+
+#include <cstddef>
+#include "coordinates.h"
+
+// Interface representing a bitmap of tiles of interest at a given zoom.
+class TileCoordinatesSet {
+public:
+	virtual bool test(TileCoordinate x, TileCoordinate y) const = 0;
+	virtual void set(TileCoordinate x, TileCoordinate y) = 0;
+	virtual size_t size() const = 0;
+	virtual size_t zoom() const = 0;
+};
+
+// Read-write implementation for precise sets; maximum zoom is
+// generally expected to be z14.
+class PreciseTileCoordinatesSet : public TileCoordinatesSet {
+public:
+	PreciseTileCoordinatesSet(unsigned int zoom);
+	bool test(TileCoordinate x, TileCoordinate y) const override;
+	size_t size() const override;
+	size_t zoom() const override;
+	void set(TileCoordinate x, TileCoordinate y) override;
+
+private:
+	unsigned int zoom_;
+	std::vector<bool> tiles;
+};
+
+// Read-only implementation for a lossy set. Used when zoom is
+// z15 or higher, extrapolates a result based on a set for a lower zoom.
+class LossyTileCoordinatesSet : public TileCoordinatesSet {
+public:
+	LossyTileCoordinatesSet(unsigned int zoom, const TileCoordinatesSet& precise);
+	bool test(TileCoordinate x, TileCoordinate y) const override;
+	size_t size() const override;
+	size_t zoom() const override;
+	void set(TileCoordinate x, TileCoordinate y) override;
+
+private:
+	unsigned int zoom_;
+	const TileCoordinatesSet& tiles;
+	unsigned int scale;
+};
+
+#endif

include/tile_data.h

@@ -11,6 +11,7 @@
 #include "append_vector.h"
 #include "clip_cache.h"
 #include "mmap_allocator.h"
+#include "tile_coordinates_set.h"
 
 #define TILE_DATA_ID_SIZE 34
 
@@ -23,24 +24,27 @@ class TileBbox;
 #define CLUSTER_ZOOM_WIDTH (1 << CLUSTER_ZOOM)
 #define CLUSTER_ZOOM_AREA (CLUSTER_ZOOM_WIDTH * CLUSTER_ZOOM_WIDTH)
 
+// TileDataSource indexes which tiles have objects in them. The indexed zoom
+// is at most z14; we'll clamp to z14 if the base zoom is higher than z14.
+//
+// As a result, we need at most 15 bits to store an X/Y coordinate. For efficiency,
+// we bucket the world into 4,096 z6 tiles, which each contain some number of 
+// z14 objects. This lets us use only 8 bits to store an X/Y coordinate.
+//
+// Because index zoom is lower than base zoom in the case where base zoom is
+// z15+, we'll get false positives when looking up objects in the index,
+// since, e.g., a single z14 tile covers 4 z15 tiles.
+//
+// This is OK: when writing the z15 tile, there's a clipping step that will filter
+// out the false positives.
+typedef uint8_t Z6Offset;
+
 struct OutputObjectXY {
 	OutputObject oo;
 	Z6Offset x;
 	Z6Offset y;
 };
 
-class TileCoordinatesSet {
-public:
-	TileCoordinatesSet(uint zoom);
-	bool test(TileCoordinate x, TileCoordinate y) const;
-	void set(TileCoordinate x, TileCoordinate y);
-	size_t size() const;
-
-private:
-	uint zoom;
-	std::vector<bool> tiles;
-};
-
 struct OutputObjectXYID {
 	OutputObject oo;
 	Z6Offset x;
@@ -51,12 +55,12 @@ struct OutputObjectXYID {
 template<typename OO> void finalizeObjects(
 	const std::string& name,
 	const size_t& threadNum,
-	const unsigned int& baseZoom,
+	const unsigned int& indexZoom,
 	typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator begin,
 	typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator end,
 	typename std::vector<std::vector<OO>>& lowZoom
 	) {
-	size_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1;
+	size_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1;
 #ifdef CLOCK_MONOTONIC
 	timespec startTs, endTs;
 	clock_gettime(CLOCK_MONOTONIC, &startTs);
@@ -105,22 +109,22 @@ template<typename OO> void finalizeObjects(
 		boost::sort::block_indirect_sort(
 			it->begin(),
 			it->end(), 
-			[baseZoom](const OO& a, const OO& b) {
+			[indexZoom](const OO& a, const OO& b) {
 				// Cluster by parent zoom, so that a subsequent search
 				// can find a contiguous range of entries for any tile
 				// at zoom 6 or higher.
 				const size_t aX = a.x;
 				const size_t aY = a.y;
 				const size_t bX = b.x;
 				const size_t bY = b.y;
-				for (size_t z = CLUSTER_ZOOM; z <= baseZoom; z++) {
-					const auto aXz = aX / (1 << (baseZoom - z));
-					const auto bXz = bX / (1 << (baseZoom - z));
+				for (size_t z = CLUSTER_ZOOM; z <= indexZoom; z++) {
+					const auto aXz = aX / (1 << (indexZoom - z));
+					const auto bXz = bX / (1 << (indexZoom - z));
 					if (aXz != bXz)
 						return aXz < bXz;
 
-					const auto aYz = aY / (1 << (baseZoom - z));
-					const auto bYz = bY / (1 << (baseZoom - z));
+					const auto aYz = aY / (1 << (indexZoom - z));
+					const auto bYz = bY / (1 << (indexZoom - z));
 
 					if (aYz != bYz)
 						return aYz < bYz;
@@ -136,13 +140,13 @@ template<typename OO> void finalizeObjects(
 }
 
 template<typename OO> void collectTilesWithObjectsAtZoomTemplate(
-	const unsigned int& baseZoom,
+	const unsigned int& indexZoom,
 	const typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator objects,
 	const size_t size,
-	std::vector<TileCoordinatesSet>& zooms
+	std::vector<std::shared_ptr<TileCoordinatesSet>>& zooms
 ) {
 	size_t maxZoom = zooms.size() - 1;
-	uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1;
+	uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1;
 	int64_t lastX = -1;
 	int64_t lastY = -1;
 	for (size_t i = 0; i < size; i++) {
@@ -155,15 +159,15 @@ template<typename OO> void collectTilesWithObjectsAtZoomTemplate(
 			TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y;
 
 			// Translate the x, y at the requested zoom level
-			TileCoordinate x = baseX / (1 << (baseZoom - maxZoom));
-			TileCoordinate y = baseY / (1 << (baseZoom - maxZoom));
+			TileCoordinate x = baseX / (1 << (indexZoom - maxZoom));
+			TileCoordinate y = baseY / (1 << (indexZoom - maxZoom));
 
 			if (lastX != x || lastY != y) {
 				lastX = x;
 				lastY = y;
 
 				for (int zoom = maxZoom; zoom >= 0; zoom--) {
-					zooms[zoom].set(x, y);
+					zooms[zoom]->set(x, y);
 					x /= 2;
 					y /= 2;
 				}
@@ -183,7 +187,7 @@ inline OutputObjectID outputObjectWithId<OutputObjectXYID>(const OutputObjectXYI
 }
 
 template<typename OO> void collectLowZoomObjectsForTile(
-	const unsigned int& baseZoom,
+	const unsigned int& indexZoom,
 	typename std::vector<std::vector<OO>> objects,
 	unsigned int zoom,
 	const TileCoordinates& dstIndex,
@@ -192,7 +196,7 @@ template<typename OO> void collectLowZoomObjectsForTile(
 	if (zoom >= CLUSTER_ZOOM)
 		throw std::runtime_error("collectLowZoomObjectsForTile should not be called for high zooms");
 
-	uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1;
+	uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1;
 
 	for (size_t i = 0; i < objects.size(); i++) {
 		const size_t z6x = i / CLUSTER_ZOOM_WIDTH;
@@ -204,8 +208,8 @@ template<typename OO> void collectLowZoomObjectsForTile(
 			TileCoordinate baseY = z6y * z6OffsetDivisor + objects[i][j].y;
 
 			// Translate the x, y at the requested zoom level
-			TileCoordinate x = baseX / (1 << (baseZoom - zoom));
-			TileCoordinate y = baseY / (1 << (baseZoom - zoom));
+			TileCoordinate x = baseX / (1 << (indexZoom - zoom));
+			TileCoordinate y = baseY / (1 << (indexZoom - zoom));
 
 			if (dstIndex.x == x && dstIndex.y == y) {
 				if (objects[i][j].oo.minZoom <= zoom) {
@@ -217,28 +221,36 @@ template<typename OO> void collectLowZoomObjectsForTile(
 }
 
 template<typename OO> void collectObjectsForTileTemplate(
-	const unsigned int& baseZoom,
+	const unsigned int& indexZoom,
 	typename std::vector<AppendVectorNS::AppendVector<OO>>::iterator objects,
 	size_t iStart,
 	size_t iEnd,
 	unsigned int zoom,
-	const TileCoordinates& dstIndex,
+	TileCoordinates dstIndex,
 	std::vector<OutputObjectID>& output
 ) {
 	if (zoom < CLUSTER_ZOOM)
 		throw std::runtime_error("collectObjectsForTileTemplate should not be called for low zooms");
 
-	uint16_t z6OffsetDivisor = baseZoom >= CLUSTER_ZOOM ? (1 << (baseZoom - CLUSTER_ZOOM)) : 1;
+	// When base zoom is z15 or higher, we need to scale down to z14.
+	unsigned int clampedZoom = zoom;
+	while(clampedZoom > indexZoom) {
+		clampedZoom--;
+		dstIndex.x /= 2;
+		dstIndex.y /= 2;
+	}
+
+	uint16_t z6OffsetDivisor = indexZoom >= CLUSTER_ZOOM ? (1 << (indexZoom - CLUSTER_ZOOM)) : 1;
 
 	for (size_t i = iStart; i < iEnd; i++) {
 		// If z >= 6, we can compute the exact bounds within the objects array.
 		// Translate to the base zoom, then do a binary search to find
 		// the starting point.
-		TileCoordinate z6x = dstIndex.x / (1 << (zoom - CLUSTER_ZOOM));
-		TileCoordinate z6y = dstIndex.y / (1 << (zoom - CLUSTER_ZOOM));
+		TileCoordinate z6x = dstIndex.x / (1 << (clampedZoom - CLUSTER_ZOOM));
+		TileCoordinate z6y = dstIndex.y / (1 << (clampedZoom - CLUSTER_ZOOM));
 
-		TileCoordinate baseX = dstIndex.x * (1 << (baseZoom - zoom));
-		TileCoordinate baseY = dstIndex.y * (1 << (baseZoom - zoom));
+		TileCoordinate baseX = dstIndex.x * (1 << (indexZoom - clampedZoom));
+		TileCoordinate baseY = dstIndex.y * (1 << (indexZoom - clampedZoom));
 
 		Z6Offset needleX = baseX - z6x * z6OffsetDivisor;
 		Z6Offset needleY = baseY - z6y * z6OffsetDivisor;
@@ -247,7 +259,7 @@ template<typename OO> void collectObjectsForTileTemplate(
 		// into two arrays, one of x/y and one of OOs. Would have better locality for
 		// searching, too.
 		OutputObject dummyOo(POINT_, 0, 0, 0, 0);
-		const size_t bz = baseZoom;
+		const size_t bz = indexZoom;
 
 		const OO targetXY = {dummyOo, needleX, needleY };
 		auto iter = std::lower_bound(
@@ -284,8 +296,8 @@ template<typename OO> void collectObjectsForTileTemplate(
 			TileCoordinate baseY = z6y * z6OffsetDivisor + iter->y;
 
 			// Translate the x, y at the requested zoom level
-			TileCoordinate x = baseX / (1 << (baseZoom - zoom));
-			TileCoordinate y = baseY / (1 << (baseZoom - zoom));
+			TileCoordinate x = baseX / (1 << (indexZoom - clampedZoom));
+			TileCoordinate y = baseY / (1 << (indexZoom - clampedZoom));
 
 			if (dstIndex.x == x && dstIndex.y == y) {
 				if (iter->oo.minZoom <= zoom) {
@@ -354,7 +366,7 @@ class TileDataSource {
 	boost::geometry::index::rtree< std::pair<Box,OutputObject>, oo_rtree_param_type> boxRtree;
 	boost::geometry::index::rtree< std::pair<Box,OutputObjectID>, oo_rtree_param_type> boxRtreeWithIds;
 
-	unsigned int baseZoom;
+	unsigned int indexZoom;
 
 	std::vector<point_store_t> pointStores;
 	std::vector<linestring_store_t> linestringStores;
@@ -367,11 +379,11 @@ class TileDataSource {
 	std::deque<std::vector<std::tuple<TileCoordinates, OutputObject, uint64_t>>> pendingSmallIndexObjects;
 
 public:
-	TileDataSource(size_t threadNum, unsigned int baseZoom, bool includeID);
+	TileDataSource(size_t threadNum, unsigned int indexZoom, bool includeID);
 
-	void collectTilesWithObjectsAtZoom(std::vector<TileCoordinatesSet>& zooms);
+	void collectTilesWithObjectsAtZoom(std::vector<std::shared_ptr<TileCoordinatesSet>>& zooms);
 
-	void collectTilesWithLargeObjectsAtZoom(std::vector<TileCoordinatesSet>& zooms);
+	void collectTilesWithLargeObjectsAtZoom(std::vector<std::shared_ptr<TileCoordinatesSet>>& zooms);
 
 	void collectObjectsForTile(uint zoom, TileCoordinates dstIndex, std::vector<OutputObjectID>& output);
 	void finalize(size_t threadNum);
@@ -486,7 +498,7 @@ class TileDataSource {
 
 void populateTilesAtZoom(
 	const std::vector<class TileDataSource *>& sources,
-	std::vector<TileCoordinatesSet>& zooms
+	std::vector<std::shared_ptr<TileCoordinatesSet>>& zooms
 );
 
 #endif //_TILE_DATA_H