From ace47359cca6d35c2a9301d3329e025931b0d3e9 Mon Sep 17 00:00:00 2001 From: Tomas Vajda Date: Fri, 29 Nov 2019 16:33:34 +0100 Subject: [PATCH 1/5] Add support for multiple mbtiles files through urls_mbtiles --- core/src/data/mbtilesDataSource.cpp | 222 +++++++++++++++------------- core/src/data/mbtilesDataSource.h | 10 +- core/src/scene/sceneLoader.cpp | 15 +- 3 files changed, 139 insertions(+), 108 deletions(-) diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index 6997003bf0..bf73541a6d 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -115,10 +115,11 @@ struct MBTilesQueries { }; -MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, std::string _path, - std::string _mime, bool _cache, bool _offlineFallback) +MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, + std::vector _paths, std::string _mime, + bool _cache, bool _offlineFallback) : m_name(_name), - m_path(_path), + m_paths(_paths), m_mime(_mime), m_cacheMode(_cache), m_offlineMode(_offlineFallback), @@ -143,7 +144,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb return loadNextSource(_task, _cb); } - if (!m_db) { return false; } + if (m_dbs.empty()) { return false; } if (_task->rawSource == this->level) { @@ -184,7 +185,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTaskCb _cb) { if (!next) { return false; } - if (!m_db) { + if (m_dbs.empty()) { return next->loadTileData(_task, _cb); } @@ -196,12 +197,12 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask if (m_cacheMode) { m_worker->enqueue([this, _task](){ - auto& task = static_cast(*_task); + auto& task = static_cast(*_task); - LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); + LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); - storeTileData(_task->tileId(), *task.rawTileData); - }); + storeTileData(_task->tileId(), *task.rawTileData); + }); } _cb.func(_task); @@ -232,77 +233,86 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask void MBTilesDataSource::openMBTiles() { - try { - auto mode = SQLite::OPEN_READONLY; - if (m_cacheMode) { - // Need to explicitly open a SQLite DB with OPEN_READWRITE - // and OPEN_CREATE flags to make a file and write. - mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE; - } + for (int i = 0; i < m_paths.size(); ++i) { + std::string path = m_paths[i]; - auto url = Url(m_path); - auto path = url.path(); - const char* vfs = ""; - if (url.scheme() == "asset") { - vfs = "ndk-asset"; - path.erase(path.begin()); // Remove leading '/'. - } - m_db = std::make_unique(path, mode, 0, vfs); - LOG("SQLite database opened: %s", path.c_str()); + try { + auto mode = SQLite::OPEN_READONLY; + if (m_cacheMode) { + // Need to explicitly open a SQLite DB with OPEN_READWRITE + // and OPEN_CREATE flags to make a file and write. + mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE; + } - } catch (std::exception& e) { - LOGE("Unable to open SQLite database: %s - %s", m_path.c_str(), e.what()); - m_db.reset(); - return; - } + auto url = Url(path); + auto path = url.path(); + const char* vfs = ""; + if (url.scheme() == "asset") { + vfs = "ndk-asset"; + path.erase(path.begin()); // Remove leading '/'. + } + m_dbs.push_back(std::make_unique(path, mode, 0, vfs)); + LOG("SQLite database opened: %s", path.c_str()); - bool ok = testSchema(*m_db); - if (ok) { - if (m_cacheMode && !m_schemaOptions.isCache) { - // TODO better description - LOGE("Cannot cache to 'externally created' MBTiles database"); - // Run in non-caching mode - m_cacheMode = false; + } catch (std::exception& e) { + LOGE("Unable to open SQLite database: %s - %s", path.c_str(), e.what()); + if (!m_dbs.empty()) { + m_dbs[m_dbs.size() - 1].reset(); + } return; } - } else if (m_cacheMode) { - // Setup the database by running the schema.sql. - initSchema(*m_db, m_name, m_mime); + if (!m_dbs.empty()) { + std::unique_ptr& db = m_dbs[m_dbs.size() - 1]; + + bool ok = testSchema(*db); + if (ok) { + if (m_cacheMode && !m_schemaOptions.isCache) { + // TODO better description + LOGE("Cannot cache to 'externally created' MBTiles database"); + // Run in non-caching mode + m_cacheMode = false; + return; + } + } else if (m_cacheMode) { + + // Setup the database by running the schema.sql. + initSchema(*db, m_name, m_mime); - ok = testSchema(*m_db); - if (!ok) { - LOGE("Unable to initialize MBTiles schema"); - m_db.reset(); - return; - } - } else { - LOGE("Invalid MBTiles schema"); - m_db.reset(); - return; - } + ok = testSchema(*db); + if (!ok) { + LOGE("Unable to initialize MBTiles schema"); + db.reset(); + return; + } + } else { + LOGE("Invalid MBTiles schema"); + db.reset(); + return; + } - if (m_schemaOptions.compression == Compression::unsupported) { - m_db.reset(); - return; - } + if (m_schemaOptions.compression == Compression::unsupported) { + db.reset(); + return; + } - try { - m_queries = std::make_unique(*m_db, m_cacheMode); - } catch (std::exception& e) { - LOGE("Unable to initialize queries: %s", e.what()); - m_db.reset(); - return; + try { + m_queries.push_back(std::make_unique(*db, m_cacheMode)); + } catch (std::exception &e) { + LOGE("Unable to initialize queries: %s", e.what()); + db.reset(); + } + } } } /** - * We check to see if the database has the MBTiles Schema. - * Sets m_schemaOptions from metadata table - * - * @param _source A pointer to a the data source in which we will setup a db. - * @return true if database contains MBTiles schema - */ +* We check to see if the database has the MBTiles Schema. +* Sets m_schemaOptions from metadata table +* +* @param _source A pointer to a the data source in which we will setup a db. +* @return true if database contains MBTiles schema +*/ bool MBTilesDataSource::testSchema(SQLite::Database& db) { bool metadata = false, tiles = false, grids = false, grid_data = false; @@ -314,7 +324,7 @@ bool MBTilesDataSource::testSchema(SQLite::Database& db) { // required if (name == "metadata") metadata = true; else if (name == "tiles") tiles = true; - // optional + // optional else if (name == "grids") grids = true; else if (name == "grid_data") grid_data = true; // schema implementation specific @@ -424,49 +434,59 @@ void MBTilesDataSource::initSchema(SQLite::Database& db, std::string _name, std: bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _data) { - auto& stmt = m_queries->getTileData; - try { - // Google TMS to WMTS - // https://github.com/mapbox/node-mbtiles/blob/ - // 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149 - int z = _tileId.z; - int y = (1 << z) - 1 - _tileId.y; + int largestLength = 0; - stmt.bind(1, z); - stmt.bind(2, _tileId.x); - stmt.bind(3, y); + for (int i = 0; i < m_queries.size(); ++i) { + + auto &stmt = m_queries[i]->getTileData; + try { + // Google TMS to WMTS + // https://github.com/mapbox/node-mbtiles/blob/ + // 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149 + int z = _tileId.z; + int y = (1 << z) - 1 - _tileId.y; + + stmt.bind(1, z); + stmt.bind(2, _tileId.x); + stmt.bind(3, y); + + if (stmt.executeStep()) { + SQLite::Column column = stmt.getColumn(0); + const int length = column.getBytes(); - if (stmt.executeStep()) { - SQLite::Column column = stmt.getColumn(0); - const char* blob = (const char*) column.getBlob(); - const int length = column.getBytes(); + if (length > largestLength) { + const char *blob = (const char *) column.getBlob(); - if ((m_schemaOptions.compression == Compression::undefined) || - (m_schemaOptions.compression == Compression::deflate)) { + if ((m_schemaOptions.compression == Compression::undefined) || + (m_schemaOptions.compression == Compression::deflate)) { - if (zlib::inflate(blob, length, _data) != 0) { - if (m_schemaOptions.compression == Compression::undefined) { + if (zlib::inflate(blob, length, _data) != 0) { + if (m_schemaOptions.compression == Compression::undefined) { + _data.resize(length); + memcpy(_data.data(), blob, length); + } else { + LOGW("Invalid deflate compression"); + } + } + } else { _data.resize(length); memcpy(_data.data(), blob, length); - } else { - LOGW("Invalid deflate compression"); } + + largestLength = length; } - } else { - _data.resize(length); - memcpy(_data.data(), blob, length); + + stmt.reset(); + //return true; } - stmt.reset(); - return true; + } catch (std::exception &e) { + LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); } - - } catch (std::exception& e) { - LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); + try { + stmt.reset(); + } catch (...) {} } - try { - stmt.reset(); - } catch(...) {} return false; } @@ -487,7 +507,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vectorputMap; + auto& stmt = m_queries[0]->putMap; stmt.bind(1, z); stmt.bind(2, _tileId.x); stmt.bind(3, y); @@ -501,7 +521,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vectorputImage; + auto& stmt = m_queries[0]->putImage; stmt.bind(1, md5id); stmt.bind(2, data, size); stmt.exec(); diff --git a/core/src/data/mbtilesDataSource.h b/core/src/data/mbtilesDataSource.h index 2a9f65ef8a..9063337c0f 100644 --- a/core/src/data/mbtilesDataSource.h +++ b/core/src/data/mbtilesDataSource.h @@ -17,8 +17,8 @@ class AsyncWorker; class MBTilesDataSource : public TileSource::DataSource { public: - MBTilesDataSource(Platform& _platform, std::string _name, std::string _path, std::string _mime, - bool _cache = false, bool _offlineFallback = false); + MBTilesDataSource(Platform& _platform, std::string _name, std::vector _paths, + std::string _mime, bool _cache = false, bool _offlineFallback = false); ~MBTilesDataSource(); @@ -38,7 +38,7 @@ class MBTilesDataSource : public TileSource::DataSource { std::string m_name; // The path to an mbtiles tile store. - std::string m_path; + std::vector m_paths; std::string m_mime; // Store tiles from next source @@ -48,8 +48,8 @@ class MBTilesDataSource : public TileSource::DataSource { bool m_offlineMode; // Pointer to SQLite DB of MBTiles store - std::unique_ptr m_db; - std::unique_ptr m_queries; + std::vector> m_dbs; + std::vector> m_queries; std::unique_ptr m_worker; // Platform reference diff --git a/core/src/scene/sceneLoader.cpp b/core/src/scene/sceneLoader.cpp index 1369b0924a..1925c6956d 100644 --- a/core/src/scene/sceneLoader.cpp +++ b/core/src/scene/sceneLoader.cpp @@ -697,6 +697,7 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s std::string type; std::string url; + std::vector urls_mbtiles; std::string mbtiles; std::vector subdomains; @@ -712,6 +713,13 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s if (auto urlNode = _source["url"]) { url = urlNode.Scalar(); } + if (auto urlsMbtilesNode = _source["urls_mbtiles"]) { + if (urlsMbtilesNode.IsSequence()) { + for (const auto &urlMbtilesNode : urlsMbtilesNode) { + urls_mbtiles.push_back(urlMbtilesNode.Scalar()); + } + } + } if (auto minDisplayZoomNode = _source["min_display_zoom"]) { YamlUtil::getInt(minDisplayZoomNode, minDisplayZoom); } @@ -805,12 +813,15 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s std::unique_ptr rawSources; - if (isMBTilesFile) { + if (!urls_mbtiles.empty() || isMBTilesFile) { #ifdef TANGRAM_MBTILES_DATASOURCE // If we have MBTiles, we know the source is tiled. tiled = true; // Create an MBTiles data source from the file at the url and add it to the source chain. - rawSources = std::make_unique(_platform, _name, url, ""); + if (urls_mbtiles.empty()) { + urls_mbtiles.push_back(url); + } + rawSources = std::make_unique(_platform, _name, urls_mbtiles, ""); #else LOGE("MBTiles support is disabled. This source will be ignored: %s", _name.c_str()); return nullptr; From 8e0b64e3abf3410eb574c13e6d6a92abc7c646aa Mon Sep 17 00:00:00 2001 From: Vectura Games Maps Date: Wed, 18 Dec 2019 12:49:54 +0100 Subject: [PATCH 2/5] Fix spaces --- core/src/data/mbtilesDataSource.cpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index bf73541a6d..5821b0716b 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -324,7 +324,7 @@ bool MBTilesDataSource::testSchema(SQLite::Database& db) { // required if (name == "metadata") metadata = true; else if (name == "tiles") tiles = true; - // optional + // optional else if (name == "grids") grids = true; else if (name == "grid_data") grid_data = true; // schema implementation specific @@ -438,7 +438,7 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d for (int i = 0; i < m_queries.size(); ++i) { - auto &stmt = m_queries[i]->getTileData; + auto& stmt = m_queries[i]->getTileData; try { // Google TMS to WMTS // https://github.com/mapbox/node-mbtiles/blob/ @@ -480,7 +480,7 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d //return true; } - } catch (std::exception &e) { + } catch (std::exception& e) { LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); } try { From 86b9dd958f01312414d3dd067993873a26876f6e Mon Sep 17 00:00:00 2001 From: Vectura Games Maps Date: Wed, 18 Dec 2019 12:50:43 +0100 Subject: [PATCH 3/5] Fix return value in MBTilesDataSource::getTileData --- core/src/data/mbtilesDataSource.cpp | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index 5821b0716b..d7379f1834 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -477,7 +477,6 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d } stmt.reset(); - //return true; } } catch (std::exception& e) { @@ -488,7 +487,7 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d } catch (...) {} } - return false; + return !_data.empty(); } void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vector& _data) { From 5029946003f245909070b40fe33419a11ff97b02 Mon Sep 17 00:00:00 2001 From: Tomas Vajda Date: Fri, 20 Dec 2019 11:41:13 +0100 Subject: [PATCH 4/5] Address changes --- core/src/data/mbtilesDataSource.cpp | 92 ++++++++++++++--------------- 1 file changed, 46 insertions(+), 46 deletions(-) diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index d7379f1834..d55cc2e2d9 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -144,7 +144,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb return loadNextSource(_task, _cb); } - if (m_dbs.empty()) { return false; } + if (m_queries.empty()) { return false; } if (_task->rawSource == this->level) { @@ -185,7 +185,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTaskCb _cb) { if (!next) { return false; } - if (m_dbs.empty()) { + if (m_queries.empty()) { return next->loadTileData(_task, _cb); } @@ -197,11 +197,11 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask if (m_cacheMode) { m_worker->enqueue([this, _task](){ - auto& task = static_cast(*_task); + auto& task = static_cast(*_task); - LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); + LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); - storeTileData(_task->tileId(), *task.rawTileData); + storeTileData(_task->tileId(), *task.rawTileData); }); } @@ -259,59 +259,59 @@ void MBTilesDataSource::openMBTiles() { if (!m_dbs.empty()) { m_dbs[m_dbs.size() - 1].reset(); } - return; + continue; } + } - if (!m_dbs.empty()) { - std::unique_ptr& db = m_dbs[m_dbs.size() - 1]; - - bool ok = testSchema(*db); - if (ok) { - if (m_cacheMode && !m_schemaOptions.isCache) { - // TODO better description - LOGE("Cannot cache to 'externally created' MBTiles database"); - // Run in non-caching mode - m_cacheMode = false; - return; - } - } else if (m_cacheMode) { + for (int i = 0; i < m_dbs.size(); ++i) { + std::unique_ptr& db = m_dbs[i]; + + bool ok = testSchema(*db); + if (ok) { + if (m_cacheMode && !m_schemaOptions.isCache) { + // TODO better description + LOGE("Cannot cache to 'externally created' MBTiles database"); + // Run in non-caching mode + m_cacheMode = false; + continue; + } + } else if (m_cacheMode) { - // Setup the database by running the schema.sql. - initSchema(*db, m_name, m_mime); + // Setup the database by running the schema.sql. + initSchema(*db, m_name, m_mime); - ok = testSchema(*db); - if (!ok) { - LOGE("Unable to initialize MBTiles schema"); - db.reset(); - return; - } - } else { - LOGE("Invalid MBTiles schema"); + ok = testSchema(*db); + if (!ok) { + LOGE("Unable to initialize MBTiles schema"); db.reset(); - return; + continue; } + } else { + LOGE("Invalid MBTiles schema"); + db.reset(); + continue; + } - if (m_schemaOptions.compression == Compression::unsupported) { - db.reset(); - return; - } + if (m_schemaOptions.compression == Compression::unsupported) { + db.reset(); + continue; + } - try { - m_queries.push_back(std::make_unique(*db, m_cacheMode)); - } catch (std::exception &e) { - LOGE("Unable to initialize queries: %s", e.what()); - db.reset(); - } + try { + m_queries.push_back(std::make_unique(*db, m_cacheMode)); + } catch (std::exception& e) { + LOGE("Unable to initialize queries: %s", e.what()); + db.reset(); } } } /** -* We check to see if the database has the MBTiles Schema. -* Sets m_schemaOptions from metadata table -* -* @param _source A pointer to a the data source in which we will setup a db. -* @return true if database contains MBTiles schema + * We check to see if the database has the MBTiles Schema. + * Sets m_schemaOptions from metadata table + * + * @param _source A pointer to a the data source in which we will setup a db. + * @return true if database contains MBTiles schema */ bool MBTilesDataSource::testSchema(SQLite::Database& db) { @@ -455,7 +455,7 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d const int length = column.getBytes(); if (length > largestLength) { - const char *blob = (const char *) column.getBlob(); + const char* blob = (const char*) column.getBlob(); if ((m_schemaOptions.compression == Compression::undefined) || (m_schemaOptions.compression == Compression::deflate)) { From 0be3933c782483363f24257eeec88faa1e7b39b0 Mon Sep 17 00:00:00 2001 From: Tomas Vajda Date: Fri, 20 Dec 2019 22:01:20 +0100 Subject: [PATCH 5/5] Add comment --- core/src/data/mbtilesDataSource.cpp | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index d55cc2e2d9..a4d7c9c45f 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -454,6 +454,7 @@ bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _d SQLite::Column column = stmt.getColumn(0); const int length = column.getBytes(); + // When multiple mbtiles contain the same tile, get the tile which has more data. if (length > largestLength) { const char* blob = (const char*) column.getBlob();