diff --git a/core/src/data/mbtilesDataSource.cpp b/core/src/data/mbtilesDataSource.cpp index 6997003bf0..a4d7c9c45f 100644 --- a/core/src/data/mbtilesDataSource.cpp +++ b/core/src/data/mbtilesDataSource.cpp @@ -115,10 +115,11 @@ struct MBTilesQueries { }; -MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, std::string _path, - std::string _mime, bool _cache, bool _offlineFallback) +MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, + std::vector _paths, std::string _mime, + bool _cache, bool _offlineFallback) : m_name(_name), - m_path(_path), + m_paths(_paths), m_mime(_mime), m_cacheMode(_cache), m_offlineMode(_offlineFallback), @@ -143,7 +144,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb return loadNextSource(_task, _cb); } - if (!m_db) { return false; } + if (m_queries.empty()) { return false; } if (_task->rawSource == this->level) { @@ -184,7 +185,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr _task, TileTaskCb bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTaskCb _cb) { if (!next) { return false; } - if (!m_db) { + if (m_queries.empty()) { return next->loadTileData(_task, _cb); } @@ -196,12 +197,12 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask if (m_cacheMode) { m_worker->enqueue([this, _task](){ - auto& task = static_cast(*_task); + auto& task = static_cast(*_task); - LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); + LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData()); - storeTileData(_task->tileId(), *task.rawTileData); - }); + storeTileData(_task->tileId(), *task.rawTileData); + }); } _cb.func(_task); @@ -232,67 +233,76 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr _task, TileTask void MBTilesDataSource::openMBTiles() { - try { - auto mode = SQLite::OPEN_READONLY; - if (m_cacheMode) { - // Need to explicitly open a SQLite DB with OPEN_READWRITE - // and OPEN_CREATE flags to make a file and write. - mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE; - } + for (int i = 0; i < m_paths.size(); ++i) { + std::string path = m_paths[i]; - auto url = Url(m_path); - auto path = url.path(); - const char* vfs = ""; - if (url.scheme() == "asset") { - vfs = "ndk-asset"; - path.erase(path.begin()); // Remove leading '/'. - } - m_db = std::make_unique(path, mode, 0, vfs); - LOG("SQLite database opened: %s", path.c_str()); + try { + auto mode = SQLite::OPEN_READONLY; + if (m_cacheMode) { + // Need to explicitly open a SQLite DB with OPEN_READWRITE + // and OPEN_CREATE flags to make a file and write. + mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE; + } - } catch (std::exception& e) { - LOGE("Unable to open SQLite database: %s - %s", m_path.c_str(), e.what()); - m_db.reset(); - return; - } + auto url = Url(path); + auto path = url.path(); + const char* vfs = ""; + if (url.scheme() == "asset") { + vfs = "ndk-asset"; + path.erase(path.begin()); // Remove leading '/'. + } + m_dbs.push_back(std::make_unique(path, mode, 0, vfs)); + LOG("SQLite database opened: %s", path.c_str()); - bool ok = testSchema(*m_db); - if (ok) { - if (m_cacheMode && !m_schemaOptions.isCache) { - // TODO better description - LOGE("Cannot cache to 'externally created' MBTiles database"); - // Run in non-caching mode - m_cacheMode = false; - return; + } catch (std::exception& e) { + LOGE("Unable to open SQLite database: %s - %s", path.c_str(), e.what()); + if (!m_dbs.empty()) { + m_dbs[m_dbs.size() - 1].reset(); + } + continue; } - } else if (m_cacheMode) { + } - // Setup the database by running the schema.sql. - initSchema(*m_db, m_name, m_mime); + for (int i = 0; i < m_dbs.size(); ++i) { + std::unique_ptr& db = m_dbs[i]; + + bool ok = testSchema(*db); + if (ok) { + if (m_cacheMode && !m_schemaOptions.isCache) { + // TODO better description + LOGE("Cannot cache to 'externally created' MBTiles database"); + // Run in non-caching mode + m_cacheMode = false; + continue; + } + } else if (m_cacheMode) { + + // Setup the database by running the schema.sql. + initSchema(*db, m_name, m_mime); - ok = testSchema(*m_db); - if (!ok) { - LOGE("Unable to initialize MBTiles schema"); - m_db.reset(); - return; + ok = testSchema(*db); + if (!ok) { + LOGE("Unable to initialize MBTiles schema"); + db.reset(); + continue; + } + } else { + LOGE("Invalid MBTiles schema"); + db.reset(); + continue; } - } else { - LOGE("Invalid MBTiles schema"); - m_db.reset(); - return; - } - if (m_schemaOptions.compression == Compression::unsupported) { - m_db.reset(); - return; - } + if (m_schemaOptions.compression == Compression::unsupported) { + db.reset(); + continue; + } - try { - m_queries = std::make_unique(*m_db, m_cacheMode); - } catch (std::exception& e) { - LOGE("Unable to initialize queries: %s", e.what()); - m_db.reset(); - return; + try { + m_queries.push_back(std::make_unique(*db, m_cacheMode)); + } catch (std::exception& e) { + LOGE("Unable to initialize queries: %s", e.what()); + db.reset(); + } } } @@ -302,7 +312,7 @@ void MBTilesDataSource::openMBTiles() { * * @param _source A pointer to a the data source in which we will setup a db. * @return true if database contains MBTiles schema - */ +*/ bool MBTilesDataSource::testSchema(SQLite::Database& db) { bool metadata = false, tiles = false, grids = false, grid_data = false; @@ -424,51 +434,61 @@ void MBTilesDataSource::initSchema(SQLite::Database& db, std::string _name, std: bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector& _data) { - auto& stmt = m_queries->getTileData; - try { - // Google TMS to WMTS - // https://github.com/mapbox/node-mbtiles/blob/ - // 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149 - int z = _tileId.z; - int y = (1 << z) - 1 - _tileId.y; - - stmt.bind(1, z); - stmt.bind(2, _tileId.x); - stmt.bind(3, y); - - if (stmt.executeStep()) { - SQLite::Column column = stmt.getColumn(0); - const char* blob = (const char*) column.getBlob(); - const int length = column.getBytes(); - - if ((m_schemaOptions.compression == Compression::undefined) || - (m_schemaOptions.compression == Compression::deflate)) { - - if (zlib::inflate(blob, length, _data) != 0) { - if (m_schemaOptions.compression == Compression::undefined) { + int largestLength = 0; + + for (int i = 0; i < m_queries.size(); ++i) { + + auto& stmt = m_queries[i]->getTileData; + try { + // Google TMS to WMTS + // https://github.com/mapbox/node-mbtiles/blob/ + // 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149 + int z = _tileId.z; + int y = (1 << z) - 1 - _tileId.y; + + stmt.bind(1, z); + stmt.bind(2, _tileId.x); + stmt.bind(3, y); + + if (stmt.executeStep()) { + SQLite::Column column = stmt.getColumn(0); + const int length = column.getBytes(); + + // When multiple mbtiles contain the same tile, get the tile which has more data. + if (length > largestLength) { + const char* blob = (const char*) column.getBlob(); + + if ((m_schemaOptions.compression == Compression::undefined) || + (m_schemaOptions.compression == Compression::deflate)) { + + if (zlib::inflate(blob, length, _data) != 0) { + if (m_schemaOptions.compression == Compression::undefined) { + _data.resize(length); + memcpy(_data.data(), blob, length); + } else { + LOGW("Invalid deflate compression"); + } + } + } else { _data.resize(length); memcpy(_data.data(), blob, length); - } else { - LOGW("Invalid deflate compression"); } + + largestLength = length; } - } else { - _data.resize(length); - memcpy(_data.data(), blob, length); + + stmt.reset(); } - stmt.reset(); - return true; + } catch (std::exception& e) { + LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); } - - } catch (std::exception& e) { - LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what()); + try { + stmt.reset(); + } catch (...) {} } - try { - stmt.reset(); - } catch(...) {} - return false; + return !_data.empty(); } void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vector& _data) { @@ -487,7 +507,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vectorputMap; + auto& stmt = m_queries[0]->putMap; stmt.bind(1, z); stmt.bind(2, _tileId.x); stmt.bind(3, y); @@ -501,7 +521,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vectorputImage; + auto& stmt = m_queries[0]->putImage; stmt.bind(1, md5id); stmt.bind(2, data, size); stmt.exec(); diff --git a/core/src/data/mbtilesDataSource.h b/core/src/data/mbtilesDataSource.h index 2a9f65ef8a..9063337c0f 100644 --- a/core/src/data/mbtilesDataSource.h +++ b/core/src/data/mbtilesDataSource.h @@ -17,8 +17,8 @@ class AsyncWorker; class MBTilesDataSource : public TileSource::DataSource { public: - MBTilesDataSource(Platform& _platform, std::string _name, std::string _path, std::string _mime, - bool _cache = false, bool _offlineFallback = false); + MBTilesDataSource(Platform& _platform, std::string _name, std::vector _paths, + std::string _mime, bool _cache = false, bool _offlineFallback = false); ~MBTilesDataSource(); @@ -38,7 +38,7 @@ class MBTilesDataSource : public TileSource::DataSource { std::string m_name; // The path to an mbtiles tile store. - std::string m_path; + std::vector m_paths; std::string m_mime; // Store tiles from next source @@ -48,8 +48,8 @@ class MBTilesDataSource : public TileSource::DataSource { bool m_offlineMode; // Pointer to SQLite DB of MBTiles store - std::unique_ptr m_db; - std::unique_ptr m_queries; + std::vector> m_dbs; + std::vector> m_queries; std::unique_ptr m_worker; // Platform reference diff --git a/core/src/scene/sceneLoader.cpp b/core/src/scene/sceneLoader.cpp index 1369b0924a..1925c6956d 100644 --- a/core/src/scene/sceneLoader.cpp +++ b/core/src/scene/sceneLoader.cpp @@ -697,6 +697,7 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s std::string type; std::string url; + std::vector urls_mbtiles; std::string mbtiles; std::vector subdomains; @@ -712,6 +713,13 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s if (auto urlNode = _source["url"]) { url = urlNode.Scalar(); } + if (auto urlsMbtilesNode = _source["urls_mbtiles"]) { + if (urlsMbtilesNode.IsSequence()) { + for (const auto &urlMbtilesNode : urlsMbtilesNode) { + urls_mbtiles.push_back(urlMbtilesNode.Scalar()); + } + } + } if (auto minDisplayZoomNode = _source["min_display_zoom"]) { YamlUtil::getInt(minDisplayZoomNode, minDisplayZoom); } @@ -805,12 +813,15 @@ std::shared_ptr SceneLoader::loadSource(const Node& _source, const s std::unique_ptr rawSources; - if (isMBTilesFile) { + if (!urls_mbtiles.empty() || isMBTilesFile) { #ifdef TANGRAM_MBTILES_DATASOURCE // If we have MBTiles, we know the source is tiled. tiled = true; // Create an MBTiles data source from the file at the url and add it to the source chain. - rawSources = std::make_unique(_platform, _name, url, ""); + if (urls_mbtiles.empty()) { + urls_mbtiles.push_back(url); + } + rawSources = std::make_unique(_platform, _name, urls_mbtiles, ""); #else LOGE("MBTiles support is disabled. This source will be ignored: %s", _name.c_str()); return nullptr;