Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add support for multiple mbtiles #2111

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open

Add support for multiple mbtiles #2111

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
ace4735
Add support for multiple mbtiles files through urls_mbtiles
rwrx Nov 29, 2019
8e0b64e
Fix spaces
Dec 18, 2019
86b9dd9
Fix return value in MBTilesDataSource::getTileData
Dec 18, 2019
5029946
Address changes
rwrx Dec 20, 2019
0be3933
Add comment
rwrx Dec 20, 2019
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
221 changes: 120 additions & 101 deletions core/src/data/mbtilesDataSource.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -115,10 +115,11 @@ struct MBTilesQueries {

};

MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name, std::string _path,
std::string _mime, bool _cache, bool _offlineFallback)
MBTilesDataSource::MBTilesDataSource(Platform& _platform, std::string _name,
std::vector<std::string> _paths, std::string _mime,
bool _cache, bool _offlineFallback)
: m_name(_name),
m_path(_path),
m_paths(_paths),
m_mime(_mime),
m_cacheMode(_cache),
m_offlineMode(_offlineFallback),
Expand All @@ -143,7 +144,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr<TileTask> _task, TileTaskCb
return loadNextSource(_task, _cb);
}

if (!m_db) { return false; }
if (m_dbs.empty()) { return false; }

if (_task->rawSource == this->level) {

Expand Down Expand Up @@ -184,7 +185,7 @@ bool MBTilesDataSource::loadTileData(std::shared_ptr<TileTask> _task, TileTaskCb
bool MBTilesDataSource::loadNextSource(std::shared_ptr<TileTask> _task, TileTaskCb _cb) {
if (!next) { return false; }

if (!m_db) {
if (m_dbs.empty()) {
return next->loadTileData(_task, _cb);
}

Expand All @@ -196,12 +197,12 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr<TileTask> _task, TileTask
if (m_cacheMode) {
m_worker->enqueue([this, _task](){

auto& task = static_cast<BinaryTileTask&>(*_task);
auto& task = static_cast<BinaryTileTask&>(*_task);

LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData());
LOGW("store tile: %s, %d", _task->tileId().toString().c_str(), task.hasData());

storeTileData(_task->tileId(), *task.rawTileData);
});
storeTileData(_task->tileId(), *task.rawTileData);
});
}

_cb.func(_task);
Expand Down Expand Up @@ -232,77 +233,86 @@ bool MBTilesDataSource::loadNextSource(std::shared_ptr<TileTask> _task, TileTask

void MBTilesDataSource::openMBTiles() {

try {
auto mode = SQLite::OPEN_READONLY;
if (m_cacheMode) {
// Need to explicitly open a SQLite DB with OPEN_READWRITE
// and OPEN_CREATE flags to make a file and write.
mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE;
}
for (int i = 0; i < m_paths.size(); ++i) {
std::string path = m_paths[i];

auto url = Url(m_path);
auto path = url.path();
const char* vfs = "";
if (url.scheme() == "asset") {
vfs = "ndk-asset";
path.erase(path.begin()); // Remove leading '/'.
}
m_db = std::make_unique<SQLite::Database>(path, mode, 0, vfs);
LOG("SQLite database opened: %s", path.c_str());
try {
auto mode = SQLite::OPEN_READONLY;
if (m_cacheMode) {
// Need to explicitly open a SQLite DB with OPEN_READWRITE
// and OPEN_CREATE flags to make a file and write.
mode = SQLite::OPEN_READWRITE | SQLite::OPEN_CREATE;
}

} catch (std::exception& e) {
LOGE("Unable to open SQLite database: %s - %s", m_path.c_str(), e.what());
m_db.reset();
return;
}
auto url = Url(path);
auto path = url.path();
const char* vfs = "";
if (url.scheme() == "asset") {
vfs = "ndk-asset";
path.erase(path.begin()); // Remove leading '/'.
}
m_dbs.push_back(std::make_unique<SQLite::Database>(path, mode, 0, vfs));
LOG("SQLite database opened: %s", path.c_str());

bool ok = testSchema(*m_db);
if (ok) {
if (m_cacheMode && !m_schemaOptions.isCache) {
// TODO better description
LOGE("Cannot cache to 'externally created' MBTiles database");
// Run in non-caching mode
m_cacheMode = false;
} catch (std::exception& e) {
LOGE("Unable to open SQLite database: %s - %s", path.c_str(), e.what());
if (!m_dbs.empty()) {
m_dbs[m_dbs.size() - 1].reset();
}
return;
Copy link
Member

@tallytalwar tallytalwar Dec 20, 2019 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Shouldn't you be doing a continue here to try the other set of mbtile path?

}
} else if (m_cacheMode) {

// Setup the database by running the schema.sql.
initSchema(*m_db, m_name, m_mime);
if (!m_dbs.empty()) {
std::unique_ptr<SQLite::Database>& db = m_dbs[m_dbs.size() - 1];

bool ok = testSchema(*db);
if (ok) {
if (m_cacheMode && !m_schemaOptions.isCache) {
// TODO better description
LOGE("Cannot cache to 'externally created' MBTiles database");
// Run in non-caching mode
m_cacheMode = false;
return;
}
} else if (m_cacheMode) {

// Setup the database by running the schema.sql.
initSchema(*db, m_name, m_mime);

ok = testSchema(*m_db);
if (!ok) {
LOGE("Unable to initialize MBTiles schema");
m_db.reset();
return;
}
} else {
LOGE("Invalid MBTiles schema");
m_db.reset();
return;
}
ok = testSchema(*db);
if (!ok) {
LOGE("Unable to initialize MBTiles schema");
db.reset();
return;
}
} else {
LOGE("Invalid MBTiles schema");
db.reset();
return;
}

if (m_schemaOptions.compression == Compression::unsupported) {
m_db.reset();
return;
}
if (m_schemaOptions.compression == Compression::unsupported) {
db.reset();
return;
}

try {
m_queries = std::make_unique<MBTilesQueries>(*m_db, m_cacheMode);
} catch (std::exception& e) {
LOGE("Unable to initialize queries: %s", e.what());
m_db.reset();
return;
try {
m_queries.push_back(std::make_unique<MBTilesQueries>(*db, m_cacheMode));
} catch (std::exception &e) {
LOGE("Unable to initialize queries: %s", e.what());
db.reset();
}
}
}
}

/**
* We check to see if the database has the MBTiles Schema.
* Sets m_schemaOptions from metadata table
*
* @param _source A pointer to a the data source in which we will setup a db.
* @return true if database contains MBTiles schema
*/
* We check to see if the database has the MBTiles Schema.
* Sets m_schemaOptions from metadata table
*
* @param _source A pointer to a the data source in which we will setup a db.
* @return true if database contains MBTiles schema
*/
bool MBTilesDataSource::testSchema(SQLite::Database& db) {

bool metadata = false, tiles = false, grids = false, grid_data = false;
Expand Down Expand Up @@ -424,51 +434,60 @@ void MBTilesDataSource::initSchema(SQLite::Database& db, std::string _name, std:

bool MBTilesDataSource::getTileData(const TileID& _tileId, std::vector<char>& _data) {

auto& stmt = m_queries->getTileData;
try {
// Google TMS to WMTS
// https://github.com/mapbox/node-mbtiles/blob/
// 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149
int z = _tileId.z;
int y = (1 << z) - 1 - _tileId.y;
int largestLength = 0;

stmt.bind(1, z);
stmt.bind(2, _tileId.x);
stmt.bind(3, y);
for (int i = 0; i < m_queries.size(); ++i) {

auto& stmt = m_queries[i]->getTileData;
try {
// Google TMS to WMTS
// https://github.com/mapbox/node-mbtiles/blob/
// 4bbfaf991969ce01c31b95184c4f6d5485f717c3/lib/mbtiles.js#L149
int z = _tileId.z;
int y = (1 << z) - 1 - _tileId.y;

stmt.bind(1, z);
stmt.bind(2, _tileId.x);
stmt.bind(3, y);

if (stmt.executeStep()) {
SQLite::Column column = stmt.getColumn(0);
const int length = column.getBytes();

if (stmt.executeStep()) {
SQLite::Column column = stmt.getColumn(0);
const char* blob = (const char*) column.getBlob();
const int length = column.getBytes();
if (length > largestLength) {
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Can you add a comment about this check please?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

If multiple mbtiles contain the same tile, it will get tile which has more data.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Sounds great. Can you put this in the code comment please, will be helpful for anyone looking at the code in future. Thanks

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Ok, I have added it.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Thanks @rwrx

const char *blob = (const char *) column.getBlob();

if ((m_schemaOptions.compression == Compression::undefined) ||
(m_schemaOptions.compression == Compression::deflate)) {
if ((m_schemaOptions.compression == Compression::undefined) ||
(m_schemaOptions.compression == Compression::deflate)) {

if (zlib::inflate(blob, length, _data) != 0) {
if (m_schemaOptions.compression == Compression::undefined) {
if (zlib::inflate(blob, length, _data) != 0) {
if (m_schemaOptions.compression == Compression::undefined) {
_data.resize(length);
memcpy(_data.data(), blob, length);
} else {
LOGW("Invalid deflate compression");
}
}
} else {
_data.resize(length);
memcpy(_data.data(), blob, length);
} else {
LOGW("Invalid deflate compression");
}

largestLength = length;
}
} else {
_data.resize(length);
memcpy(_data.data(), blob, length);

stmt.reset();
}

stmt.reset();
return true;
} catch (std::exception& e) {
LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what());
}

} catch (std::exception& e) {
LOGE("MBTiles SQLite get tile_data statement failed: %s", e.what());
try {
stmt.reset();
} catch (...) {}
}
try {
stmt.reset();
} catch(...) {}

return false;
return !_data.empty();
}

void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vector<char>& _data) {
Expand All @@ -487,7 +506,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vector<c
std::string md5id = md5(data, size);

try {
auto& stmt = m_queries->putMap;
auto& stmt = m_queries[0]->putMap;
stmt.bind(1, z);
stmt.bind(2, _tileId.x);
stmt.bind(3, y);
Expand All @@ -501,7 +520,7 @@ void MBTilesDataSource::storeTileData(const TileID& _tileId, const std::vector<c
}

try {
auto& stmt = m_queries->putImage;
auto& stmt = m_queries[0]->putImage;
stmt.bind(1, md5id);
stmt.bind(2, data, size);
stmt.exec();
Expand Down
10 changes: 5 additions & 5 deletions core/src/data/mbtilesDataSource.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ class AsyncWorker;
class MBTilesDataSource : public TileSource::DataSource {
public:

MBTilesDataSource(Platform& _platform, std::string _name, std::string _path, std::string _mime,
bool _cache = false, bool _offlineFallback = false);
MBTilesDataSource(Platform& _platform, std::string _name, std::vector<std::string> _paths,
std::string _mime, bool _cache = false, bool _offlineFallback = false);

~MBTilesDataSource();

Expand All @@ -38,7 +38,7 @@ class MBTilesDataSource : public TileSource::DataSource {
std::string m_name;

// The path to an mbtiles tile store.
std::string m_path;
std::vector<std::string> m_paths;
std::string m_mime;

// Store tiles from next source
Expand All @@ -48,8 +48,8 @@ class MBTilesDataSource : public TileSource::DataSource {
bool m_offlineMode;

// Pointer to SQLite DB of MBTiles store
std::unique_ptr<SQLite::Database> m_db;
std::unique_ptr<MBTilesQueries> m_queries;
std::vector<std::unique_ptr<SQLite::Database>> m_dbs;
std::vector<std::unique_ptr<MBTilesQueries>> m_queries;
std::unique_ptr<AsyncWorker> m_worker;

// Platform reference
Expand Down
15 changes: 13 additions & 2 deletions core/src/scene/sceneLoader.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -697,6 +697,7 @@ std::shared_ptr<TileSource> SceneLoader::loadSource(const Node& _source, const s

std::string type;
std::string url;
std::vector<std::string> urls_mbtiles;
std::string mbtiles;
std::vector<std::string> subdomains;

Expand All @@ -712,6 +713,13 @@ std::shared_ptr<TileSource> SceneLoader::loadSource(const Node& _source, const s
if (auto urlNode = _source["url"]) {
url = urlNode.Scalar();
}
if (auto urlsMbtilesNode = _source["urls_mbtiles"]) {
if (urlsMbtilesNode.IsSequence()) {
for (const auto &urlMbtilesNode : urlsMbtilesNode) {
urls_mbtiles.push_back(urlMbtilesNode.Scalar());
}
}
}
if (auto minDisplayZoomNode = _source["min_display_zoom"]) {
YamlUtil::getInt(minDisplayZoomNode, minDisplayZoom);
}
Expand Down Expand Up @@ -805,12 +813,15 @@ std::shared_ptr<TileSource> SceneLoader::loadSource(const Node& _source, const s

std::unique_ptr<TileSource::DataSource> rawSources;

if (isMBTilesFile) {
if (!urls_mbtiles.empty() || isMBTilesFile) {
#ifdef TANGRAM_MBTILES_DATASOURCE
// If we have MBTiles, we know the source is tiled.
tiled = true;
// Create an MBTiles data source from the file at the url and add it to the source chain.
rawSources = std::make_unique<MBTilesDataSource>(_platform, _name, url, "");
if (urls_mbtiles.empty()) {
urls_mbtiles.push_back(url);
}
rawSources = std::make_unique<MBTilesDataSource>(_platform, _name, urls_mbtiles, "");
#else
LOGE("MBTiles support is disabled. This source will be ignored: %s", _name.c_str());
return nullptr;
Expand Down