diff --git a/carryover.py b/carryover.py new file mode 100755 index 00000000000..60aeb7ac269 --- /dev/null +++ b/carryover.py @@ -0,0 +1,59 @@ +#!/usr/bin/env python + +import sqlite3 +import sys +import zlib + +in_db = sqlite3.connect('cache.db') + +with in_db: + in_db.row_factory = sqlite3.Row + in_cur = in_db.cursor() + in_cur.execute('select url, data, kind, compressed from http_cache where status = 1 order by kind asc') + in_rows = in_cur.fetchall() + out_db = sqlite3.connect('cache.mbtiles') + with out_db: + out_cur = out_db.cursor() + out_cur.executescript(""" + drop table if exists tiles; + create table tiles (zoom_level integer, tile_column integer, tile_row integer, tile_data blob); + create unique index tile_index on tiles (zoom_level, tile_column, tile_row); + drop table if exists metadata; + create table metadata (name text,value text); + create unique index name on metadata (name); + """) + for in_row in in_rows: + kind = in_row['kind'] + data = in_row['data'] + if in_row['compressed'] == 1: + data = zlib.decompress(data) + if kind == 3: + parts = in_row['url'].split('/') + z = int(parts[4]) + x = int(parts[5]) + y = int(parts[6].split('.')[0]) + y = pow(2, z) - y - 1 + out_cur.execute('insert into tiles (zoom_level, tile_column, tile_row, tile_data) ' \ + 'values (?, ?, ?, ?)', (z, x, y, sqlite3.Binary(data))) + print 'inserted tile %s,%s,%s (%s bytes)' % (z, x, y, len(data)) + elif kind in (2, 4, 5): + if kind == 2: + prefix = 'gl_source' + elif kind == 4: + prefix = 'gl_glyph' + else: + prefix = 'gl_sprite_image' + out_cur.execute('insert into metadata (name, value) values (?, ?)', + (prefix + '_' + in_row['url'], sqlite3.Binary(data))) + label = prefix.replace('gl_', '').replace('_', ' ') + print 'inserted %s (%s bytes)' % (label, len(data)) + elif kind in (1, 6): + if kind == 1: + prefix = 'gl_style' + else: + prefix = 'gl_sprite_metadata' + out_cur.execute('insert into metadata (name, value) values (?, ?)', + (prefix + '_' + in_row['url'], data)) + label = prefix.replace('gl_', '').replace('_', ' ') + print 'inserted %s (%s bytes)' % (label, len(data)) + out_db.commit() diff --git a/include/mbgl/ios/MGLMapView.h b/include/mbgl/ios/MGLMapView.h index e10932bd27a..262fd82f6dd 100644 --- a/include/mbgl/ios/MGLMapView.h +++ b/include/mbgl/ios/MGLMapView.h @@ -41,6 +41,8 @@ IB_DESIGNABLE * @return An initialized map view. */ - (instancetype)initWithFrame:(CGRect)frame styleURL:(nullable NSURL *)styleURL; +- (instancetype)initWithFrame:(CGRect)frame styleURL:(nullable NSURL *)styleURL offlineMapPath:(NSString *)offlineMapPath; + #pragma mark - Accessing Map Properties /** @name Accessing Map Properties */ diff --git a/include/mbgl/storage/default_file_source.hpp b/include/mbgl/storage/default_file_source.hpp index 3689b9e9329..2ea36567050 100644 --- a/include/mbgl/storage/default_file_source.hpp +++ b/include/mbgl/storage/default_file_source.hpp @@ -9,7 +9,7 @@ class FileCache; class DefaultFileSource : public FileSource { public: - DefaultFileSource(FileCache*, const std::string& root = ""); + DefaultFileSource(FileCache*, const std::string& root = "", const std::string& offlinePath = ""); ~DefaultFileSource() override; void setAccessToken(const std::string&); diff --git a/include/mbgl/util/string.hpp b/include/mbgl/util/string.hpp index 9e2b2d88192..f61441df6e2 100644 --- a/include/mbgl/util/string.hpp +++ b/include/mbgl/util/string.hpp @@ -2,6 +2,7 @@ #define MBGL_UTIL_STRING #include +#include #pragma GCC diagnostic push #pragma GCC diagnostic ignored "-Wunknown-pragmas" @@ -22,7 +23,6 @@ inline std::string toString(int8_t num) { return boost::lexical_cast(int(num)); } - template inline std::string sprintf(const char *msg, Args... args) { char res[max]; @@ -35,6 +35,17 @@ inline std::string sprintf(const std::string &msg, Args... args) { return sprintf(msg.c_str(), args...); } +inline std::vector split(const std::string& text, const std::string& delimiter) { + std::vector tokens; + size_t start = 0, end = 0; + while ((end = text.find(delimiter, start)) != std::string::npos) { + tokens.push_back(text.substr(start, end - start)); + start = end + 1; + } + tokens.push_back(text.substr(start)); + return tokens; +} + } // namespace util } // namespace mbgl diff --git a/ios/app/MBXViewController.mm b/ios/app/MBXViewController.mm index 21a6a2aeace..b96bdeac57e 100644 --- a/ios/app/MBXViewController.mm +++ b/ios/app/MBXViewController.mm @@ -60,7 +60,11 @@ - (void)viewDidLoad { [super viewDidLoad]; - self.mapView = [[MGLMapView alloc] initWithFrame:self.view.bounds]; + NSString *offlineMapPath = [[NSBundle mainBundle] pathForResource:@"cache" ofType:@"mbtiles"]; + + self.mapView = [[MGLMapView alloc] initWithFrame:self.view.bounds + styleURL:[MGLStyle streetsStyleURL] + offlineMapPath:offlineMapPath]; self.mapView.autoresizingMask = UIViewAutoresizingFlexibleWidth | UIViewAutoresizingFlexibleHeight; self.mapView.delegate = self; [self.view addSubview:self.mapView]; @@ -91,6 +95,10 @@ - (void)viewDidLoad [self.mapView addGestureRecognizer:[[UILongPressGestureRecognizer alloc] initWithTarget:self action:@selector(handleLongPress:)]]; [self restoreState:nil]; + + [self.mapView setCenterCoordinate:CLLocationCoordinate2DMake(45.468780541878154, -122.65276581137326) + zoomLevel:15 + animated:YES]; } - (void)saveState:(__unused NSNotification *)notification diff --git a/ios/app/cache.mbtiles b/ios/app/cache.mbtiles new file mode 100644 index 00000000000..1f477a055be Binary files /dev/null and b/ios/app/cache.mbtiles differ diff --git a/ios/app/mapboxgl-app.gypi b/ios/app/mapboxgl-app.gypi index 799cb62e783..3646ba67c47 100644 --- a/ios/app/mapboxgl-app.gypi +++ b/ios/app/mapboxgl-app.gypi @@ -14,7 +14,8 @@ './polyline.geojson', './threestates.geojson', './Settings.bundle/', - './app-info.plist' + './app-info.plist', + './cache.mbtiles' ], 'dependencies': [ diff --git a/platform/default/sqlite3.cpp b/platform/default/sqlite3.cpp index 742c7e3217d..89756561c73 100644 --- a/platform/default/sqlite3.cpp +++ b/platform/default/sqlite3.cpp @@ -1,3 +1,5 @@ +#include + #include "sqlite3.hpp" #include @@ -24,8 +26,15 @@ const static bool sqliteVersionCheck = []() { namespace mapbox { namespace sqlite { +void trace_callback(void* udp, const char* sql) { + (void)udp; + printf("%s\n", sql); +// mbgl::Log::Debug(mbgl::Event::Database, sql); +}; + Database::Database(const std::string &filename, int flags) { const int err = sqlite3_open_v2(filename.c_str(), &db, flags, nullptr); +// sqlite3_trace(db, trace_callback, NULL); if (err != SQLITE_OK) { const auto message = sqlite3_errmsg(db); db = nullptr; diff --git a/platform/ios/src/MGLMapView.mm b/platform/ios/src/MGLMapView.mm index d6980e16569..b5202d3c324 100644 --- a/platform/ios/src/MGLMapView.mm +++ b/platform/ios/src/MGLMapView.mm @@ -183,6 +183,8 @@ @implementation MGLMapView BOOL _delegateHasStrokeColorsForShapeAnnotations; BOOL _delegateHasFillColorsForShapeAnnotations; BOOL _delegateHasLineWidthsForShapeAnnotations; + + NSString *_offlineMapPath; } #pragma mark - Setup & Teardown - @@ -214,6 +216,17 @@ - (instancetype)initWithFrame:(CGRect)frame styleURL:(nullable NSURL *)styleURL return self; } +- (instancetype)initWithFrame:(CGRect)frame styleURL:(nullable NSURL *)styleURL offlineMapPath:(NSString *)offlineMapPath +{ + if (self = [super initWithFrame:frame]) + { + _offlineMapPath = [offlineMapPath copy]; + [self commonInit]; + self.styleURL = styleURL; + } + return self; +} + - (instancetype)initWithCoder:(nonnull NSCoder *)decoder { if (self = [super initWithCoder:decoder]) @@ -280,10 +293,14 @@ - (void)commonInit fileCachePath = [libraryDirectory stringByAppendingPathComponent:@"cache.db"]; } _mbglFileCache = mbgl::SharedSQLiteCache::get([fileCachePath UTF8String]); - _mbglFileSource = new mbgl::DefaultFileSource(_mbglFileCache.get()); + _mbglFileSource = new mbgl::DefaultFileSource(_mbglFileCache.get(), "", _offlineMapPath ? _offlineMapPath.UTF8String : ""); // setup mbgl map - _mbglMap = new mbgl::Map(*_mbglView, *_mbglFileSource, mbgl::MapMode::Continuous); + _mbglMap = new mbgl::Map(*_mbglView, + *_mbglFileSource, + mbgl::MapMode::Continuous, + mbgl::GLContextMode::Unique, + mbgl::ConstrainMode::HeightOnly); // setup refresh driver _displayLink = [CADisplayLink displayLinkWithTarget:self selector:@selector(updateFromDisplayLink)]; diff --git a/src/mbgl/storage/default_file_source.cpp b/src/mbgl/storage/default_file_source.cpp index b2ab5abd6cc..c4afd9820e4 100644 --- a/src/mbgl/storage/default_file_source.cpp +++ b/src/mbgl/storage/default_file_source.cpp @@ -1,19 +1,22 @@ #include #include +#include namespace mbgl { class DefaultFileSource::Impl { public: - Impl(FileCache* cache, const std::string& root) - : onlineFileSource(cache, root) { + Impl(FileCache* cache, const std::string& root, const std::string& offlinePath) + : offlineFileSource(offlinePath), + onlineFileSource(cache, root) { } + OfflineFileSource offlineFileSource; OnlineFileSource onlineFileSource; }; -DefaultFileSource::DefaultFileSource(FileCache* cache, const std::string& root) - : impl(std::make_unique(cache, root)) { +DefaultFileSource::DefaultFileSource(FileCache* cache, const std::string& root, const std::string& offlinePath) + : impl(std::make_unique(cache, root, offlinePath)) { } DefaultFileSource::~DefaultFileSource() = default; @@ -27,7 +30,11 @@ std::string DefaultFileSource::getAccessToken() const { } std::unique_ptr DefaultFileSource::request(const Resource& resource, Callback callback) { - return impl->onlineFileSource.request(resource, callback); + if (impl->offlineFileSource.handlesResource(resource)) { + return impl->offlineFileSource.request(resource, callback); + } else { + return impl->onlineFileSource.request(resource, callback); + } } } // namespace mbgl diff --git a/src/mbgl/storage/offline_file_source.cpp b/src/mbgl/storage/offline_file_source.cpp new file mode 100644 index 00000000000..70affd228d6 --- /dev/null +++ b/src/mbgl/storage/offline_file_source.cpp @@ -0,0 +1,207 @@ +#include +#include + +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +#include "sqlite3.hpp" +#include + +namespace mbgl { + +using namespace mapbox::sqlite; + +class OfflineFileRequest : public FileRequest { +public: + OfflineFileRequest(const Resource& resource_, + OfflineFileSource& fileSource_) + : resource(resource_), + fileSource(fileSource_) { + } + + Resource resource; + OfflineFileSource& fileSource; + + std::unique_ptr workRequest; +}; + +OfflineFileSource::OfflineFileSource(const std::string& path) + : thread(std::make_unique>(util::ThreadContext{ "OfflineFileSource", util::ThreadType::Unknown, util::ThreadPriority::Low }, path)) { +} + +OfflineFileSource::~OfflineFileSource() = default; + +class OfflineFileSource::Impl { +public: + explicit Impl(const std::string& path); + ~Impl(); + + bool handlesResource(const Resource&); + void handleRequest(Resource, FileRequest*, Callback); + +private: + void openDatabase(); + std::unique_ptr statementForResource(const Resource&, bool checkOnly = false); + +private: + const std::string path; + std::unique_ptr<::mapbox::sqlite::Database> db; +}; + +OfflineFileSource::Impl::Impl(const std::string& path_) + : path(path_) { +} + +OfflineFileSource::Impl::~Impl() { + try { + db.reset(); + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } +} + +void OfflineFileSource::Impl::openDatabase() { + db = std::make_unique(path.c_str(), ReadOnly); +} + +std::unique_ptr OfflineFileSource::Impl::statementForResource(const Resource& resource, bool checkOnly) { + try { + if (!db) { + openDatabase(); + } + + if (resource.kind == Resource::Kind::Tile) { + + const auto canonicalURL = util::mapbox::canonicalURL(resource.url); + auto parts = util::split(canonicalURL, "/"); + const int8_t z = atoi(parts[parts.size() - 3].c_str()); + const int32_t x = atoi(parts[parts.size() - 2].c_str()); + const int32_t y = atoi(util::split(util::split(parts[parts.size() - 1], ".")[0], "@")[0].c_str()); + + const auto id = TileID(z, x, (pow(2, z) - y - 1), z); // flip y for MBTiles + + const auto sql = (checkOnly ? + "SELECT COUNT(`tile_data`) FROM `tiles` WHERE `zoom_level` = ? AND `tile_column` = ? AND `tile_row` = ?" : + "SELECT `tile_data` FROM `tiles` WHERE `zoom_level` = ? AND `tile_column` = ? AND `tile_row` = ?"); + + Statement getStmt = db->prepare(sql); + + getStmt.bind(1, (int)id.z); + getStmt.bind(2, (int)id.x); + getStmt.bind(3, (int)id.y); + + return std::make_unique(std::move(getStmt)); + + } else if (resource.kind != Resource::Kind::Unknown) { + + std::string key = ""; + if (resource.kind == Resource::Kind::Glyphs) { + key = "gl_glyph"; + } else if (resource.kind == Resource::Kind::Source) { + key = "gl_source"; + } else if (resource.kind == Resource::Kind::SpriteImage) { + key = "gl_sprite_image"; + } else if (resource.kind == Resource::Kind::SpriteJSON) { + key = "gl_sprite_metadata"; + } else if (resource.kind == Resource::Kind::Style) { + key = "gl_style"; + } + assert(key.length()); + + const auto sql = (checkOnly ? + "SELECT COUNT(`value`) FROM `metadata` WHERE `name` = ?" : + "SELECT `value` FROM `metadata` WHERE `name` = ?"); + + Statement getStmt = db->prepare(sql); + + const auto name = key + "_" + util::mapbox::canonicalURL(resource.url); + getStmt.bind(1, name.c_str()); + + return std::make_unique(std::move(getStmt)); + + } + } catch (mapbox::sqlite::Exception& ex) { + Log::Error(Event::Database, ex.code, ex.what()); + } catch (std::runtime_error& ex) { + Log::Error(Event::Database, ex.what()); + } + + return nullptr; +} + +bool OfflineFileSource::Impl::handlesResource(const Resource& resource) { + const auto getStmt = statementForResource(resource, true); + + if (getStmt != nullptr) { + return (getStmt->run() && getStmt->get(0) > 0); + } + + return false; +} + +void OfflineFileSource::Impl::handleRequest(Resource resource, FileRequest* req, Callback callback) { + (void)req; + + std::shared_ptr res = std::make_shared(); + + const auto getStmt = statementForResource(resource); + + if (getStmt != nullptr) { + if (getStmt->run()) { + res->data = std::make_shared(std::move(getStmt->get(0))); + } else { + res->error = std::make_unique(Response::Error::Reason::NotFound); + } + } + + callback(*res); +} + +bool OfflineFileSource::handlesResource(const Resource& res) { + return thread->invokeSync(&Impl::handlesResource, res); +} + +std::unique_ptr OfflineFileSource::request(const Resource& resource, Callback callback) { + if (!callback) { + throw util::MisuseException("FileSource callback can't be empty"); + } + + std::string url; + + switch (resource.kind) { + case Resource::Kind::Style: + url = mbgl::util::mapbox::normalizeStyleURL(resource.url, "foo"); + break; + + case Resource::Kind::Source: + url = util::mapbox::normalizeSourceURL(resource.url, "foo"); + break; + + case Resource::Kind::Glyphs: + url = util::mapbox::normalizeGlyphsURL(resource.url, "foo"); + break; + + case Resource::Kind::SpriteImage: + case Resource::Kind::SpriteJSON: + url = util::mapbox::normalizeSpriteURL(resource.url, "foo"); + break; + + default: + url = resource.url; + } + + Resource res { resource.kind, url }; + auto req = std::make_unique(res, *this); + req->workRequest = thread->invokeWithCallback(&Impl::handleRequest, callback, res, req.get()); + return std::move(req); +} + +} // namespace mbgl diff --git a/src/mbgl/storage/offline_file_source.hpp b/src/mbgl/storage/offline_file_source.hpp new file mode 100644 index 00000000000..b8696395fe1 --- /dev/null +++ b/src/mbgl/storage/offline_file_source.hpp @@ -0,0 +1,30 @@ +#ifndef MBGL_STORAGE_OFFLINE_FILE_SOURCE +#define MBGL_STORAGE_OFFLINE_FILE_SOURCE + +#include +#include + +namespace mbgl { + +namespace util { +template class Thread; +} // namespace util + +class OfflineFileSource : public FileSource { +public: + OfflineFileSource(const std::string& path); + ~OfflineFileSource() override; + + bool handlesResource(const Resource&); + std::unique_ptr request(const Resource&, Callback) override; + +public: + friend class FrontlineFileRequest; + + class Impl; + const std::unique_ptr> thread; +}; + +} // namespace mbgl + +#endif diff --git a/test.mbtiles b/test.mbtiles new file mode 100644 index 00000000000..1921b6cb87a Binary files /dev/null and b/test.mbtiles differ