Offline (mbgl-core implementation)

.gitignore

@@ -4,6 +4,7 @@
 *.actual.png
 *.diff.png
 *.pyc
+offline.db
 /platform/android/debug
 /platform/android/sdk
 /platform/android/**/.classpath
@@ -23,8 +24,8 @@
 /test/fixtures/api/1.png
 /test/fixtures/api/2.png
 /test/fixtures/database/*.db
-/test/fixtures/*/*/actual.png
-/test/fixtures/*/*/diff.png
+/test/fixtures/**/actual.png
+/test/fixtures/**/diff.png
 /test/output
 /include/mbgl/shader/shaders.hpp
 /src/shader/shaders_gl.cpp

Makefile

@@ -116,11 +116,11 @@ ifeq ($(BUILD),osx)
 xtest: ; $(RUN) HOST=osx HOST_VERSION=x86_64 Xcode/test
 endif
 
-.PHONY: render xrender
+.PHONY: render
 render: ; $(RUN) Makefile/mbgl-render
-ifeq ($(BUILD),osx)
-xrender: ; $(RUN) HOST=osx HOST_VERSION=x86_64 Xcode/mbgl-render
-endif
+
+.PHONY: offline
+offline: ; $(RUN) Makefile/mbgl-offline
 
 
 ##### Maintenace operations ####################################################

bin/offline.cpp

@@ -0,0 +1,76 @@
+#include <mbgl/util/run_loop.hpp>
+#include <mbgl/util/string.hpp>
+#include <mbgl/util/io.hpp>
+
+#include <mbgl/storage/default_file_source.hpp>
+
+#include <cstdlib>
+#include <iostream>
+
+using namespace std::literals::chrono_literals;
+
+int main(int, char * []) {
+ using namespace mbgl;
+
+ unlink("offline.db");
+
+ util::RunLoop loop;
+ DefaultFileSource fileSource("offline.db", ".");
+
+ fileSource.setAccessToken(getenv("MAPBOX_ACCESS_TOKEN"));
+
+ LatLngBounds bayArea = LatLngBounds::hull(LatLng(37.2, -122.8), LatLng(38.1, -121.7));
+ OfflineTilePyramidRegionDefinition definition("mapbox://styles/mapbox/streets-v8", bayArea, 0, 15, 1.0);
+ OfflineRegionMetadata metadata;
+
+ class Observer : public OfflineRegionObserver {
+ public:
+ Observer(util::RunLoop& loop_)
+ : loop(loop_),
+ start(SystemClock::now()) {
+ }
+
+ void statusChanged(OfflineRegionStatus status) override {
+ std::string bytesPerSecond = "-";
+
+ auto elapsedSeconds = (SystemClock::now() - start) / 1s;
+ if (elapsedSeconds != 0) {
+ bytesPerSecond = util::toString(status.completedResourceSize / elapsedSeconds);
+ }
+
+ std::cout << status.completedResourceCount << " / " << status.requiredResourceCount
+ << " resources"
+ << (status.requiredResourceCountIsIndeterminate ? " (indeterminate); " : "; ")
+ << status.completedResourceSize << " bytes downloaded"
+ << " (" << bytesPerSecond << " bytes/sec)"
+ << std::endl;
+
+ if (status.complete()) {
+ std::cout << "Finished" << std::endl;
+ loop.stop();
+ }
+ }
+
+ void responseError(Response::Error error) override {
+ std::cerr << error.reason << " downloading resource: " << error.message << std::endl;
+ }
+
+ util::RunLoop& loop;
+ SystemTimePoint start;
+ };
+
+ fileSource.createOfflineRegion(definition, metadata, [&] (std::exception_ptr error, optional<OfflineRegion> region) {
+ if (error) {
+ std::cerr << "Error creating region: " << util::toString(error) << std::endl;
+ loop.stop();
+ exit(1);
+ } else {
+ assert(region);
+ fileSource.setOfflineRegionObserver(*region, std::make_unique<Observer>(loop));
+ fileSource.setOfflineRegionDownloadState(*region, OfflineRegionDownloadState::Active);
+ }
+ });
+
+ loop.run();
+ return 0;
+}

bin/offline.gypi

@@ -0,0 +1,44 @@
+{
+ 'includes': [
+ '../gyp/common.gypi',
+ ],
+ 'targets': [
+ { 'target_name': 'mbgl-offline',
+ 'product_name': 'mbgl-offline',
+ 'type': 'executable',
+
+ 'dependencies': [
+ 'mbgl.gyp:core',
+ 'mbgl.gyp:platform-<(platform_lib)',
+ 'mbgl.gyp:headless-<(headless_lib)',
+ 'mbgl.gyp:http-<(http_lib)',
+ 'mbgl.gyp:asset-<(asset_lib)',
+ 'mbgl.gyp:copy_certificate_bundle',
+ ],
+
+ 'include_dirs': [
+ '../src',
+ ],
+
+ 'sources': [
+ './offline.cpp',
+ ],
+
+ 'variables' : {
+ 'cflags_cc': [
+ '<@(boost_cflags)',
+ ],
+ },
+
+ 'conditions': [
+ ['OS == "mac"', {
+ 'xcode_settings': {
+ 'OTHER_CPLUSPLUSFLAGS': [ '<@(cflags_cc)' ],
+ }
+ }, {
+ 'cflags_cc': [ '<@(cflags_cc)' ],
+ }]
+ ],
+ },
+ ],
+}

gyp/osx.gyp

@@ -6,5 +6,6 @@
  '../platform/linux/mapboxgl-app.gypi',
  '../test/test.gypi',
  '../bin/render.gypi',
+ '../bin/offline.gypi',
  ],
 }

gyp/platform-android.gypi

@@ -21,7 +21,12 @@
  '../platform/default/timer.cpp',
  '../platform/default/default_file_source.cpp',
  '../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline.hpp',
+ '../platform/default/mbgl/storage/offline.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
+ '../platform/default/mbgl/storage/offline_download.hpp',
+ '../platform/default/mbgl/storage/offline_download.cpp',
  '../platform/default/sqlite3.hpp',
  '../platform/default/sqlite3.cpp',
  ],
@@ -34,6 +39,7 @@
  '<@(nunicode_cflags)',
  '<@(boost_cflags)',
  '<@(sqlite_cflags)',
+ '<@(rapidjson_cflags)',
  ],
  'ldflags': [
  '<@(libpng_ldflags)',
@@ -56,6 +62,7 @@
  'include_dirs': [
  '../include',
  '../src',
+ '../platform/default',
  ],
 
  'conditions': [

gyp/platform-ios.gypi

@@ -16,7 +16,12 @@
  '../platform/default/timer.cpp',
  '../platform/default/default_file_source.cpp',
  '../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline.hpp',
+ '../platform/default/mbgl/storage/offline.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
+ '../platform/default/mbgl/storage/offline_download.hpp',
+ '../platform/default/mbgl/storage/offline_download.cpp',
  '../platform/default/sqlite3.hpp',
  '../platform/default/sqlite3.cpp',
  '../platform/darwin/log_nslog.mm',
@@ -71,6 +76,7 @@
  '<@(boost_cflags)',
  '<@(sqlite_cflags)',
  '<@(zlib_cflags)',
+ '<@(rapidjson_cflags)',
  ],
  'ldflags': [
  '<@(sqlite_ldflags)',
@@ -98,6 +104,7 @@
  '../include/mbgl/darwin',
  '../include',
  '../src',
+ '../platform/default',
  ],
 
  'xcode_settings': {

gyp/platform-linux.gypi

@@ -23,7 +23,12 @@
  '../platform/default/timer.cpp',
  '../platform/default/default_file_source.cpp',
  '../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline.hpp',
+ '../platform/default/mbgl/storage/offline.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
+ '../platform/default/mbgl/storage/offline_download.hpp',
+ '../platform/default/mbgl/storage/offline_download.cpp',
  '../platform/default/sqlite3.hpp',
  '../platform/default/sqlite3.cpp',
  ],
@@ -37,6 +42,7 @@
  '<@(boost_cflags)',
  '<@(sqlite_cflags)',
  '<@(webp_cflags)',
+ '<@(rapidjson_cflags)',
  ],
  'ldflags': [
  '<@(libpng_ldflags)',
@@ -61,6 +67,7 @@
  'include_dirs': [
  '../include',
  '../src',
+ '../platform/default',
  ],
 
  'conditions': [

gyp/platform-osx.gypi

@@ -15,7 +15,12 @@
  '../platform/default/timer.cpp',
  '../platform/default/default_file_source.cpp',
  '../platform/default/online_file_source.cpp',
- '../platform/default/sqlite_cache.cpp',
+ '../platform/default/mbgl/storage/offline.hpp',
+ '../platform/default/mbgl/storage/offline.cpp',
+ '../platform/default/mbgl/storage/offline_database.hpp',
+ '../platform/default/mbgl/storage/offline_database.cpp',
+ '../platform/default/mbgl/storage/offline_download.hpp',
+ '../platform/default/mbgl/storage/offline_download.cpp',
  '../platform/default/sqlite3.hpp',
  '../platform/default/sqlite3.cpp',
  '../platform/darwin/log_nslog.mm',
@@ -62,6 +67,7 @@
  '<@(boost_cflags)',
  '<@(sqlite_cflags)',
  '<@(zlib_cflags)',
+ '<@(rapidjson_cflags)',
  ],
  'ldflags': [
  '<@(zlib_ldflags)',
@@ -83,6 +89,7 @@
  '../include/mbgl/darwin',
  '../include',
  '../src',
+ '../platform/default',
  ],
 
  'xcode_settings': {

include/mbgl/storage/default_file_source.hpp

@@ -2,25 +2,100 @@
 #define MBGL_STORAGE_DEFAULT_FILE_SOURCE
 
 #include <mbgl/storage/file_source.hpp>
+#include <mbgl/storage/offline.hpp>
+#include <mbgl/util/constants.hpp>
+
+#include <vector>
 
 namespace mbgl {
 
+namespace util {
+template <typename T> class Thread;
+} // namespace util
+
 class DefaultFileSource : public FileSource {
 public:
- DefaultFileSource(const std::string& cachePath, const std::string& assetRoot);
+ DefaultFileSource(const std::string& cachePath,
+ const std::string& assetRoot,
+ uint64_t maximumCacheSize = util::DEFAULT_MAX_CACHE_SIZE);
  ~DefaultFileSource() override;
 
  void setAccessToken(const std::string&);
  std::string getAccessToken() const;
 
- void setMaximumCacheSize(uint64_t size);
- void setMaximumCacheEntrySize(uint64_t size);
-
  std::unique_ptr<FileRequest> request(const Resource&, Callback) override;
 
-private:
+ /*
+ * Retrieve all regions in the offline database.
+ *
+ * The query will be executed asynchronously and the results passed to the given
+ * callback, which will be executed on the database thread; it is the responsibility
+ * of the SDK bindings to re-execute a user-provided callback on the main thread.
+ */
+ void listOfflineRegions(std::function<void (std::exception_ptr,
+ optional<std::vector<OfflineRegion>>)>);
+
+ /*
+ * Create an offline region in the database.
+ *
+ * When the initial database queries have completed, the provided callback will be
+ * executed on the database thread; it is the responsibility of the SDK bindings
+ * to re-execute a user-provided callback on the main thread.
+ *
+ * Note that the resulting region will be in an inactive download state; to begin
+ * downloading resources, call `setOfflineRegionDownloadState(OfflineRegionDownloadState::Active)`,
+ * optionally registering an `OfflineRegionObserver` beforehand.
+ */
+ void createOfflineRegion(const OfflineRegionDefinition& definition,
+ const OfflineRegionMetadata& metadata,
+ std::function<void (std::exception_ptr,
+ optional<OfflineRegion>)>);
+
+ /*
+ * Register an observer to be notified when the state of the region changes.
+ */
+ void setOfflineRegionObserver(OfflineRegion&, std::unique_ptr<OfflineRegionObserver>);
+
+ /*
+ * Pause or resume downloading of regional resources.
+ */
+ void setOfflineRegionDownloadState(OfflineRegion&, OfflineRegionDownloadState);
+
+ /*
+ * Retrieve the current status of the region. The query will be executed
+ * asynchronously and the results passed to the given callback, which will be
+ * executed on the database thread; it is the responsibility of the SDK bindings
+ * to re-execute a user-provided callback on the main thread.
+ */
+ void getOfflineRegionStatus(OfflineRegion&, std::function<void (std::exception_ptr,
+ optional<OfflineRegionStatus>)>) const;
+
+ /*
+ * Remove an offline region from the database and perform any resources evictions
+ * necessary as a result.
+ *
+ * Eviction works by removing the least-recently requested resources not also required
+ * by other regions, until the database shrinks below a certain size.
+ *
+ * Note that this method takes ownership of the input, reflecting the fact that once
+ * region deletion is initiated, it is not legal to perform further actions with the
+ * region.
+ *
+ * When the operation is complete or encounters an error, the given callback will be
+ * executed on the database thread; it is the responsibility of the SDK bindings
+ * to re-execute a user-provided callback on the main thread.
+ */
+ void deleteOfflineRegion(OfflineRegion&&, std::function<void (std::exception_ptr)>);
+
+ // For testing only.
+ void put(const Resource&, const Response&);
+ void goOffline();
+
  class Impl;
- const std::unique_ptr<Impl> impl;
+
+private:
+ const std::unique_ptr<util::Thread<Impl>> thread;
+ const std::unique_ptr<FileSource> assetFileSource;
 };
 
 } // namespace mbgl