From 5130ca8843a342a1b708bb63263fd44c6c908120 Mon Sep 17 00:00:00 2001 From: John Firebaugh Date: Tue, 15 Mar 2016 15:23:25 -0700 Subject: [PATCH] [core] Implement a vacuum strategy for the offline database Enable `PRAGMA auto_vacuum = INCREMENTAL`, and perform a `PRAGMA incremental_vacuum` when deleting an offline region. --- .gitignore | 1 + .../default/mbgl/storage/offline_database.cpp | 39 ++++++++++-------- .../default/mbgl/storage/offline_database.hpp | 2 + test/fixtures/offline/v2.db | Bin 0 -> 43008 bytes test/storage/offline_database.cpp | 28 +++++++++++++ 5 files changed, 53 insertions(+), 17 deletions(-) create mode 100644 test/fixtures/offline/v2.db diff --git a/.gitignore b/.gitignore index 1b6ed5386b2..68e242a31b9 100644 --- a/.gitignore +++ b/.gitignore @@ -26,6 +26,7 @@ offline.db /test/fixtures/api/1.png /test/fixtures/api/2.png /test/fixtures/database/*.db +/test/fixtures/offline/v3.db /test/fixtures/**/actual.png /test/fixtures/**/diff.png /test/output diff --git a/platform/default/mbgl/storage/offline_database.cpp b/platform/default/mbgl/storage/offline_database.cpp index a42591d60e1..e1cd6936620 100644 --- a/platform/default/mbgl/storage/offline_database.cpp +++ b/platform/default/mbgl/storage/offline_database.cpp @@ -14,9 +14,6 @@ namespace mbgl { using namespace mapbox::sqlite; -// If you change the schema you must write a migration from the previous version. -static const uint32_t schemaVersion = 2; - OfflineDatabase::Statement::~Statement() { stmt.reset(); stmt.clearBindings(); @@ -50,15 +47,12 @@ void OfflineDatabase::ensureSchema() { try { connect(ReadWrite); - { - auto userVersionStmt = db->prepare("PRAGMA user_version"); - userVersionStmt.run(); - switch (userVersionStmt.get(0)) { - case 0: break; // cache-only database; ok to delete - case 1: break; // cache-only database; ok to delete - case 2: return; - default: throw std::runtime_error("unknown schema version"); - } + switch (userVersion()) { + case 0: break; // cache-only database; ok to delete + case 1: break; // cache-only database; ok to delete + case 2: migrateToVersion3(); // fall through + case 3: return; + default: throw std::runtime_error("unknown schema version"); } removeExisting(); @@ -76,8 +70,17 @@ void OfflineDatabase::ensureSchema() { #include "offline_schema.cpp.include" connect(ReadWrite | Create); + + // If you change the schema you must write a migration from the previous version. + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); db->exec(schema); - db->exec("PRAGMA user_version = " + util::toString(schemaVersion)); + db->exec("PRAGMA user_version = 3"); +} + +int OfflineDatabase::userVersion() { + auto stmt = db->prepare("PRAGMA user_version"); + stmt.run(); + return stmt.get(0); } void OfflineDatabase::removeExisting() { @@ -92,6 +95,11 @@ void OfflineDatabase::removeExisting() { } } +void OfflineDatabase::migrateToVersion3() { + db->exec("PRAGMA auto_vacuum = INCREMENTAL"); + db->exec("VACUUM"); +} + OfflineDatabase::Statement OfflineDatabase::getStatement(const char * sql) { auto it = statements.find(sql); @@ -458,6 +466,7 @@ void OfflineDatabase::deleteRegion(OfflineRegion&& region) { stmt->run(); evict(0); + db->exec("PRAGMA incremental_vacuum"); // Ensure that the cached offlineTileCount value is recalculated. offlineMapboxTileCount = {}; @@ -614,10 +623,6 @@ bool OfflineDatabase::evict(uint64_t neededFreeSize) { uint64_t pageSize = getPragma("PRAGMA page_size"); uint64_t pageCount = getPragma("PRAGMA page_count"); - if (pageSize * pageCount > maximumCacheSize) { - Log::Warning(mbgl::Event::Database, "Current size is larger than the maximum size. Database won't get truncated."); - } - auto usedSize = [&] { return pageSize * (pageCount - getPragma("PRAGMA freelist_count")); }; diff --git a/platform/default/mbgl/storage/offline_database.hpp b/platform/default/mbgl/storage/offline_database.hpp index eb18cc18d2e..1e77d560d47 100644 --- a/platform/default/mbgl/storage/offline_database.hpp +++ b/platform/default/mbgl/storage/offline_database.hpp @@ -58,8 +58,10 @@ class OfflineDatabase : private util::noncopyable { private: void connect(int flags); + int userVersion(); void ensureSchema(); void removeExisting(); + void migrateToVersion3(); class Statement { public: diff --git a/test/fixtures/offline/v2.db b/test/fixtures/offline/v2.db new file mode 100644 index 0000000000000000000000000000000000000000..8fadec4abe5e9216a1b1c43f49b246d22295c7bb GIT binary patch literal 43008 zcmeHQ3tSY{8lRan`+l$wa!o3x-97r(}sJU4t zm9M;-k4#PTbty_Q-`uW`yz0-&G#_YJAxkk8AL;F$8JHE7g>LSzU&)yt^PO+bcjkOE z|Cx_-=6v6tlRhNb>ab8_3QG#i4k`j8F&xMGQxt|_QE=}D_h7i=a2LRxggd*@c?5gl zA&tU9>+3mmLC{DSk%M1ABcTG<-Q_1MNuElj!e8Y$%%k%y4r{)}&JfOkG(%j5fy#)B zPc~3Y78N9t%_`|73T6I#tJYc+R#tGSl4B5R4o z&dTUY3kq|rW2_eUozQs}hj}b>dGtY!*noPzi>Cahb^( zR2W^*oSofzF?Ao4LTEgGGbO zyW>2Szdt@DgsBtWdSPh^tjx_2ch%$0{@t&~{cpuMc%$qCbAwHz_~(0=@M-A@uFBx= z>1ny~G3i+rTAzS6XFl1&HiKx}#E;_n*nhDKOwC%TOCu0rQFQ4FF`vwJIEs4p=wT@{ zLxZw(hp%Rj*k>CrjMtsy)NTsN4%pSjBs=$g*3BjI%9vGYowJ_p{oM@xgta{Dn$@o4 zO*3DqNiOI*a^Q|HZ(Xp3N4~DuvLdTIZ2gX$oZa8+Uxip|{Z4qA7_J~VEw5=`Ni`sl$ z8oX_7`UlqrE$s;fB##S(kopn?2I>Hp_>nk7d`7%SOd;}!!9+Ci82C3qfGV&MOabFS z8i)lHERME^3V}yK4BpCr_3$uX4-$Q>(vzg8!jmLM?nx3Y^CXG#@g#|qdXhv)JW0aE zo+NsaCrOyFGl*&pomhbfK~KI1K@87>Ae!qz5XJEzh$KA-B7g@$IP7vq30ir|dI@}& z3Gg2bT!92_2myq^!-s%`0RA}J_qY2Bao{H?v5d7A+U&Ib-}P1h@ZCMSBZL4#zyktw z{~y`E2XIg}Lf|1sfNuYB@Bs$@(1s8|2>cll$ig|8GYXriwmZh>TTIY{qwb|v>(rwQ zOKmxJb+3`3v0>fAbfNm5bi(iq`9tI%q?X|L-0*^TXr+vjb>>eA)^n_#TXdUBtjw zpdSHMh!7#kZiEm;H2vd!$ zTc6De9wp+c^#1rnrat$Gh5HnDj`Xg2-P!JfblvV;D#%^GbGU$~3JAc({Y$KqoMW!D62Y@azP>a}Ts1_|B2US;XV1LPPP% z%9fb8(Q7X2J{Goyja%wXeAr6c!-y?sZv!W`46F-c)sN3rrH8hI6?=-^jh8z|1Z{#b zAE%PbQ~CMf&Pgq>gMkXnljpU9l>PSwDdbqjSZ!7qqhh;1Rw2h?&n~eRF+w_IO)E)m68%jC8QWqF)4ynL<%7lk^)EtBp*^f$%B+f zav|lC97s7N2`MCy0+N7~05}jd`TxNSwEbTJKY_F0Bsc=Ce-GFRHiJ*VT4?)=!MosX z@H&hEm;?)=4IzLKct8ZWqyWc#eVIh5WDboMmJz>89|EJNZ;lb1X!#la%mOhEUw8X{pNX zcsH@!9hGd3UDldfx3uM%E!4Z^R1nM3Ap(E3xRw|pSM|GsKT!Ywo>v_mA_O`Z0`qGJ z*)KQ~GlI^7)#A4WZ*Jw^5Z(GA;@5eNgK~?qe;*=hD6_#uwTBjOJfo$W(Cj6AvY44SF28ek)fBqS#oJ-*ydv2uY$=di}#t& zZLIx%LH`9WFD-lQ$+*?M5BX=D8Q|CyxHs=u*NU6^ly~HBg`a=2)SmKMM7DUSSbOYL z!sj1Uule$+)bcNSxPn4c97`L@h8Z87-?%reN^)mIPbZ+y?0kgL^r-RzP$C-gm2TXILdp>Oz#_XD{^5-^u-J+;KL)^ z*6di8Wed4IdRAh!cJb-A`l(WMSL5d9Y_bhMajJUz{(5U!>Dq$QMFJbY`x)1|b1}>P zEMu?x1Rj@Vf4Xmd)Y6ygd;YLmGw#iWK1VkUGuK8->lOxdijLd9{M?RPeJU{JjE#B2 zEvDSwxu?epLoX{XjIaOK@jgfOoo$uHA=k!L{>+b<=l{~EDdJNThm5Rq0r~9eE}|Y| zB+A?1xL^9O!?))yyAd69u?_>8@X+M0XB(bFaYT-7b$!S@?Ro9?Gj*HmLq6Iw z`s)LgM^_%cu_(TP}a%=9ra7`p&o-wqNNtZPmMpToS29yVx7jsqT&(PI`x0$cH)K?osRSB zqG~`%XX`YbZ+x!Yzh#<)9>DX$|0-nu$o!G{cUbesko_b3NA{2GA9epx_aAltJ6!i4 zwg2ezAASCJ+|Pex|H%H4{UiHF!~bab9}WL^yy1V;{-gFEwg0I7N9{jq|55wj0o#8R z|ARUJr~}Y3I{-SZTL0^=waw7_U%zeLHnjeiw^qRWL)QOtueaCj zH`u>cA;MzXSpQG&ttI*3zoM=#t-Kv=-ZDw3Ex@IfwWG~jOY%W&T9?+R9c|uPk`HRr zxU|xCw0UbuKB!IY(n{LV=B*|9ptdJnT5&ttytO1B)JD0qqIR@-VM#v5B%wBcmsZ%0 zwy+0Z0Fc%87}SWN{y*yfqyB%#?EfSCN5B6;zyImjzyCq)KWhI``;XdxrvL6Z(|`XD6XiFz literal 0 HcmV?d00001 diff --git a/test/storage/offline_database.cpp b/test/storage/offline_database.cpp index 23269a98edb..11d56c237be 100644 --- a/test/storage/offline_database.cpp +++ b/test/storage/offline_database.cpp @@ -7,6 +7,7 @@ #include #include +#include #include #include #include @@ -660,3 +661,30 @@ TEST(OfflineDatabase, OfflineMapboxTileCount) { db.deleteRegion(std::move(region1)); EXPECT_EQ(0, db.getOfflineMapboxTileCount()); } + +static int databasePageCount(const std::string& path) { + mapbox::sqlite::Database db(path, mapbox::sqlite::ReadOnly); + mapbox::sqlite::Statement stmt = db.prepare("pragma page_count"); + stmt.run(); + return stmt.get(0); +} + +TEST(OfflineDatabase, MigrateFromV2Schema) { + using namespace mbgl; + + // v2.db is a v2 database containing a single offline region with a small number of resources. + + deleteFile("test/fixtures/offline/v3.db"); + writeFile("test/fixtures/offline/v3.db", util::read_file("test/fixtures/offline/v2.db")); + + { + OfflineDatabase db("test/fixtures/offline/v3.db", 0); + auto regions = db.listRegions(); + for (auto& region : regions) { + db.deleteRegion(std::move(region)); + } + } + + EXPECT_LT(databasePageCount("test/fixtures/offline/v3.db"), + databasePageCount("test/fixtures/offline/v2.db")); +}