Skip to content

Commit

Permalink
Add full-text search
Browse files Browse the repository at this point in the history
  • Loading branch information
di72nn committed Nov 17, 2019
1 parent 8e0253f commit 5ca1245
Show file tree
Hide file tree
Showing 6 changed files with 227 additions and 3 deletions.
2 changes: 1 addition & 1 deletion app/build.gradle
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ android {
}

greendao {
schemaVersion 103
schemaVersion 104
daoPackage 'fr.gaulupeau.apps.Poche.data.dao'
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,7 @@
import fr.gaulupeau.apps.Poche.data.dao.ArticleDao;
import fr.gaulupeau.apps.Poche.data.dao.ArticleTagsJoinDao;
import fr.gaulupeau.apps.Poche.data.dao.DaoSession;
import fr.gaulupeau.apps.Poche.data.dao.FtsDao;
import fr.gaulupeau.apps.Poche.data.dao.TagDao;
import fr.gaulupeau.apps.Poche.data.dao.entities.Article;
import fr.gaulupeau.apps.Poche.data.dao.entities.ArticleTagsJoin;
Expand Down Expand Up @@ -286,6 +287,7 @@ public static void deleteArticle(Context context, int articleID) {
public static void wipeDB(Settings settings) {
DaoSession daoSession = DbConnection.getSession();

FtsDao.deleteAllArticles(daoSession.getDatabase());
daoSession.getArticleContentDao().deleteAll();
daoSession.getArticleDao().deleteAll();
daoSession.getTagDao().deleteAll();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import fr.gaulupeau.apps.Poche.data.dao.ArticleContentDao;
import fr.gaulupeau.apps.Poche.data.dao.ArticleDao;
import fr.gaulupeau.apps.Poche.data.dao.DaoMaster;
import fr.gaulupeau.apps.Poche.data.dao.FtsDao;
import fr.gaulupeau.apps.Poche.data.dao.QueueItemDao;
import fr.gaulupeau.apps.Poche.data.dao.entities.QueueItem;
import fr.gaulupeau.apps.Poche.events.OfflineQueueChangedEvent;
Expand All @@ -30,12 +31,20 @@ public WallabagDbOpenHelper(Context context, String name, SQLiteDatabase.CursorF
this.context = context;
}

@Override
public void onCreate(Database db) {
Log.d(TAG, "onCreate() creating tables");

super.onCreate(db);
FtsDao.createAll(db, false);
}

@Override
public void onUpgrade(Database db, int oldVersion, int newVersion) {
Log.i(TAG, "Upgrading schema from version " + oldVersion + " to " + newVersion);

boolean migrationDone = false;
if (oldVersion >= 101 && newVersion <= 103) {
if (oldVersion >= 101 && newVersion <= 104) {
try {
if (oldVersion < 102) {
Log.i(TAG, "Migrating to version " + 102);
Expand Down Expand Up @@ -68,6 +77,19 @@ public void onUpgrade(Database db, int oldVersion, int newVersion) {
db.execSQL("update " + ArticleContentDao.TABLENAME + " set CONTENT = null;");
}

if (oldVersion < 104) {
Log.i(TAG, "Migrating to version " + 104);
FtsDao.createAll(db, false);

db.execSQL("insert into " + FtsDao.TABLE_NAME +
"(" + FtsDao.COLUMN_ID +
", " + FtsDao.COLUMN_TITLE +
", " + FtsDao.COLUMN_CONTENT + ")" +
" select rowid, " + ArticleDao.Properties.Title.columnName +
", " + ArticleContentDao.Properties.Content.columnName +
" from " + FtsDao.VIEW_FOR_FTS_NAME);
}

migrationDone = true;
} catch (Exception e) {
Log.e(TAG, "Migration error", e);
Expand Down Expand Up @@ -101,6 +123,7 @@ private void genericMigration(Database db, int oldVersion, int newVersion) {
}
}

FtsDao.dropAll(db, true);
DaoMaster.dropAllTables(db, true);
onCreate(db);

Expand Down
193 changes: 193 additions & 0 deletions app/src/main/java/fr/gaulupeau/apps/Poche/data/dao/FtsDao.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,193 @@
package fr.gaulupeau.apps.Poche.data.dao;

import android.os.Build;

import org.greenrobot.greendao.database.Database;

public class FtsDao {

public static final String TABLE_NAME = "article_fts";

public static final String COLUMN_ID = "docid";
public static final String COLUMN_TITLE = "title";
public static final String COLUMN_CONTENT = "content";

public static final String VIEW_FOR_FTS_NAME = "article_content_for_fts";

private static final String[] TRIGGER_NAMES = new String[]{
"article_added_insert_fts_tr",
"article_added_update_fts_tr",
"article_content_added_insert_fts_tr",
"article_content_added_update_fts_tr",
"article_before_updated_tr",
"article_after_updated_tr",
"article_content_before_updated_tr",
"article_content_after_updated_tr",
"article_before_delete_tr",
"article_after_delete_tr",
"article_content_before_delete_tr",
"article_content_after_delete_tr"
};

public static String getQueryString() {
return "select " + COLUMN_ID + " from " + TABLE_NAME + " where " + TABLE_NAME + " match ";
}

public static void createAll(Database db, boolean ifNotExists) {
createViewForFts(db, ifNotExists);
createTable(db, ifNotExists);
createTriggers(db, ifNotExists);
}

public static void dropAll(Database db, boolean ifExists) {
dropTriggers(db, ifExists);
dropTable(db, ifExists);
dropViewForFts(db, ifExists);
}

public static void deleteAllArticles(Database db) {
dropTable(db, true);
createTable(db, true);
}

private static void createTable(Database db, boolean ifNotExists) {
String options = ", content=\"" + VIEW_FOR_FTS_NAME + "\"";
if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.LOLLIPOP) {
options += ", tokenize=unicode61";
}

db.execSQL("create virtual table " + getIfNotExistsConstraint(ifNotExists) +
TABLE_NAME + " using fts4(" +
COLUMN_TITLE + ", " +
COLUMN_CONTENT +
options + ");");
}

private static void dropTable(Database db, boolean ifExists) {
db.execSQL("drop table " + getIfExistsConstraint(ifExists) + TABLE_NAME + ";");
}

private static void createViewForFts(Database db, boolean ifNotExists) {
final String id = ArticleDao.Properties.Id.columnName;

String viewQuery = "select a." + id + " rowid" +
", a." + ArticleDao.Properties.Title.columnName +
", c." + ArticleContentDao.Properties.Content.columnName +
" from " + ArticleDao.TABLENAME + " a join " + ArticleContentDao.TABLENAME + " c" +
" using (" + id + ")";

String createViewQuery = "create view " + getIfNotExistsConstraint(ifNotExists) +
VIEW_FOR_FTS_NAME + " as " + viewQuery;

db.execSQL(createViewQuery);
}

private static void dropViewForFts(Database db, boolean ifExists) {
db.execSQL("drop view " + getIfExistsConstraint(ifExists) + VIEW_FOR_FTS_NAME);
}

private static void createTriggers(Database db, boolean ifNotExists) {
final String sqlId = "rowid";
final String daoId = ArticleDao.Properties.Id.columnName;
final String fts = TABLE_NAME;
final String ftsId = COLUMN_ID;
final String ftsTitle = COLUMN_TITLE;
final String ftsContent = COLUMN_CONTENT;
final String article = ArticleDao.TABLENAME;
final String articleTitle = ArticleDao.Properties.Title.columnName;
final String articleContent = ArticleContentDao.TABLENAME;
final String articleContentContent = ArticleContentDao.Properties.Content.columnName;

String[] triggers = {
"after insert on " + article +
" when not exists (select " + ftsId + " from " + fts + " where " + ftsId + " = new." + sqlId + ")" +
" begin" +
" insert into " + fts + "(" + ftsId + ", " + ftsTitle + ", " + ftsContent + ")" +
" values(new." + sqlId + ", new." + articleTitle + "," +
" coalesce((select " + articleContentContent + " from " + articleContent +
" where " + daoId + " = new." + sqlId + "), null)" +
" );" +
" end",
"after insert on " + article +
" when exists (select " + ftsId + " from " + fts + " where " + ftsId + " = new." + sqlId + ")" +
" begin" +
" update " + fts + " set " + ftsTitle + " = new." + articleTitle +
" where " + ftsId + " = new." + sqlId + ";" +
" end",
"after insert on " + articleContent +
" when not exists (select " + ftsId + " from " + fts + " where " + ftsId + " = new." + sqlId + ")" +
" begin" +
" insert into " + fts + "(" + ftsId + ", " + ftsTitle + ", " + ftsContent + ")" +
" values(new." + sqlId + ", coalesce((select " + articleTitle + " from " + article +
" where " + daoId + " = new." + sqlId + "), null), new." + articleContentContent + ");" +
" end",
"after insert on " + articleContent +
" when exists (select " + ftsId + " from " + fts + " where " + ftsId + " = new." + sqlId + ")" +
" begin" +
" update " + fts + " set " + ftsTitle + " = coalesce((select " + articleTitle + " from " + article +
" where " + daoId + " = new." + sqlId + "), null), " + ftsContent + " = new." + articleContentContent +
" where " + ftsId + " = new." + sqlId + ";" +
" end",
"before update of " + articleTitle + " on " + article +
" begin" +
" update " + fts + " set " + ftsTitle + " = null where " + ftsId + " = old." + sqlId + ";" +
" end",
"after update of " + articleTitle + " on " + article +
" begin" +
" update " + fts + " set " + ftsTitle + " = new." + articleTitle +
" where " + ftsId + " = new." + sqlId + ";" +
" end",
"before update of " + articleContentContent + " on " + articleContent +
" begin" +
" update " + fts + " set " + ftsContent + " = null" +
" where " + ftsId + " = old." + sqlId + ";" +
" end",
"after update of " + articleContentContent + " on " + articleContent +
" begin" +
" update " + fts + " set " + ftsContent + " = new." + articleContentContent +
" where " + ftsId + " = new." + sqlId + ";" +
" end",
"before delete on " + article +
" when exists (select " + daoId + " from " + articleContent + " where " + daoId + " = old." + sqlId + ")" +
" begin" +
" update " + fts + " set " + ftsTitle + " = null where " + ftsId + " = old." + sqlId + ";" +
" end",
"before delete on " + article +
" when not exists (select " + daoId + " from " + articleContent + " where " + daoId + " = old." + sqlId + ")" +
" begin" +
" delete from " + fts + " where " + ftsId + " = old." + sqlId + ";" +
" end",
"before delete on " + articleContent +
" when exists (select " + daoId + " from " + article + " where " + daoId + " = old." + sqlId + ")" +
" begin" +
" update " + fts + " set " + ftsContent + " = null where " + ftsId + " = old." + sqlId + ";" +
" end",
"before delete on " + articleContent +
" when not exists (select " + daoId + " from " + article + " where " + daoId + " = old." + sqlId + ")" +
" begin" +
" delete from " + fts + " where " + ftsId + " = old." + sqlId + ";" +
" end"
};

for (int i = 0; i < triggers.length; i++) {
String triggerBody = triggers[i];
db.execSQL("create trigger " + getIfNotExistsConstraint(ifNotExists) +
TRIGGER_NAMES[i] + " " + triggerBody);
}
}

private static void dropTriggers(Database db, boolean ifExists) {
for (String trigger : TRIGGER_NAMES) {
db.execSQL("drop trigger " + getIfExistsConstraint(ifExists) + trigger);
}
}

private static String getIfNotExistsConstraint(boolean ifNotExists) {
return ifNotExists ? "if not exists ": "";
}

private static String getIfExistsConstraint(boolean ifExists) {
return ifExists ? "if exists " : "";
}

}
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import fr.gaulupeau.apps.Poche.data.dao.ArticleDao;
import fr.gaulupeau.apps.Poche.data.dao.ArticleTagsJoinDao;
import fr.gaulupeau.apps.Poche.data.dao.DaoSession;
import fr.gaulupeau.apps.Poche.data.dao.FtsDao;
import fr.gaulupeau.apps.Poche.data.dao.TagDao;
import fr.gaulupeau.apps.Poche.data.dao.entities.Article;
import fr.gaulupeau.apps.Poche.data.dao.entities.ArticleContent;
Expand Down Expand Up @@ -71,6 +72,7 @@ public ArticlesChangedEvent update(UpdateType updateType, long latestUpdatedItem
try {
if(clean) {
Log.d(TAG, "update() deleting old DB entries");
FtsDao.deleteAllArticles(daoSession.getDatabase());
daoSession.getArticleTagsJoinDao().deleteAll();
daoSession.getArticleContentDao().deleteAll();
daoSession.getArticleDao().deleteAll();
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
import android.app.Activity;
import android.content.Context;
import android.content.Intent;
import android.database.DatabaseUtils;
import android.os.Bundle;
import androidx.recyclerview.widget.DiffUtil;
import androidx.recyclerview.widget.RecyclerView;
Expand All @@ -15,6 +16,7 @@

import org.greenrobot.greendao.query.LazyList;
import org.greenrobot.greendao.query.QueryBuilder;
import org.greenrobot.greendao.query.WhereCondition;

import java.util.ArrayList;
import java.util.List;
Expand All @@ -27,6 +29,7 @@
import fr.gaulupeau.apps.Poche.data.dao.ArticleDao;
import fr.gaulupeau.apps.Poche.data.dao.ArticleTagsJoinDao;
import fr.gaulupeau.apps.Poche.data.dao.DaoSession;
import fr.gaulupeau.apps.Poche.data.dao.FtsDao;
import fr.gaulupeau.apps.Poche.data.dao.TagDao;
import fr.gaulupeau.apps.Poche.data.dao.entities.Article;
import fr.gaulupeau.apps.Poche.data.dao.entities.ArticleTagsJoin;
Expand Down Expand Up @@ -212,7 +215,8 @@ private QueryBuilder<Article> getQueryBuilder() {
}

if(!TextUtils.isEmpty(searchQuery)) {
qb.where(ArticleDao.Properties.Title.like("%" + searchQuery + "%"));
qb.where(new WhereCondition.StringCondition(ArticleDao.Properties.Id.columnName + " IN (" +
FtsDao.getQueryString() + DatabaseUtils.sqlEscapeString(searchQuery) + ")"));
}

switch(sortOrder) {
Expand Down

0 comments on commit 5ca1245

Please sign in to comment.