Merge pull request nicolas-raoul#116 from hssm/col-2.0.14

Catch up on changes to collection.py

src/com/ichi2/libanki/Collection.java

@@ -144,8 +144,11 @@ public Collection(AnkiDb db, String path, boolean server) {
  mStartReps = 0;
  mStartTime = 0;
  mSched = new Sched(this);
- // check for improper shutdown
- cleanup();
+ if (!mConf.optBoolean("newBury", false)) {
+ boolean mod = mDb.getMod();
+ mSched.unburyCards();
+ mDb.setMod(mod);
+ }
  }
 
 
@@ -160,19 +163,20 @@ public String name() {
  * DB-related *************************************************************** ********************************
  */
 
- public boolean load() {
+ public void load() {
  Cursor cursor = null;
  try {
  // Read in deck table columns
  cursor = mDb.getDatabase().rawQuery(
- "SELECT crt, mod, scm, dty, usn, ls, conf, " + "models, decks, dconf, tags FROM col", null);
+ "SELECT crt, mod, scm, dty, usn, ls, " +
+ "conf, models, decks, dconf, tags FROM col", null);
  if (!cursor.moveToFirst()) {
- return false;
+ return;
  }
  mCrt = cursor.getLong(0);
  mMod = cursor.getLong(1);
  mScm = cursor.getLong(2);
- mDty = cursor.getInt(3) == 1;
+ mDty = cursor.getInt(3) == 1; // No longer used
  mUsn = cursor.getInt(4);
  mLs = cursor.getLong(5);
  try {
@@ -183,7 +187,6 @@ public boolean load() {
  mModels.load(cursor.getString(7));
  mDecks.load(cursor.getString(8), cursor.getString(9));
  mTags.load(cursor.getString(10));
- return true;
  } finally {
  if (cursor != null) {
  cursor.close();
@@ -281,12 +284,12 @@ public synchronized void close() {
 
 
  public synchronized void close(boolean save) {
- // if (wait) {
- // Wait for any thread working on the deck to finish.
- // DeckTask.waitToFinish();
- // }
  if (mDb != null) {
- cleanup();
+ if (!mConf.optBoolean("newBury", false)) {
+ boolean mod = mDb.getMod();
+ mSched.unburyCards();
+ mDb.setMod(mod);
+ }
  try {
  SQLiteDatabase db = getDb().getDatabase();
  if (save) {
@@ -335,6 +338,7 @@ public void modSchema(boolean check) {
  }
  }
  mScm = Utils.intNow(1000);
+ setMod();
  }
 
 
@@ -344,28 +348,6 @@ public boolean schemaChanged() {
  }
 
 
- /**
- * Signal there are temp. suspended cards that need cleaning up on close.
- */
- public void setDirty() {
- mDty = true;
- }
- public void setDirty(boolean dty) {
- mDty = dty;
- }
-
-
- /**
- * Unsuspend any temporarily suspended cards.
- */
- public void cleanup() {
- if (mDty) {
- mSched.unburyCards();
- mDty = false;
- }
- }
-
-
  public int usn() {
  if (mServer) {
  return mUsn;
@@ -377,16 +359,21 @@ public int usn() {
 
  /** called before a full upload */
  public void beforeUpload() {
- String[] tables = new String[] { "notes", "cards", "revlog", "graves" };
+ String[] tables = new String[] { "notes", "cards", "revlog" };
  for (String t : tables) {
  mDb.execute("UPDATE " + t + " SET usn=0 WHERE usn=-1");
  }
+ // we can save space by removing the log of deletions
+ mDb.execute("delete from graves");
  mUsn += 1;
  mModels.beforeUpload();
  mTags.beforeUpload();
  mDecks.beforeUpload();
  modSchema();
  mLs = mScm;
+ // ensure db is compacted before upload
+ mDb.execute("vacuum");
+ mDb.execute("analyze");
  close();
  }
 
@@ -929,7 +916,7 @@ public HashMap<String, String> _renderQA(Object[] data, List<String> args) {
  for (String fname : fmap.keySet()) {
  fields.put(fname, flist[fmap.get(fname).first]);
  }
- fields.put("Tags", (String) data[5]);
+ fields.put("Tags", ((String)data[5]).trim());
  try {
  fields.put("Type", (String) model.get("name"));
  fields.put("Deck", mDecks.name((Long) data[3]));
@@ -1191,6 +1178,9 @@ public long undo() {
  // and delete revlog entry
  long last = mDb.queryLongScalar("SELECT id FROM revlog WHERE cid = " + c.getId() + " ORDER BY id DESC LIMIT 1");
  mDb.execute("DELETE FROM revlog WHERE id = " + last);
+ // restore any siblings
+ mDb.execute("update cards set queue=type,mod=?,usn=? where queue=-2 and nid=?",
+ new Object[]{ Utils.intNow(), usn(), c.getNid() });
  // and finally, update daily count
  int n = c.getQueue() == 3 ? 1 : c.getQueue();
  String type = (new String[] { "new", "lrn", "rev" })[n];
@@ -1223,7 +1213,6 @@ public long undo() {
  return 0;
 
  case UNDO_BURY_NOTE:
- setDirty((Boolean)data[1]);
  for (Card cc : (ArrayList<Card>)data[2]) {
  cc.flush(false);
  }
@@ -1302,6 +1291,51 @@ public void markReview(Card card) {
  * DB maintenance *********************************************************** ************************************
  */
 
+
+ /*
+ * Basic integrity check for syncing. True if ok.
+ */
+ public boolean basicCheck() {
+ // cards without notes
+ if (mDb.queryScalar("select 1 from cards where nid not in (select id from notes) limit 1", false) > 0) {
+ return false;
+ }
+ boolean badNotes = mDb.queryScalar(String.format(Locale.US,
+ "select 1 from notes where id not in (select distinct nid from cards) " +
+ "or mid not in %s limit 1", Utils.ids2str(mModels.ids())), false) > 0;
+ // notes without cards or models
+ if (badNotes) {
+ return false;
+ }
+ try {
+ // invalid ords
+ for (JSONObject m : mModels.all()) {
+ // ignore clozes
+ if (m.getInt("type") != Sched.MODEL_STD) {
+ continue;
+ }
+ // Make a list of valid ords for this model
+ JSONArray tmpls = m.getJSONArray("tmpls");
+ int[] ords = new int[tmpls.length()];
+ for (int t = 0; t < tmpls.length(); t++) {
+ ords[t] = tmpls.getJSONObject(t).getInt("ord");
+ }
+
+ boolean badOrd = mDb.queryScalar(String.format(Locale.US,
+ "select 1 from cards where ord not in %s and nid in ( " +
+ "select id from notes where mid = %d) limit 1",
+ Utils.ids2str(ords), m.getLong("id")), false) > 0;
+ if (badOrd) {
+ return false;
+ }
+ }
+ } catch (JSONException e) {
+ throw new RuntimeException(e);
+ }
+ return true;
+ }
+
+
  /** Fix possible problems and rebuild caches. */
  public long fixIntegrity() {
  File file = new File(mPath);
@@ -1321,23 +1355,50 @@ public long fixIntegrity() {
  problems.add("Deleted " + ids.size() + " note(s) with missing note type.");
  _remNotes(Utils.arrayList2array(ids));
  }
- // cards with invalid ordinal
+ // for each model
  for (JSONObject m : mModels.all()) {
- // ignore clozes
- if (m.getInt("type") != Sched.MODEL_STD) {
- continue;
- }
- ArrayList<Integer> ords = new ArrayList<Integer>();
- JSONArray tmpls = m.getJSONArray("tmpls");
- for (int t = 0; t < tmpls.length(); t++) {
- ords.add(tmpls.getJSONObject(t).getInt("ord"));
- }
- ids = mDb.queryColumn(Long.class,
- "SELECT id FROM cards WHERE ord NOT IN " + Utils.ids2str(ords) + " AND nid IN (SELECT id FROM notes WHERE mid = " + m.getLong("id") + ")", 0);
- if (ids.size() > 0) {
- problems.add("Deleted " + ids.size() + " card(s) with missing template.");
- remCards(Utils.arrayList2array(ids));
- }
+ // cards with invalid ordinal
+ if (m.getInt("type") == Sched.MODEL_STD) {
+ ArrayList<Integer> ords = new ArrayList<Integer>();
+ JSONArray tmpls = m.getJSONArray("tmpls");
+ for (int t = 0; t < tmpls.length(); t++) {
+ ords.add(tmpls.getJSONObject(t).getInt("ord"));
+ }
+ ids = mDb.queryColumn(Long.class,
+ "SELECT id FROM cards WHERE ord NOT IN " + Utils.ids2str(ords) + " AND nid IN ( " +
+ "SELECT id FROM notes WHERE mid = " + m.getLong("id") + ")", 0);
+ if (ids.size() > 0) {
+ problems.add("Deleted " + ids.size() + " card(s) with missing template.");
+ remCards(Utils.arrayList2array(ids));
+ }
+ }
+ // notes with invalid field counts
+ ids = new ArrayList<Long>();
+ Cursor cur = null;
+ try {
+ cur = mDb.getDatabase().rawQuery("select id, flds from notes where mid = " + m.getLong("id"), null);
+ while (cur.moveToNext()) {
+ String flds = cur.getString(1);
+ long id = cur.getLong(0);
+ int fldsCount = 0;
+ for (int i = 0; i < flds.length(); i++) {
+ if (flds.charAt(i) == 0x1f) {
+ fldsCount++;
+ }
+ }
+ if (fldsCount + 1 != m.getJSONArray("flds").length()) {
+ ids.add(id);
+ }
+ }
+ if (ids.size() > 0) {
+ problems.add("Deleted " + ids.size() + " note(s) with wrong field count.");
+ _remNotes(Utils.arrayList2array(ids));
+ }
+ } finally {
+ if (cur != null && !cur.isClosed()) {
+ cur.close();
+ }
+ }
  }
  // delete any notes with missing cards
  ids = mDb.queryColumn(Long.class,
@@ -1346,7 +1407,7 @@ public long fixIntegrity() {
  problems.add("Deleted " + ids.size() + " note(s) with missing no cards.");
  _remNotes(Utils.arrayList2array(ids));
  }
-
+ // cards with missing notes
  ids = mDb.queryColumn(Long.class,
  "SELECT id FROM cards WHERE nid NOT IN (SELECT id FROM notes)", 0);
  if (ids.size() != 0) {

src/com/ichi2/libanki/Decks.java

@@ -67,7 +67,7 @@ public class Decks {
  // currently
  // used
  "'initialFactor': 2500, " + "'separate': True, " + "'order': " + Sched.NEW_CARDS_DUE + ", "
- + "'perDay': 20 }, " + "'lapse': {" + "'delays': [10], " + "'mult': 0, " + "'minInt': 1, "
+ + "'perDay': 20, 'bury': True }, " + "'lapse': {" + "'delays': [10], " + "'mult': 0, " + "'minInt': 1, "
  + "'leechFails': 8, "
  + "'leechAction': 0 }, "
  // type 0=suspend, 1=tagonly

src/com/ichi2/libanki/Finder.java

@@ -459,11 +459,11 @@ private String _findCardState(String val) {
  } else if (val.equals("new")) {
  n = 0;
  } else {
- return "queue IN (1, 3)";
+ return "queue IN (1, 3)";
  }
  return "type = " + n;
  } else if (val.equals("suspended")) {
- return "c.queue = -1";
+ return "c.queue in (-1, -2)";
  } else if (val.equals("due")) {
  return "(c.queue in (2,3) and c.due <= " + mCol.getSched().getToday() +
  ") or (c.queue = 1 and c.due <= " + mCol.getSched().getDayCutoff() + ")";

src/com/ichi2/libanki/sync/FullSyncer.java

@@ -112,6 +112,9 @@ public Object[] upload() {
  if (!mCol.getDb().queryString("PRAGMA integrity_check").equalsIgnoreCase("ok")) {
  return new Object[] { "dbError" };
  }
+ if (!mCol.basicCheck()) {
+ return new Object[] { "dbError" };
+ }
  // apply some adjustments, then upload
  mCol.beforeUpload();
  String filePath = mCol.getPath();

src/com/ichi2/libanki/sync/Syncer.java

@@ -82,8 +82,6 @@ public Object[] sync() {
  public Object[] sync(Connection con) {
  // if the deck has any pending changes, flush them first and bump mod time
  mCol.save();
- // do a cleanup (unbury cards) --> anki desktop does it by closing the collection
- mCol.cleanup();
  // step 1: login & metadata
  HttpResponse ret = mServer.meta();
  if (ret == null) {