Txdb tests and updates

CHANGELOG.md

@@ -2,8 +2,12 @@
 
 ## Unreleased
 
-### Wallet API:
+**When upgrading to this version of hsd, you must pass `--wallet-migrate=3` when
+you run it for the first time.**
+
+### Wallet Changes:
 
+- Add migration that recalculates txdb balances to fix any inconsistencies.
 - HTTP Changes:
   - All transaction creating endpoints now accept `hardFee` for specifying the
     exact fee.

lib/wallet/layout.js

@@ -10,97 +10,159 @@ const bdb = require('bdb');
 
 /*
  * Wallet Database Layout:
+ *  WDB State
+ *  ---------
  *  V -> db version
  *  O -> flags
- *  R -> chain sync state
  *  D -> wallet id depth
- *  p[addr-hash] -> wallet ids
- *  P[wid][addr-hash] -> path data
- *  r[wid][index][hash] -> path account index
+ *  M -> migration state
+ *
+ *  Chain Sync
+ *  ----------
+ *  R -> chain sync state
+ *  h[height] -> block hash
+ *
+ *  WID mappings
+ *  --------
+ *  b[height] -> block->wid map
+ *  T[tx-hash] -> tx->wid map
+ *  o[tx-hash][index] -> outpoint->wid map
+ *  p[addr-hash] -> address->wid map
+ *  N[name-hash] -> name->wid map
+ *
+ *  Wallet
+ *  ------
+ *  l[id] -> wid
  *  w[wid] -> wallet
  *  W[wid] -> wallet id
- *  l[id] -> wid
+ *
+ *  Wallet Account
+ *  --------------
  *  a[wid][index] -> account
  *  i[wid][name] -> account index
  *  n[wid][index] -> account name
- *  h[height] -> recent block hash
- *  b[height] -> block->wid map
- *  o[hash][index] -> outpoint->wid map
- *  T[hash] -> tx->wid map
+ *
+ *  Wallet Path
+ *  -----------
+ *  P[wid][addr-hash] -> path data
+ *  r[wid][index][addr-hash] -> dummy (addr by account)
+ *
+ *  TXDB
+ *  ----
  *  t[wid]* -> txdb
- *  N[hash256] -> name map
- *  M -> migration state
  */
 
 exports.wdb = {
+  // WDB State
   V: bdb.key('V'),
   O: bdb.key('O'),
-  R: bdb.key('R'),
   D: bdb.key('D'),
+  M: bdb.key('M'),
+
+  // Chain Sync
+  R: bdb.key('R'),
+  h: bdb.key('h', ['uint32']),
+
+  // WID Mappings
+  b: bdb.key('b', ['uint32']),
+  T: bdb.key('T', ['hash256']),
   p: bdb.key('p', ['hash']),
-  P: bdb.key('P', ['uint32', 'hash']),
-  r: bdb.key('r', ['uint32', 'uint32', 'hash']),
+  o: bdb.key('o', ['hash256', 'uint32']),
+  N: bdb.key('N', ['hash256']),
+
+  // Wallet
+  l: bdb.key('l', ['ascii']),
   w: bdb.key('w', ['uint32']),
   W: bdb.key('W', ['uint32']),
-  l: bdb.key('l', ['ascii']),
+
+  // Wallet Account
   a: bdb.key('a', ['uint32', 'uint32']),
   i: bdb.key('i', ['uint32', 'ascii']),
   n: bdb.key('n', ['uint32', 'uint32']),
-  h: bdb.key('h', ['uint32']),
-  b: bdb.key('b', ['uint32']),
-  o: bdb.key('o', ['hash256', 'uint32']),
-  T: bdb.key('T', ['hash256']),
-  t: bdb.key('t', ['uint32']),
-  N: bdb.key('N', ['hash256']),
-  M: bdb.key('M')
+
+  // Wallet Path
+  P: bdb.key('P', ['uint32', 'hash']),
+  r: bdb.key('r', ['uint32', 'uint32', 'hash']),
+
+  // TXDB
+  t: bdb.key('t', ['uint32'])
 };
 
 /*
  * TXDB Database Layout:
+ *   Balance
+ *   -------
  *   R -> wallet balance
  *   r[account] -> account balance
- *   t[hash] -> extended tx
- *   c[hash][index] -> coin
- *   d[hash][index] -> undo coin
- *   s[hash][index] -> spent by hash
- *   p[hash] -> dummy (pending flag)
- *   m[time][hash] -> dummy (tx by time)
- *   h[height][hash] -> dummy (tx by height)
- *   T[account][hash] -> dummy (tx by account)
- *   P[account][hash] -> dummy (pending tx by account)
- *   M[account][time][hash] -> dummy (tx by time + account)
- *   H[account][height][hash] -> dummy (tx by height + account)
- *   C[account][hash][index] -> dummy (coin by account)
+ *
+ *   Coin
+ *   ----
+ *   c[tx-hash][index] -> coin
+ *   C[account][tx-hash][index] -> dummy (coin by account)
+ *   d[tx-hash][index] -> undo coin
+ *   s[tx-hash][index] -> spent by hash
+ *
+ *   Transaction
+ *   -----------
+ *   t[tx-hash] -> extended tx
+ *   T[account][tx-hash] -> dummy (tx by account)
+ *   m[time][tx-hash] -> dummy (tx by time)
+ *   M[account][time][tx-hash] -> dummy (tx by time + account)
+ *
+ *   Confirmed
+ *   ---------
  *   b[height] -> block record
+ *   h[height][tx-hash] -> dummy (tx by height)
+ *   H[account][height][tx-hash] -> dummy (tx by height + account)
+ *
+ *   Unconfirmed
+ *   -----------
+ *   p[hash] -> dummy (pending tx)
+ *   P[account][tx-hash] -> dummy (pending tx by account)
+ *
+ *   Names
+ *   -----
+ *   A[name-hash] -> name record (name record by name hash)
+ *   U[tx-hash] -> name undo record (name undo record by tx hash)
+ *   i[name-hash][tx-hash][index] -> bid (BlindBid by name + tx + index)
+ *   B[name-hash][tx-hash][index] -> reveal (BidReveal by name + tx + index)
+ *   v[blind-hash] -> blind (Blind Value by blind hash)
+ *   o[name-hash] -> tx hash OPEN only (tx hash by name hash)
  */
 
 exports.txdb = {
   prefix: bdb.key('t', ['uint32']),
+
+  // Balance
   R: bdb.key('R'),
   r: bdb.key('r', ['uint32']),
-  t: bdb.key('t', ['hash256']),
+
+  // Coin
   c: bdb.key('c', ['hash256', 'uint32']),
+  C: bdb.key('C', ['uint32', 'hash256', 'uint32']),
   d: bdb.key('d', ['hash256', 'uint32']),
   s: bdb.key('s', ['hash256', 'uint32']),
-  p: bdb.key('p', ['hash256']),
-  m: bdb.key('m', ['uint32', 'hash256']),
-  h: bdb.key('h', ['uint32', 'hash256']),
+
+  // Transaction
+  t: bdb.key('t', ['hash256']),
   T: bdb.key('T', ['uint32', 'hash256']),
-  P: bdb.key('P', ['uint32', 'hash256']),
+  m: bdb.key('m', ['uint32', 'hash256']),
   M: bdb.key('M', ['uint32', 'uint32', 'hash256']),
-  H: bdb.key('H', ['uint32', 'uint32', 'hash256']),
-  C: bdb.key('C', ['uint32', 'hash256', 'uint32']),
+
+  // Confirmed
   b: bdb.key('b', ['uint32']),
-  // Name records
+  h: bdb.key('h', ['uint32', 'hash256']),
+  H: bdb.key('H', ['uint32', 'uint32', 'hash256']),
+
+  // Unconfirmed
+  p: bdb.key('p', ['hash256']),
+  P: bdb.key('P', ['uint32', 'hash256']),
+
+  // Names
   A: bdb.key('A', ['hash256']),
-  // Name undo records
   U: bdb.key('U', ['hash256']),
-  // Bids
   i: bdb.key('i', ['hash256', 'hash256', 'uint32']),
-  // Reveals
   B: bdb.key('B', ['hash256', 'hash256', 'uint32']),
-  // Blinds
   v: bdb.key('v', ['hash256']),
-  // Opens
   o: bdb.key('o', ['hash256'])
 };

lib/wallet/migrations.js

@@ -110,11 +110,7 @@ class MigrateChangeAddress extends AbstractMigration {
    */
 
   async migrate(b, pending) {
-    const wids = await this.ldb.keys({
-      gte: layout.W.min(),
-      lte: layout.W.max(),
-      parse: key => layout.W.decode(key)[0]
-    });
+    const wids = await this.db.getWallets();
 
     let total = 0;
     for (const wid of wids) {
@@ -249,6 +245,50 @@ class MigrateAccountLookahead extends AbstractMigration {
   }
 }
 
+class MigrateTXDBBalances extends AbstractMigration {
+  /**
+   * Create TXDB Balance migration object.
+   * @param {WalletMigratorOptions} options
+   * @constructor
+   */
+
+  constructor(options) {
+    super(options);
+
+    this.options = options;
+    this.logger = options.logger.context('txdb-balance-migration');
+    this.db = options.db;
+    this.ldb = options.ldb;
+  }
+
+  /**
+   * We always migrate.
+   * @returns {Promise}
+   */
+
+  async check() {
+    return types.MIGRATE;
+  }
+
+  /**
+   * Actual migration
+   * @param {Batch} b
+   * @param {WalletMigrationResult} pending
+   * @returns {Promise}
+   */
+
+  async migrate(b, pending) {
+    await this.db.recalculateBalances();
+  }
+
+  static info() {
+    return {
+      name: 'TXDB balance refresh',
+      description: 'Refresh balances for TXDB after txdb updates'
+    };
+  }
+}
+
 /**
  * Wallet migration results.
  * @alias module:blockchain.WalletMigrationResult
@@ -382,12 +422,14 @@ exports.WalletMigrationResult = WalletMigrationResult;
 exports.migrations = {
   0: MigrateMigrations,
   1: MigrateChangeAddress,
-  2: MigrateAccountLookahead
+  2: MigrateAccountLookahead,
+  3: MigrateTXDBBalances
 };
 
 // Expose migrations
 exports.MigrateChangeAddress = MigrateChangeAddress;
 exports.MigrateMigrations = MigrateMigrations;
 exports.MigrateAccountLookahead = MigrateAccountLookahead;
+exports.MigrateTXDBBalances = MigrateTXDBBalances;
 
 module.exports = exports;