Merge PR #888 from 'nodech/wallet-pagination'

CHANGELOG.md

@@ -32,15 +32,19 @@ process and allows parallel rescans.
       - `nextCompaction` - when will the next compaction trigger after restart.
       - `lastCompaction` - when was the last compaction run.
   - Introduce `scan interactive` hook (start, filter, fullLock)
+  - Add `get median time` hook to get median time past for a blockhash.
+  - Add `get entries` hook to get entries. Similar to `get hashes`, but returns
+    encoded entries.
 
-### Node HTTP Client:
+### hs-client Node
   - Introduce `scanInteractive` method that starts interactive rescan.
     - expects ws hook for `block rescan interactive` params `rawEntry, rawTXs`
       that returns scanAction object.
     - expects ws hook for `block rescan interactive abort` param `message`.
   - Add `getMempoolRejectionFilter` and `checkMempoolRejectionFilter` NodeClient
   aliases.
   - Add `getFee`, an HTTP alternative to estimateFee socket call.
+  - Adds `getEntries(start, end)` that returns encoded chain entries.
 
 ### Wallet Changes
 - Add migration that recalculates txdb balances to fix any inconsistencies.
@@ -54,6 +58,8 @@ process and allows parallel rescans.
 - Add `--wallet-preload-all` (or `--preload-all` for standalone wallet node)
   that will open all wallets before starting other services (e.g. HTTP).
   By default this is set to `false`.
+- Add `--wallet-max-history-txs` (or `--max-history-txs` for standalone wallet
+  node) that will be the hard limit of confirmed and unconfirmed histories.
 
 #### Wallet API
 
@@ -64,8 +70,26 @@ process and allows parallel rescans.
   - `open()` no longer calls scan, instead only rollbacks and waits for
     sync to do the rescan.
   - emits events for: `open`, `close`, `connect`, `disconnect`, `sync done`.
-
-#### Wallet HTTP
+  - Wallet now has additional methods for quering history:
+    - `listUnconfirmed(acc, { limit, reverse })` - Get first or last `limit`
+      unconfirmed transactions.
+    - `listUnconfirmedAfter(acc, { hash, limit, reverse })` - Get first or last `limit`
+      unconfirmed transactions after/before tx with hash: `hash`.
+    - `listUnconfirmedFrom(acc, { hash, limit, reverse })` - Get first or last `limit`
+      unconfirmed transactions after/before tx with hash `hash`, inclusive.
+    - `listUnconfirmedByTime(acc, { time, limit, reverse })` - Get first or last
+      `limit` unconfirmed transactions after/before `time`, inclusive.
+    - `listHistory(acc, { limit, reverse })` - Get first or last `limit`
+      unconfirmed/confirmed transactions.
+    - `listHistoryAfter(acc, { hash, limit, reverse })` - Get first or last `limit`
+      unconfirmed/confirmed transactions after/before tx with hash `hash`.
+    - `listHistoryFrom(acc, { hash, limit, reverse })` - Get first or last `limit`
+      confirmed/unconfirmed transactions after/before tx with hash `hash`, inclusive.
+    - `listUnconfirmedByTime(acc, { time, limit, reverse })` - Get first or last
+      `limit` confirmed/unconfirmed transactions after/before `time`, inclusive.
+    - NOTE: Default is ascending order, from the oldest.
+
+##### Wallet HTTP API
   - All transaction creating endpoints now accept `hardFee` for specifying the
     exact fee.
   - All transaction sending endpoints now fundlock/queue tx creation. (no more
@@ -84,6 +108,78 @@ process and allows parallel rescans.
     - `GET /wallet/:id/auction/:name` (`getAuctionByName`)
     - `GET /wallet/:id/reveal` (`getReveals`)
     - `GET /wallet/:id/reveal/:name` (`getRevealsByName`)
+  - `GET /wallet/:id/tx/history` - The params are now `time`, `after`,
+  `limit`, and `reverse`.
+  - `GET /wallet/:id/tx/unconfirmed` - The params are are same as above.
+
+These endpoints have been deprecated:
+  - `GET /wallet/:id/tx/range` - Instead use the `time` param for the history and
+    unconfirmed endpoints.
+  - `GET /wallet/:id/tx/last` - Instead use `reverse` param for the history and
+    unconfirmed endpoints.
+
+##### Examples
+
+```
+GET /wallet/:id/tx/history?after=<txid>&limit=50&reverse=false
+GET /wallet/:id/tx/history?after=<txid>&limit=50&reverse=true
+```
+By using `after=<txid>` we can anchor pages so that results will not shift
+when new blocks and transactions arrive. With `reverse=true` we can change
+the order the transactions are returned as _latest to genesis_. The
+`limit=<number>` specifies the maximum number of transactions to return
+in the result.
+
+```
+GET /wallet/:id/tx/history?time=<median-time-past>&limit=50&reverse=false
+GET /wallet/:id/tx/history?time=<median-time-past>&limit=50&reverse=true
+```
+The param `time` is in epoch seconds and indexed based on median-time-past
+(MTP) and `date` is ISO 8601 format. Because multiple transactions can share
+the same time, this can function as an initial query, and then switch to the
+above `after` format for the following pages.
+
+```
+GET /wallet/:id/tx/unconfirmed?after=<txid>&limit=50&reverse=false
+GET /wallet/:id/tx/unconfirmed?after=<txid>&limit=50&reverse=true
+GET /wallet/:id/tx/unconfirmed?time=<time-received>&limit=50&reverse=false
+```
+The same will apply to unconfirmed transactions. The `time` is in epoch
+seconds and indexed based on when the transaction was added to the wallet.
+
+##### Wallet RPC
+
+The following new methods have been added:
+  - `listhistory` - List history with a limit and in reverse order.
+  - `listhistoryafter` - List history after a txid _(subsequent pages)_.
+  - `listhistorybytime` - List history by giving a timestamp in epoch seconds
+    _(block median time past)_.
+  - `listunconfirmed` - List unconfirmed transactions with a limit and in
+    reverse order.
+  - `listunconfirmedafter` - List unconfirmed transactions after a txid
+    _(subsequent pages)_.
+  - `listunconfirmedbytime` - List unconfirmed transactions by time they
+    where added.
+
+The following methods have been deprecated:
+
+- `listtransactions` - Use `listhistory` and the related methods and the
+  `after` argument for results that do not shift when new blocks arrive.
+
+##### Wallet CLI (hsw-cli)
+  - `history` now accepts new args on top of `--account`: `--reverse`,
+    `--limit`, `--after`, `--after`.
+  - `pending` now accepts new args, same as above.
+
+
+### Client changes
+#### Wallet HTTP Client
+
+  - `getHistory` and `Wallet.getHistory` no longer accept `account`,
+    instead accepts object with properties: `account`, `time`, `after`,
+    `limit`, and `reverse`.
+  - `getPending` and `Wallet.getPending` have the same changes as
+    `getHistory` above.
 
 ## v6.0.0
 

bin/hsw-cli

@@ -271,15 +271,29 @@ class CLI {
   }
 
   async getWalletHistory() {
-    const account = this.config.str('account');
-    const txs = await this.wallet.getHistory(account);
+    const options = {
+      account: this.config.str('account'),
+      limit: this.config.uint('limit'),
+      reverse: this.config.bool('reverse'),
+      after: this.config.str('after'),
+      time: this.config.uint('time')
+    };
+
+    const txs = await this.wallet.getHistory(options);
 
     this.log(txs);
   }
 
   async getWalletPending() {
-    const account = this.config.str('account');
-    const txs = await this.wallet.getPending(account);
+    const options = {
+      account: this.config.str('account'),
+      limit: this.config.uint('limit'),
+      reverse: this.config.bool('reverse'),
+      after: this.config.str('after'),
+      time: this.config.uint('time')
+    };
+
+    const txs = await this.wallet.getPending(options);
 
     this.log(txs);
   }

lib/blockchain/chain.js

@@ -371,7 +371,7 @@ class Chain extends AsyncEmitter {
   /**
    * Calculate median time past.
    * @param {ChainEntry} prev
-   * @returns {Promise} - Returns Number.
+   * @returns {Promise<Number>}
    */
 
   async getMedianTime(prev) {
@@ -2905,8 +2905,8 @@ class Chain extends AsyncEmitter {
 
   /**
    * Find the corresponding block entry by hash or height.
-   * @param {Hash|Number} hash/height
-   * @returns {Promise} - Returns {@link ChainEntry}.
+   * @param {Hash|Number} hash
+   * @returns {Promise<ChainEntry?>}
    */
 
   getEntry(hash) {
@@ -2997,13 +2997,24 @@ class Chain extends AsyncEmitter {
    * Get range of hashes.
    * @param {Number} [start=-1]
    * @param {Number} [end=-1]
-   * @returns {Promise}
+   * @returns {Promise<Hash[]>}
    */
 
   getHashes(start = -1, end = -1) {
     return this.db.getHashes(start, end);
   }
 
+  /**
+   * Get range of entries.
+   * @param {Number} [start=-1]
+   * @param {Number} [end=-1]
+   * @returns {Promise<ChainEntry[]>}
+   */
+
+  getEntries(start = -1, end = -1) {
+    return this.db.getEntries(start, end);
+  }
+
   /**
    * Get a coin (unspents only).
    * @private

lib/blockchain/chaindb.js

@@ -1116,7 +1116,7 @@ class ChainDB {
    * Get hash range.
    * @param {Number} [start=-1]
    * @param {Number} [end=-1]
-   * @returns {Promise}
+   * @returns {Promise<Hash[]>}
    */
 
   async getHashes(start = -1, end = -1) {
@@ -1136,16 +1136,27 @@ class ChainDB {
   }
 
   /**
-   * Get all entries.
-   * @returns {Promise} - Returns {@link ChainEntry}[].
+   * Get entries range.
+   * @param {Number} [start=-1]
+   * @param {Number} [end=-1]
+   * @returns {Promise<ChainEntry[]>}
    */
 
-  async getEntries() {
-    return this.db.values({
-      gte: layout.e.min(),
-      lte: layout.e.max(),
-      parse: data => ChainEntry.decode(data)
-    });
+  async getEntries(start = -1, end = -1) {
+    if (start === -1)
+      start = 0;
+
+    if (end === -1)
+      end >>>= 0;
+
+    assert((start >>> 0) === start);
+    assert((end >>> 0) === end);
+
+    const hashes = await this.getHashes(start, end);
+
+    return Promise.all(hashes.map((hash) => {
+      return this.getEntryByHash(hash);
+    }));
   }
 
   /**

lib/blockchain/migrations.js

@@ -351,6 +351,8 @@ class MigrateBlockStore extends AbstractMigration {
     this.ldb = options.ldb;
     this.blocks = options.db.blocks;
     this.layout = MigrateBlockStore.layout();
+
+    this.batchWriteSize = 10000;
   }
 
   /**
@@ -407,7 +409,7 @@ class MigrateBlockStore extends AbstractMigration {
       await this.blocks.writeBlock(hash, value);
       parent.del(key);
 
-      if (++total % 10000 === 0) {
+      if (++total % this.batchWriteSize === 0) {
         await parent.write();
         this.logger.debug('Migrated up %d blocks.', total);
         parent = this.ldb.batch();
@@ -441,7 +443,7 @@ class MigrateBlockStore extends AbstractMigration {
       await this.blocks.writeUndo(hash, value);
       parent.del(key);
 
-      if (++total % 10000 === 0) {
+      if (++total % this.batchWriteSize === 0) {
         await parent.write();
         this.logger.debug('Migrated up %d undo blocks.', total);
         parent = this.ldb.batch();

lib/client/node.js

@@ -255,25 +255,36 @@ class NodeClient extends Client {
 
   /**
    * Get chain entry.
-   * @param {Hash} hash
-   * @returns {Promise}
+   * @param {Hash} blockhash
+   * @returns {Promise<Buffer>} - {@link ChainEntry}
    */
 
-  getEntry(block) {
-    return this.call('get entry', block);
+  getEntry(blockhash) {
+    return this.call('get entry', blockhash);
   }
 
   /**
    * Get hashes.
    * @param {Number} [start=-1]
    * @param {Number} [end=-1]
-   * @returns {Promise}
+   * @returns {Promise<Hash[]>}
    */
 
   getHashes(start, end) {
     return this.call('get hashes', start, end);
   }
 
+  /**
+   * Get entries.
+   * @param {Number} [start=-1]
+   * @param {Number} [end=-1]
+   * @returns {Promise<Buffer[]>} - {@link ChainEntry}
+   */
+
+  getEntries(start, end) {
+    return this.call('get entries', start, end);
+  }
+
   /**
    * Send a transaction. Do not wait for promise.
    * @param {TX} tx