Merge pull request #392 from braydonf/large-queries

Memory optimizations for large address queries

integration/regtest-node.js

@@ -28,6 +28,7 @@ var Transaction = index.Transaction;
 var BitcoreNode = index.Node;
 var AddressService = index.services.Address;
 var BitcoinService = index.services.Bitcoin;
+var encoding = require('../lib/services/address/encoding');
 var DBService = index.services.DB;
 var testWIF = 'cSdkPxkAjA4HDr5VHgsebAPDEh9Gyub4HK8UJr2DFGGqKKy4K5sG';
 var testKey;
@@ -43,22 +44,6 @@ describe('Node Functionality', function() {
  before(function(done) {
  this.timeout(30000);
 
- // Add the regtest network
- bitcore.Networks.remove(bitcore.Networks.testnet);
- bitcore.Networks.add({
- name: 'regtest',
- alias: 'regtest',
- pubkeyhash: 0x6f,
- privatekey: 0xef,
- scripthash: 0xc4,
- xpubkey: 0x043587cf,
- xprivkey: 0x04358394,
- networkMagic: 0xfabfb5da,
- port: 18444,
- dnsSeeds: [ ]
- });
- regtest = bitcore.Networks.get('regtest');
-
  var datadir = __dirname + '/data';
 
  testKey = bitcore.PrivateKey(testWIF);
@@ -93,6 +78,9 @@ describe('Node Functionality', function() {
 
  node = new BitcoreNode(configuration);
 
+ regtest = bitcore.Networks.get('regtest');
+ should.exist(regtest);
+
  node.on('error', function(err) {
  log.error(err);
  });
@@ -208,7 +196,7 @@ describe('Node Functionality', function() {
 
  // We need to add a transaction to the mempool so that the next block will
  // have a different hash as the hash has been invalidated.
- client.sendToAddress(testKey.toAddress().toString(), 10, function(err) {
+ client.sendToAddress(testKey.toAddress(regtest).toString(), 10, function(err) {
  if (err) {
  throw err;
  }
@@ -250,7 +238,7 @@ describe('Node Functionality', function() {
  var address;
  var unspentOutput;
  before(function() {
- address = testKey.toAddress().toString();
+ address = testKey.toAddress(regtest).toString();
  });
  it('should be able to get the balance of the test address', function(done) {
  node.services.address.getBalance(address, false, function(err, balance) {
@@ -333,19 +321,19 @@ describe('Node Functionality', function() {
  /* jshint maxstatements: 50 */
 
  testKey2 = bitcore.PrivateKey.fromWIF('cNfF4jXiLHQnFRsxaJyr2YSGcmtNYvxQYSakNhuDGxpkSzAwn95x');
- address2 = testKey2.toAddress().toString();
+ address2 = testKey2.toAddress(regtest).toString();
 
  testKey3 = bitcore.PrivateKey.fromWIF('cVTYQbaFNetiZcvxzXcVMin89uMLC43pEBMy2etgZHbPPxH5obYt');
- address3 = testKey3.toAddress().toString();
+ address3 = testKey3.toAddress(regtest).toString();
 
  testKey4 = bitcore.PrivateKey.fromWIF('cPNQmfE31H2oCUFqaHpfSqjDibkt7XoT2vydLJLDHNTvcddCesGw');
- address4 = testKey4.toAddress().toString();
+ address4 = testKey4.toAddress(regtest).toString();
 
  testKey5 = bitcore.PrivateKey.fromWIF('cVrzm9gCmnzwEVMGeCxY6xLVPdG3XWW97kwkFH3H3v722nb99QBF');
- address5 = testKey5.toAddress().toString();
+ address5 = testKey5.toAddress(regtest).toString();
 
  testKey6 = bitcore.PrivateKey.fromWIF('cPfMesNR2gsQEK69a6xe7qE44CZEZavgMUak5hQ74XDgsRmmGBYF');
- address6 = testKey6.toAddress().toString();
+ address6 = testKey6.toAddress(regtest).toString();
 
  var tx = new Transaction();
  tx.from(unspentOutput);
@@ -726,7 +714,7 @@ describe('Node Functionality', function() {
  node.services.bitcoind.sendTransaction(tx.serialize());
 
  setImmediate(function() {
- var addrObj = node.services.address._getAddressInfo(address);
+ var addrObj = encoding.getAddressInfo(address);
  node.services.address._getOutputsMempool(address, addrObj.hashBuffer,
  addrObj.hashTypeBuffer, function(err, outs) {
  if (err) {

lib/services/address/constants.js

@@ -0,0 +1,54 @@
+'use strict';
+
+var exports = {};
+
+exports.PREFIXES = {
+ OUTPUTS: new Buffer('02', 'hex'), // Query outputs by address and/or height
+ SPENTS: new Buffer('03', 'hex'), // Query inputs by address and/or height
+ SPENTSMAP: new Buffer('05', 'hex') // Get the input that spends an output
+};
+
+exports.MEMPREFIXES = {
+ OUTPUTS: new Buffer('01', 'hex'), // Query mempool outputs by address
+ SPENTS: new Buffer('02', 'hex'), // Query mempool inputs by address
+ SPENTSMAP: new Buffer('03', 'hex') // Query mempool for the input that spends an output
+};
+
+// To save space, we're only storing the PubKeyHash or ScriptHash in our index.
+// To avoid intentional unspendable collisions, which have been seen on the blockchain,
+// we must store the hash type (PK or Script) as well.
+exports.HASH_TYPES = {
+ PUBKEY: new Buffer('01', 'hex'),
+ REDEEMSCRIPT: new Buffer('02', 'hex')
+};
+
+// Translates from our enum type back into the hash types returned by
+// bitcore-lib/address.
+exports.HASH_TYPES_READABLE = {
+ '01': 'pubkeyhash',
+ '02': 'scripthash'
+};
+
+exports.HASH_TYPES_MAP = {
+ 'pubkeyhash': exports.HASH_TYPES.PUBKEY,
+ 'scripthash': exports.HASH_TYPES.REDEEMSCRIPT
+};
+
+exports.SPACER_MIN = new Buffer('00', 'hex');
+exports.SPACER_MAX = new Buffer('ff', 'hex');
+exports.SPACER_HEIGHT_MIN = new Buffer('0000000000', 'hex');
+exports.SPACER_HEIGHT_MAX = new Buffer('ffffffffff', 'hex');
+exports.TIMESTAMP_MIN = new Buffer('0000000000000000', 'hex');
+exports.TIMESTAMP_MAX = new Buffer('ffffffffffffffff', 'hex');
+
+// The maximum number of inputs that can be queried at once
+exports.MAX_INPUTS_QUERY_LENGTH = 50000;
+// The maximum number of outputs that can be queried at once
+exports.MAX_OUTPUTS_QUERY_LENGTH = 50000;
+// The maximum number of transactions that can be queried at once
+exports.MAX_HISTORY_QUERY_LENGTH = 100;
+// The maximum number of addresses that can be queried at once
+exports.MAX_ADDRESSES_QUERY = 10000;
+
+module.exports = exports;
+