From c1097507d86a44cf7b0c886eb4031a4a02647c5d Mon Sep 17 00:00:00 2001 From: Willy Bruns Date: Sun, 9 Oct 2016 18:34:45 -0700 Subject: [PATCH 1/4] Add tools for generating a simulated Payment History for testing - `js/lib/transactionHelpers.js`: Tools for generating and validating (w/ Joi) Ledger transactions - `npm run add-simulated-payment-history`: new npm script which generates a simulated payment history, allowing testing related features without spending BTC fixes #4199 --- js/lib/transactionHelpers.js | 198 ++++++++++++++++++++++++ package.json | 1 + tools/addSimulatedPaymentHistory.js | 11 ++ tools/clean.js | 16 +- tools/lib/simulateLedgerTransactions.js | 30 ++++ tools/lib/utilApp/index.js | 20 +++ tools/utilAppRunner.js | 21 +++ 7 files changed, 283 insertions(+), 14 deletions(-) create mode 100644 js/lib/transactionHelpers.js create mode 100644 tools/addSimulatedPaymentHistory.js create mode 100644 tools/lib/simulateLedgerTransactions.js create mode 100644 tools/utilAppRunner.js diff --git a/js/lib/transactionHelpers.js b/js/lib/transactionHelpers.js new file mode 100644 index 00000000000..b02d6d0cabd --- /dev/null +++ b/js/lib/transactionHelpers.js @@ -0,0 +1,198 @@ +let Joi = require('joi') + +const SATOSHIS_PER_BTC = Math.pow(10, 8) + +const VALID_CURRENCY_CODE_REGEX = /^[A-Z]+$/ +const VALID_HOSTNAME_REGEX = /^(([a-zA-Z0-9]|[a-zA-Z0-9][a-zA-Z0-9\-]*[a-zA-Z0-9])\.)*([A-Za-z0-9]|[A-Za-z0-9][A-Za-z0-9\-]*[A-Za-z0-9])$/ + +const transactionSchema = Joi.object().keys({ + viewingId: Joi.string().guid().required().description('a unique id for the transaction'), + surveyorId: Joi.string().length(32, 'base64').required().description('a unique id for the surveyor. 32 bytes, base64 encoded'), + contribution: Joi.object().keys({ + fiat: Joi.object().keys({ + amount: Joi.number().min(0).precision(2).required(), + currency: Joi.string().required() + }).required(), + rates: Joi.object().pattern(VALID_CURRENCY_CODE_REGEX, Joi.number().min(0).precision(2)), + satoshis: Joi.number().integer().min(0).required().description('contribution amount in satoshis (10E-8 BTC)'), + fee: Joi.number().integer().min(0).required().description('transaction fee in satoshis for the contribution (10E-8 BTC)') + }).required().description('object describing contribution in fiat and BTC, with exchange rate at time of contribution and network transaction fee'), + submissionStamp: Joi.date().timestamp().required(), + /** submissionId is 32 bytes, 64 chars hex-encoded **/ + submissionId: Joi.string().hex().length(32, 'hex').required(), + count: Joi.number().integer().min(0), + + /** credential: + * complicated JSON BLOB from anonize2, come back to this later + **/ + credential: Joi.string(), + + /** surveyorIds: + * an array of random 32-byte ids (base64-encoded). + * length should equal sibling value `count` + **/ + surveyorIds: Joi.array().items(Joi.string().length(32, 'base64')), // ideally something like .length(Joi.ref('count')) + + satoshis: Joi.ref('contribution.satoshis'), + votes: Joi.ref('count'), + ballots: Joi.object().pattern(VALID_HOSTNAME_REGEX, Joi.number().integer().min(0)) +}) + +const validateTransaction = function (tx) { + return Joi.validate(tx, transactionSchema) +} + +const generateTransaction = function () { + const count = Math.round(Math.random() * 100) + + const viewingId = generateViewingId() + const surveyorId = generateSurveyorId() + const contribution = generateContribution() + const submissionStamp = generateSubmissionStamp() + const submissionDate = new Date(submissionStamp) + const submissionId = generateSubmissionId() + const credential = generateCredential() // what args needed? + const surveyorIds = generateSurveyorIds(count) + const satoshis = contribution.satoshis + const votes = count + const ballots = generateBallots(votes) + + return { + viewingId, + surveyorId, + contribution, + submissionStamp, + submissionDate, + submissionId, + credential, + surveyorIds, + count, + satoshis, + votes, + ballots + } +} + +/** code for generating transaction object components **/ +const uuid = require('node-uuid') +const crypto = require('crypto') +const randomBytes = crypto.randomBytes + +const generateViewingId = function () { + return uuid.v4().toLowerCase() +} + +const generateSurveyorId = function () { + /** + Random 32 bytes, generated in node-anonize2-relic/anonize2/anon.cpp:createSurvey as a random Big: + --- + Big vid; + rand_int(vid); + --- + Expressed in base64 everywhere in JS-land + **/ + return randomBytes(32).toString('base64') +} + +const generateSurveyorIds = function (count) { + if (!count) { + return [] + } + + return (new Array(count)).fill(null).map(generateSurveyorId) +} + +const generateContribution = function (satoshis, currency, rate, fee) { + let plusOrMinusTenPercentFactor = 1 + (2 * (Math.random() - 0.5) * 0.1) + let randomExchangeRateUSDPerBTC = 620 * plusOrMinusTenPercentFactor + let randomExchangeRateSatoshisPerUSD = (1 / randomExchangeRateUSDPerBTC) * SATOSHIS_PER_BTC + let randomContributionAmountUSD = [5, 10, 15][ Math.round(Math.random() * 2) ] + + currency = currency || 'USD' + currency = currency.toUpperCase() + rate = rate || randomExchangeRateUSDPerBTC + satoshis = satoshis || Math.round(randomContributionAmountUSD * randomExchangeRateSatoshisPerUSD) + fee = fee || (0.0001 * SATOSHIS_PER_BTC) + + let rates = {} + rates[currency] = parseFloat(rate.toFixed(2)) + + if (currency.toUpperCase() !== 'USD') { + rates.USD = randomExchangeRateUSDPerBTC + } + + let contribution = { + fiat: { + amount: parseFloat((satoshis / SATOSHIS_PER_BTC * rate).toFixed(2)), + currency: currency + }, + rates: rates, + satoshis: satoshis, + fee: fee + } + + return contribution +} + +const generateSubmissionStamp = function () { + return (new Date()).getTime() +} + +const generateSubmissionId = function () { + return randomBytes(32).toString('hex') +} + +// this one is a TODO, as it is a complicated JSON blob from anonize +const generateCredential = function () { + return 'PLACEHOLDER_CREDENTIAL_STRING' +} + +const generateBallots = function (votes) { + let ballots = {} + + let votesRemaining = votes + + while (votesRemaining) { + let votesToCast = Math.min(Math.round(Math.random() * votesRemaining) + 1, votesRemaining) + let host = _generateRandomHost() + ballots[host] = votesToCast + votesRemaining -= votesToCast + } + + return ballots +} + +const ALPHABET = 'abcdefghijklmnopqrstuvwxyz' +const _chooseRandomLetter = function () { + return ALPHABET[Math.round(Math.random() * (ALPHABET.length - 1))] +} + +const _generateRandomString = function (len) { + return (new Array(len)).fill(null).map(_chooseRandomLetter).join('') +} + +const TLDS = ['com', 'net', 'org', 'io', 'info'] + +const _generateRandomHost = function (maxLength, minLength) { + maxLength = maxLength || 10 + minLength = minLength || 4 + + let len = Math.max(Math.round(Math.random() * maxLength), minLength) + + let tld = TLDS[Math.round(Math.random() * (TLDS.length - 1))] + + let numParts = Math.round(Math.random()) + 1 + + let host = (new Array(numParts)).fill(null).map(function () { + let partLen = Math.max(Math.round(Math.random() * len), minLength) + return _generateRandomString(partLen) + }).join('.') + '.' + tld + + return host +} + +module.exports = { + transactionSchema, + validateTransaction, + generateTransaction +} diff --git a/package.json b/package.json index d1c35cebc4f..163d6fc7c41 100644 --- a/package.json +++ b/package.json @@ -7,6 +7,7 @@ "port": "8080" }, "scripts": { + "add-simulated-payment-history": "node ./tools/addSimulatedPaymentHistory.js", "build-installer": "node ./tools/buildInstaller.js", "build-package": "node ./tools/buildPackage.js", "check-security": "nsp check", diff --git a/tools/addSimulatedPaymentHistory.js b/tools/addSimulatedPaymentHistory.js new file mode 100644 index 00000000000..cc7e4e0f433 --- /dev/null +++ b/tools/addSimulatedPaymentHistory.js @@ -0,0 +1,11 @@ +let runUtilApp = require('./utilAppRunner') + +let cmd = 'addSimulatedLedgerTransactions' + +// if user has specified number of simulated transactions to add +if (process.argv[3]) { + cmd += ' ' + process.argv[3] +} + +console.log(`passing runUtilApp the command "${cmd}"`) +runUtilApp(cmd) diff --git a/tools/clean.js b/tools/clean.js index 4dd07ac387f..1283914f9c4 100644 --- a/tools/clean.js +++ b/tools/clean.js @@ -1,22 +1,10 @@ const path = require('path') -const proc = require('child_process') const rimraf = require('./lib/rimraf') const rootDir = path.join(__dirname, '..') -function runUtilApp (cmd, file) { - process.env.NODE_ENV = process.env.NODE_ENV || 'development' - const utilAppDir = path.join(__dirname, 'lib', 'utilApp') - const options = { - env: process.env, - cwd: utilAppDir - } - cmd = cmd.split(' ') - const utilApp = proc.spawnSync('electron', [utilAppDir].concat(cmd), options) - if (utilApp.error) { - console.log('Could not run utilApp - run `npm install electron-prebuilt` and try again', utilApp.error) - } -} +process.env.NODE_ENV = process.env.NODE_ENV || 'development' +const runUtilApp = require('./utilAppRunner') module.exports.nodeModules = () => { console.warn('removing node_modules...') diff --git a/tools/lib/simulateLedgerTransactions.js b/tools/lib/simulateLedgerTransactions.js new file mode 100644 index 00000000000..0fbc1661736 --- /dev/null +++ b/tools/lib/simulateLedgerTransactions.js @@ -0,0 +1,30 @@ +const TxHelpers = require('../../js/lib/transactionHelpers') + +let currentTimestamp = (new Date()).getTime() + +const getNthContributionPeriodBack = function (n) { + return currentTimestamp - (1000 * 3600 * 24 * 30) * n +} + +function simulateLedgerTransactions (numTx) { + let numTransactions = numTx || 10 + + let transactions = (new Array(numTransactions)) + .fill(null) + .map(function (nothing, idx) { + let tx = TxHelpers.generateTransaction() + tx.submissionStamp = getNthContributionPeriodBack(idx) + tx.submissionDate = new Date(tx.submissionStamp) + + let validatorOutput = TxHelpers.validateTransaction(tx) + if (validatorOutput.error) { + console.error(validatorOutput.error) + } + + return tx + }) + + return transactions +} + +module.exports = simulateLedgerTransactions diff --git a/tools/lib/utilApp/index.js b/tools/lib/utilApp/index.js index 15a219211f2..c8115ad1cf3 100644 --- a/tools/lib/utilApp/index.js +++ b/tools/lib/utilApp/index.js @@ -16,12 +16,32 @@ const cleanUserData = (location) => { } } +const simulateLedgerTransactions = require('../simulateLedgerTransactions') +const fs = require('fs') + +function addSimulatedLedgerTransactions (numTx) { + let userDataPath = app.getPath('userData') + let ledgerStatePath = path.join(userDataPath, 'ledger-state.json') + + let ledgerState = JSON.parse(fs.readFileSync(ledgerStatePath).toString()) + + let simulatedTransactions = simulateLedgerTransactions(numTx) + + ledgerState.transactions = (ledgerState.transactions || []).concat(simulatedTransactions) + + fs.writeFileSync(ledgerStatePath, JSON.stringify(ledgerState, null, 2)) +} + app.on('ready', () => { const cmd = process.argv[2] switch (cmd) { case 'cleanUserData': cleanUserData(process.argv[3]) break + case 'addSimulatedLedgerTransactions': + addSimulatedLedgerTransactions(process.argv[3]) + break } + process.exit(0) }) diff --git a/tools/utilAppRunner.js b/tools/utilAppRunner.js new file mode 100644 index 00000000000..eb7c1c29f7d --- /dev/null +++ b/tools/utilAppRunner.js @@ -0,0 +1,21 @@ +const path = require('path') +const proc = require('child_process') + +function runUtilApp (cmd, file) { + console.log('runUtilApp: ') + console.log(JSON.stringify(arguments, null, 2)) + + process.env.NODE_ENV = process.env.NODE_ENV || 'development' + const utilAppDir = path.join(__dirname, 'lib', 'utilApp') + const options = { + env: process.env, + cwd: utilAppDir + } + cmd = cmd.split(' ') + const utilApp = proc.spawnSync('electron', [utilAppDir].concat(cmd), options) + if (utilApp.error) { + console.log('Could not run utilApp - run `npm install electron-prebuilt` and try again', utilApp.error) + } +} + +module.exports = runUtilApp From e71bf994e893104df58fd05f3c13727c18a25198 Mon Sep 17 00:00:00 2001 From: Willy Bruns Date: Mon, 10 Oct 2016 11:06:32 -0700 Subject: [PATCH 2/4] Fix argument allowing # of simulated transactions to be specified and remove log lines --- tools/addSimulatedPaymentHistory.js | 5 ++--- tools/utilAppRunner.js | 1 - 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/tools/addSimulatedPaymentHistory.js b/tools/addSimulatedPaymentHistory.js index cc7e4e0f433..0ed22ed7240 100644 --- a/tools/addSimulatedPaymentHistory.js +++ b/tools/addSimulatedPaymentHistory.js @@ -3,9 +3,8 @@ let runUtilApp = require('./utilAppRunner') let cmd = 'addSimulatedLedgerTransactions' // if user has specified number of simulated transactions to add -if (process.argv[3]) { - cmd += ' ' + process.argv[3] +if (process.argv[2]) { + cmd += ' ' + process.argv[2] } -console.log(`passing runUtilApp the command "${cmd}"`) runUtilApp(cmd) diff --git a/tools/utilAppRunner.js b/tools/utilAppRunner.js index eb7c1c29f7d..0c3403dea5e 100644 --- a/tools/utilAppRunner.js +++ b/tools/utilAppRunner.js @@ -3,7 +3,6 @@ const proc = require('child_process') function runUtilApp (cmd, file) { console.log('runUtilApp: ') - console.log(JSON.stringify(arguments, null, 2)) process.env.NODE_ENV = process.env.NODE_ENV || 'development' const utilAppDir = path.join(__dirname, 'lib', 'utilApp') From d8ecf93e298622c3317f4fa6a3a987561958c3b4 Mon Sep 17 00:00:00 2001 From: Willy Bruns Date: Mon, 10 Oct 2016 11:34:46 -0700 Subject: [PATCH 3/4] Added error message for "npm run add-simulated-payment-history" - added informative error message for if /\/ledger-state.json does not exist - tool scripts spawning Electron can pass stdio argument to utilAppRunner to see Electron logs - added submissionDate to Joi schema to correct an error --- js/lib/transactionHelpers.js | 1 + tools/addSimulatedPaymentHistory.js | 2 +- tools/lib/utilApp/index.js | 16 ++++++++++++---- tools/utilAppRunner.js | 6 ++++-- 4 files changed, 18 insertions(+), 7 deletions(-) diff --git a/js/lib/transactionHelpers.js b/js/lib/transactionHelpers.js index b02d6d0cabd..c71dc4b2e0e 100644 --- a/js/lib/transactionHelpers.js +++ b/js/lib/transactionHelpers.js @@ -20,6 +20,7 @@ const transactionSchema = Joi.object().keys({ submissionStamp: Joi.date().timestamp().required(), /** submissionId is 32 bytes, 64 chars hex-encoded **/ submissionId: Joi.string().hex().length(32, 'hex').required(), + submissionDate: Joi.date(), count: Joi.number().integer().min(0), /** credential: diff --git a/tools/addSimulatedPaymentHistory.js b/tools/addSimulatedPaymentHistory.js index 0ed22ed7240..ae7dfd02e33 100644 --- a/tools/addSimulatedPaymentHistory.js +++ b/tools/addSimulatedPaymentHistory.js @@ -7,4 +7,4 @@ if (process.argv[2]) { cmd += ' ' + process.argv[2] } -runUtilApp(cmd) +runUtilApp(cmd, undefined, ['inherit', 'inherit', 'inherit']) diff --git a/tools/lib/utilApp/index.js b/tools/lib/utilApp/index.js index c8115ad1cf3..22af9d761e4 100644 --- a/tools/lib/utilApp/index.js +++ b/tools/lib/utilApp/index.js @@ -23,13 +23,21 @@ function addSimulatedLedgerTransactions (numTx) { let userDataPath = app.getPath('userData') let ledgerStatePath = path.join(userDataPath, 'ledger-state.json') - let ledgerState = JSON.parse(fs.readFileSync(ledgerStatePath).toString()) + try { + let ledgerState = JSON.parse(fs.readFileSync(ledgerStatePath).toString()) - let simulatedTransactions = simulateLedgerTransactions(numTx) + let simulatedTransactions = simulateLedgerTransactions(numTx) - ledgerState.transactions = (ledgerState.transactions || []).concat(simulatedTransactions) + ledgerState.transactions = (ledgerState.transactions || []).concat(simulatedTransactions) - fs.writeFileSync(ledgerStatePath, JSON.stringify(ledgerState, null, 2)) + fs.writeFileSync(ledgerStatePath, JSON.stringify(ledgerState, null, 2)) + + console.log(`Updated Ledger data file at ${ledgerStatePath}`) + } catch (exc) { + console.error('ERROR in addSimulatedLedgerTransactions: could not find/open/parse Ledger data file.') + console.error(`Expected path to Ledger data file: ${ledgerStatePath}`) + console.error('Probable solution: Run Brave and enable Payments, then execute this script. Enabling/disabling Payments (or restarting Brave) should then show the generated transactions in Brave.') + } } app.on('ready', () => { diff --git a/tools/utilAppRunner.js b/tools/utilAppRunner.js index 0c3403dea5e..bcfb70ac792 100644 --- a/tools/utilAppRunner.js +++ b/tools/utilAppRunner.js @@ -1,20 +1,22 @@ const path = require('path') const proc = require('child_process') -function runUtilApp (cmd, file) { +function runUtilApp (cmd, file, stdioOptions) { console.log('runUtilApp: ') process.env.NODE_ENV = process.env.NODE_ENV || 'development' const utilAppDir = path.join(__dirname, 'lib', 'utilApp') const options = { env: process.env, - cwd: utilAppDir + cwd: utilAppDir, + stdio: stdioOptions } cmd = cmd.split(' ') const utilApp = proc.spawnSync('electron', [utilAppDir].concat(cmd), options) if (utilApp.error) { console.log('Could not run utilApp - run `npm install electron-prebuilt` and try again', utilApp.error) } + return utilApp } module.exports = runUtilApp From b4329bb252b4861e9b2fcb3d5fc56e388de38e81 Mon Sep 17 00:00:00 2001 From: Willy Bruns Date: Mon, 10 Oct 2016 15:49:22 -0700 Subject: [PATCH 4/4] Moved js/lib/transactionHelpers.js -> tools/lib/transactionHelpers.js --- tools/lib/simulateLedgerTransactions.js | 2 +- {js => tools}/lib/transactionHelpers.js | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename {js => tools}/lib/transactionHelpers.js (100%) diff --git a/tools/lib/simulateLedgerTransactions.js b/tools/lib/simulateLedgerTransactions.js index 0fbc1661736..6f1ce7fa691 100644 --- a/tools/lib/simulateLedgerTransactions.js +++ b/tools/lib/simulateLedgerTransactions.js @@ -1,4 +1,4 @@ -const TxHelpers = require('../../js/lib/transactionHelpers') +const TxHelpers = require('./transactionHelpers') let currentTimestamp = (new Date()).getTime() diff --git a/js/lib/transactionHelpers.js b/tools/lib/transactionHelpers.js similarity index 100% rename from js/lib/transactionHelpers.js rename to tools/lib/transactionHelpers.js