From 81fcbbe6413b8fcf063a3304252bb944e9787a49 Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Mon, 10 Dec 2018 12:56:49 -0500 Subject: [PATCH 01/10] feat(config): add ledger env vars and config --- config/custom-environment-variables.json | 8 ++++++++ config/development.json | 8 ++++++++ 2 files changed, 16 insertions(+) diff --git a/config/custom-environment-variables.json b/config/custom-environment-variables.json index f838e61406e..103d11c9f9f 100644 --- a/config/custom-environment-variables.json +++ b/config/custom-environment-variables.json @@ -100,5 +100,13 @@ "recaptcha": { "siteKey": "RECAPTCHA_SITE_KEY", "secretKey": "RECAPTCHA_SECRET_KEY" + }, + "ledger": { + "url": "LEDGER_URL", + "transactionUrl": "LEDGER_TRANSACTION_URL" + }, + "ledgerQueue": { + "url": "LEDGER_QUEUE_URL", + "transactionQueue": "TRANSACTION_QUEUE_NAME" } } diff --git a/config/development.json b/config/development.json index 1241437f824..7b4ee51a606 100644 --- a/config/development.json +++ b/config/development.json @@ -30,5 +30,13 @@ "github": { "clientID": "cdc163e5f5695e16787b", "clientSecret": "1cc8e3a615af3a8ca47cd1f15adcfcdd1e880db4" + }, + "ledger": { + "url": "http://localhost:3070", + "transactionUrl": "http://localhost:3070/transactions" + }, + "ledgerQueue": { + "url": "amqp://localhost", + "transactionQueue": "transactions" } } From 146d18834c47650693681597753a93882497ecf9 Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Mon, 10 Dec 2018 14:20:50 -0500 Subject: [PATCH 02/10] feat(lib/transactions): add methods that gets the ledger transactions and parse them into the api format --- server/lib/transactions.js | 116 +++++++++++++++++++++++++++++++++++++ 1 file changed, 116 insertions(+) diff --git a/server/lib/transactions.js b/server/lib/transactions.js index f7cdb67759e..36d5b1f0e18 100644 --- a/server/lib/transactions.js +++ b/server/lib/transactions.js @@ -1,3 +1,4 @@ +import { get } from 'lodash'; import models, { Op, sequelize } from '../models'; import errors from '../lib/errors'; import { TransactionTypes } from '../constants/transactions'; @@ -5,6 +6,14 @@ import { getFxRate } from '../lib/currency'; import { exportToCSV } from '../lib/utils'; import { toNegative } from '../lib/math'; +const ledgerTransactionCategories = Object.freeze({ + PLATFORM: 'Platform Fee', + PAYMENT_PROVIDER: 'Payment Provider Fee', + WALLET_PROVIDER: 'Wallet Provider Fee', + ACCOUNT: 'Account to Account', + CURRENCY_CONVERSION: 'Currency Conversion', +}); + /** * Export transactions as CSV * @param {*} transactions @@ -161,6 +170,113 @@ export async function createTransactionFromInKindDonation(expenseTransaction) { }); } +/** + * Gets "ledger"(from the ledger service) transactions and format them + * to the api transactions + * @param {Number} legacyId - corresponding id from the opencollective-api Transactions table + * @param {Array} transactions - array of transactions representing one "legacy" transaction + * @param {Object} legacyInformation - extra information from corresponding legacy transaction + * @return {Object} returns a transaction with a similar format of the model Transaction + */ +export function parseLedgerTransactionToApiFormat(legacyId, transactions, legacyInformation) { + const { AccountId, legacyUuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId } = legacyInformation; + const creditTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.ACCOUNT || + t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && + t.type === 'CREDIT' + ); + }); + // setting up type, From and to accounts + const FromAccountId = parseInt(creditTransaction[0].FromAccountId); + const ToAccountId = parseInt(creditTransaction[0].ToAccountId); + const type = AccountId === FromAccountId ? 'DEBIT' : 'CREDIT'; + // finding fees, accounts and currency conversion transactions + // separately + const platformFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.PLATFORM || + t.category === `REFUND: ${ledgerTransactionCategories.PLATFORM}`) && + t.type === 'DEBIT' + ); + }); + const paymentFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.PAYMENT_PROVIDER || + t.category === `REFUND: ${ledgerTransactionCategories.PAYMENT_PROVIDER}`) && + t.type === 'DEBIT' + ); + }); + const hostFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.WALLET_PROVIDER || + t.category === `REFUND: ${ledgerTransactionCategories.WALLET_PROVIDER}`) && + t.type === 'DEBIT' + ); + }); + const accountTransaction = + type === 'CREDIT' + ? creditTransaction + : transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.ACCOUNT || + t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && + t.type === type + ); + }); + // setting up currency and amount information + const hostFeeInHostCurrency = hostFeeTransaction.length > 0 ? hostFeeTransaction[0].amount : 0; + const platformFeeInHostCurrency = platformFeeTransaction.length > 0 ? platformFeeTransaction[0].amount : 0; + const paymentProcessorFeeInHostCurrency = paymentFeeTransaction.length > 0 ? paymentFeeTransaction[0].amount : 0; + let amount = accountTransaction[0].amount; + let netAmountInCollectiveCurrency = + amount + hostFeeInHostCurrency + platformFeeInHostCurrency + paymentProcessorFeeInHostCurrency; + // if type is DEBIT, amount and netAmount are calculated differently + if (type === 'DEBIT') { + amount = amount - hostFeeInHostCurrency - platformFeeInHostCurrency - paymentProcessorFeeInHostCurrency; + netAmountInCollectiveCurrency = accountTransaction[0].amount; + } + const currency = accountTransaction[0].forexRateSourceCoin; + const hostCurrency = accountTransaction[0].forexRateDestinationCoin; + const forexRate = accountTransaction[0].forexRate; + + const parsedTransaction = { + id: legacyId, + type, + amount, + currency, + HostCollectiveId, + hostCurrency, + hostFeeInHostCurrency, + platformFeeInHostCurrency, + paymentProcessorFeeInHostCurrency, + hostCurrencyFxRate: forexRate, + netAmountInCollectiveCurrency: parseInt(netAmountInCollectiveCurrency), + fromCollective: { id: type === 'DEBIT' ? ToAccountId : FromAccountId }, + collective: { id: type === 'DEBIT' ? FromAccountId : ToAccountId }, + description: accountTransaction[0].description, + createdAt: accountTransaction[0].createdAt, + updatedAt: accountTransaction[0].updatedAt, + uuid: legacyUuid, + UsingVirtualCardFromCollectiveId: VirtualCardCollectiveId, + // createdByUser: { type: UserType }, + // privateMessage: { type: GraphQLString }, + }; + // setting refund transaction + if (RefundTransactionId) { + parsedTransaction.refundTransaction = { + id: RefundTransactionId, + }; + } + // setting paymentMethod + if (get(creditTransaction[0], 'fromWallet.PaymentMethodId')) { + parsedTransaction.paymentMethod = { + id: parseInt(get(creditTransaction[0], 'fromWallet.PaymentMethodId')), + }; + } + return parsedTransaction; +} + /** * Calculate net amount of a transaction in the currency of the collective * Notes: From 5a33f026ab3490c317c8f5611cec92b5604fb5bc Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Mon, 10 Dec 2018 14:22:49 -0500 Subject: [PATCH 03/10] fix(graphql/TransactionInterface): validating transaction interface in case it's not a sequelize transaction object --- server/graphql/v1/TransactionInterface.js | 97 ++++++++++++++++++----- 1 file changed, 77 insertions(+), 20 deletions(-) diff --git a/server/graphql/v1/TransactionInterface.js b/server/graphql/v1/TransactionInterface.js index 7ada0e8ee09..4da07757e5a 100644 --- a/server/graphql/v1/TransactionInterface.js +++ b/server/graphql/v1/TransactionInterface.js @@ -1,3 +1,5 @@ +import { get } from 'lodash'; +import models from '../../../server/models'; import { GraphQLInt, GraphQLFloat, @@ -65,7 +67,10 @@ const TransactionFields = () => { refundTransaction: { type: TransactionInterfaceType, resolve(transaction) { - return transaction.getRefundTransaction(); + if (transaction && transaction.getRefundTransaction) { + return transaction.getRefundTransaction(); + } + return null; }, }, uuid: { @@ -74,7 +79,10 @@ const TransactionFields = () => { if (!req.remoteUser) { return null; } - return transaction.getDetailsForUser(req.remoteUser); + if (transaction && transaction.getDetailsForUser) { + return transaction.getDetailsForUser(req.remoteUser); + } + return null; }, }, type: { @@ -140,32 +148,69 @@ const TransactionFields = () => { }, host: { type: UserCollectiveType, - resolve(transaction) { - return transaction.getHostCollective(); + async resolve(transaction) { + if (transaction && transaction.getHostCollective) { + return transaction.getHostCollective(); + } + const FromCollectiveId = transaction.fromCollective.id; + const CollectiveId = transaction.collective.id; + let HostCollectiveId = transaction.HostCollectiveId; + // if the transaction is from the perspective of the fromCollective + if (!HostCollectiveId) { + const fromCollective = await models.Collective.findById(FromCollectiveId); + HostCollectiveId = await fromCollective.getHostCollectiveId(); + // if fromCollective has no host, we try the collective + if (!HostCollectiveId) { + const collective = await models.Collective.findById(CollectiveId); + HostCollectiveId = await collective.getHostCollectiveId(); + } + } + return models.Collective.findById(HostCollectiveId); }, }, createdByUser: { type: UserType, resolve(transaction) { - return transaction.getCreatedByUser(); + if (transaction && transaction.getCreatedByUser) { + return transaction.getCreatedByUser(); + } + return null; }, }, fromCollective: { type: CollectiveInterfaceType, resolve(transaction) { - return transaction.getFromCollective(); + if (transaction && transaction.getFromCollective) { + return transaction.getFromCollective(); + } + if (get(transaction, 'fromCollective.id')) { + return models.Collective.findById(get(transaction, 'fromCollective.id')); + } + return null; }, }, usingVirtualCardFromCollective: { type: CollectiveInterfaceType, resolve(transaction) { - return transaction.getVirtualCardEmitterCollective(); + if (transaction && transaction.getVirtualCardEmitterCollective) { + return transaction.getVirtualCardEmitterCollective(); + } + if (transaction && transaction.UsingVirtualCardFromCollectiveId) { + return models.Collective.findById(transaction.UsingVirtualCardFromCollectiveId); + } + return null; }, }, collective: { type: CollectiveInterfaceType, resolve(transaction) { - return transaction.getCollective(); + if (transaction && transaction.getCollective) { + return transaction.getCollective(); + } + if (get(transaction, 'collective.id')) { + return models.Collective.findById(get(transaction, 'collective.id')); + } + return null; }, }, createdAt: { @@ -183,9 +228,10 @@ const TransactionFields = () => { paymentMethod: { type: PaymentMethodType, resolve(transaction, args, req) { - if (!transaction.PaymentMethodId) return null; + const paymentMethodId = transaction.PaymentMethodId || get(transaction, 'paymentMethod.id'); + if (!paymentMethodId) return null; // TODO: put behind a login check - return req.loaders.paymentMethods.findById.load(transaction.PaymentMethodId); + return req.loaders.paymentMethods.findById.load(paymentMethodId); }, }, }; @@ -200,25 +246,34 @@ export const TransactionExpenseType = new GraphQLObjectType({ description: { type: GraphQLString, resolve(transaction) { - return transaction.description || transaction.getExpense().then(expense => expense && expense.description); + const expense = transaction.getExpense + ? transaction.getExpense().then(expense => expense && expense.description) + : null; + return transaction.description || expense; }, }, privateMessage: { type: GraphQLString, resolve(transaction, args, req) { - return transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.privateMessage); + return transaction.getExpenseForViewer + ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.privateMessage) + : null; }, }, category: { type: GraphQLString, resolve(transaction, args, req) { - return transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.category); + return transaction.getExpenseForViewer + ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.category) + : null; }, }, attachment: { type: GraphQLString, resolve(transaction, args, req) { - return transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.attachment); + return transaction.getExpenseForViewer + ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.attachment) + : null; }, }, }; @@ -235,32 +290,34 @@ export const TransactionOrderType = new GraphQLObjectType({ description: { type: GraphQLString, resolve(transaction) { - return transaction.description || transaction.getOrder().then(order => order && order.description); + const getOrder = transaction.getOrder + ? transaction.getOrder().then(order => order && order.description) + : null; + return transaction.description || getOrder; }, }, privateMessage: { type: GraphQLString, resolve(transaction) { - // TODO: Put behind a login check - return transaction.getOrder().then(order => order && order.privateMessage); + return transaction.getOrder ? transaction.getOrder().then(order => order && order.privateMessage) : null; }, }, publicMessage: { type: GraphQLString, resolve(transaction) { - return transaction.getOrder().then(order => order && order.publicMessage); + return transaction.getOrder ? transaction.getOrder().then(order => order && order.publicMessage) : null; }, }, order: { type: OrderType, resolve(transaction) { - return transaction.getOrder(); + return transaction.getOrder ? transaction.getOrder() : null; }, }, subscription: { type: SubscriptionType, resolve(transaction) { - return transaction.getOrder().then(order => order && order.getSubscription()); + return transaction.getOrder ? transaction.getOrder().then(order => order && order.getSubscription()) : null; }, }, }; From a3b7344afd3d963c4f10e982ab246f2c821a6421 Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Mon, 10 Dec 2018 14:25:30 -0500 Subject: [PATCH 04/10] feat(graphql/queries/allTransactions): add fetchDataFromLedger flag to instead of returning normal api transactions, return from ledger --- server/graphql/v1/queries.js | 63 ++++++++++++++++++++++++++++++++++-- 1 file changed, 61 insertions(+), 2 deletions(-) diff --git a/server/graphql/v1/queries.js b/server/graphql/v1/queries.js index be9e554fbee..fe696ed2c09 100644 --- a/server/graphql/v1/queries.js +++ b/server/graphql/v1/queries.js @@ -1,8 +1,12 @@ import Promise from 'bluebird'; -import { find, get, uniq } from 'lodash'; +import { find, get, uniq, groupBy } from 'lodash'; +import axios from 'axios'; +import config from 'config'; +import queryString from 'query-string'; import algolia from '../../lib/algolia'; import errors from '../../lib/errors'; +import { parseLedgerTransactionToApiFormat } from '../../lib/transactions'; import { GraphQLList, GraphQLNonNull, GraphQLString, GraphQLInt, GraphQLBoolean } from 'graphql'; @@ -317,8 +321,20 @@ const queries = { dateTo: { type: GraphQLString }, /** @deprecated since 2018-11-29: Virtual cards now included by default when necessary */ includeVirtualCards: { type: GraphQLBoolean }, + fetchDataFromLedger: { type: GraphQLBoolean }, }, async resolve(_, args) { + const fetchDataFromLedger = args.fetchDataFromLedger || process.env.GET_TRANSACTIONS_FROM_LEDGER || false; + let attributes; + if (fetchDataFromLedger) { + attributes = [ + 'id', + 'uuid', // because the stored invoice pdf in aws uses the uuid as reference + 'UsingVirtualCardFromCollectiveId', // because virtual cards will only work for wallets and we're skipping wallets in the transactions details for now + 'HostCollectiveId', // because we're skipping wallets and using host on transactions details for now + 'RefundTransactionId', // because the ledger refundTransactionId refers to the ledger id and not the legacy one + ]; + } // Load collective const { CollectiveId, collectiveSlug } = args; if (!CollectiveId && !collectiveSlug) throw new Error('You must specify a collective ID or a Slug'); @@ -327,7 +343,8 @@ const queries = { if (!collective) throw new Error('This collective does not exist'); // Load transactions - return collective.getTransactions({ + const allTransactions = await collective.getTransactions({ + attributes: attributes, order: [['createdAt', 'DESC']], type: args.type, limit: args.limit, @@ -335,6 +352,48 @@ const queries = { startDate: args.dateFrom, endDate: args.dateTo, }); + if (!fetchDataFromLedger) return allTransactions; + + const ledgerQuery = { + where: { + ToAccountId: args.CollectiveId, + }, + limit: args.limit, + offset: args.offset, + }; + ledgerQuery.where = JSON.stringify(ledgerQuery.where); + const transactionsEndpointResult = await axios.get( + `${config.ledger.transactionUrl}?${queryString.stringify(ledgerQuery)}`, + ); + const ledgerTransactions = groupBy(transactionsEndpointResult.data || [], 'LegacyCreditTransactionId'); + // sort keys of result by legacy id DESC as lodash groupBy changes the order + const ledgerFormattedTransactions = []; + for (const key of Object.keys(ledgerTransactions).sort((a, b) => b - a)) { + // mapping legacy info to parse inside ledger mapping + const legacyMatchingTransaction = allTransactions.filter(t => { + return ( + t.id === ledgerTransactions[key][0].LegacyDebitTransactionId || + t.id === ledgerTransactions[key][0].LegacyCreditTransactionId + ); + })[0]; + let uuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId; + if (legacyMatchingTransaction) { + uuid = legacyMatchingTransaction.uuid; + VirtualCardCollectiveId = legacyMatchingTransaction.UsingVirtualCardFromCollectiveId; + HostCollectiveId = legacyMatchingTransaction.HostCollectiveId; + RefundTransactionId = legacyMatchingTransaction.RefundTransactionId; + } + ledgerFormattedTransactions.push( + parseLedgerTransactionToApiFormat(key, ledgerTransactions[key], { + AccountId: args.CollectiveId, + legacyUuid: uuid, + VirtualCardCollectiveId, + HostCollectiveId, + RefundTransactionId, + }), + ); + } + return ledgerFormattedTransactions; }, }, From d135669a407c2516d89a30f4be2eebf3b22f2352 Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Tue, 11 Dec 2018 08:42:02 -0500 Subject: [PATCH 05/10] feat(lib): create ledger lib and remove methods from transactions lib --- server/lib/ledger.js | 188 +++++++++++++++++++++++++++++++++++++ server/lib/transactions.js | 116 ----------------------- 2 files changed, 188 insertions(+), 116 deletions(-) create mode 100644 server/lib/ledger.js diff --git a/server/lib/ledger.js b/server/lib/ledger.js new file mode 100644 index 00000000000..0d9265f43d6 --- /dev/null +++ b/server/lib/ledger.js @@ -0,0 +1,188 @@ +import { get, groupBy } from 'lodash'; +import axios from 'axios'; +import config from 'config'; +import queryString from 'query-string'; + +const ledgerTransactionCategories = Object.freeze({ + PLATFORM: 'Platform Fee', + PAYMENT_PROVIDER: 'Payment Provider Fee', + WALLET_PROVIDER: 'Wallet Provider Fee', + ACCOUNT: 'Account to Account', + CURRENCY_CONVERSION: 'Currency Conversion', +}); + +/** + * Fetches transactions from the ledger service and return an array + * @param {Object} args - The graphql arguments(graphql/queries) + * @return {Map} returns a map where the key is the legacy id(api id) + * and value is the array with the ledger transactions + */ +export async function fetchLedgerTransactions(args) { + const ledgerQuery = { + where: { + ToAccountId: args.CollectiveId, + }, + limit: args.limit, + offset: args.offset, + }; + ledgerQuery.where = JSON.stringify(ledgerQuery.where); + return axios.get(`${config.ledger.transactionUrl}?${queryString.stringify(ledgerQuery)}`); +} +/** + * Fetches transactions from the ledger service and return them + * grouped by the LegacyCreditTransactionId(meaning api id) + * @param {Object} args - The graphql arguments(graphql/queries) + * @return {Map} returns a map where the key is the legacy id(api id) + * and value is the array with the ledger transactions + */ +export async function fetchLedgerTransactionsGroupedByLegacyIds(args) { + const transactionsEndpointResult = await fetchLedgerTransactions(args); + return groupBy(transactionsEndpointResult.data || [], 'LegacyCreditTransactionId'); +} + +/** + * Fetches transactions from the ledger service and return them + * grouped by the LegacyCreditTransactionId(meaning api id) + * @param {Number} CollectiveId - the CollectiveId that we're returning information + * @param {Map} ledgerTransactions - The map found for the ledger transactions(see method fetchLedgerTransactionsGroupedByLegacyIds) + * @param {Array} apiTransactions - The "api" transactions found for the givem CollectiveId + * @return {Map} returns a map where the key is the legacy id(api id) + * and value is the array with the ledger transactions + */ +export function parseLedgerTransactions(CollectiveId, ledgerTransactions, apiTransactions) { + // sort keys of result by legacy id DESC as lodash groupBy changes the order + const ledgerFormattedTransactions = []; + for (const key of Object.keys(ledgerTransactions).sort((a, b) => b - a)) { + // mapping legacy info to parse inside ledger mapping + const legacyMatchingTransaction = apiTransactions.filter(t => { + return ( + t.id === ledgerTransactions[key][0].LegacyDebitTransactionId || + t.id === ledgerTransactions[key][0].LegacyCreditTransactionId + ); + })[0]; + let uuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId; + if (legacyMatchingTransaction) { + uuid = legacyMatchingTransaction.uuid; + VirtualCardCollectiveId = legacyMatchingTransaction.UsingVirtualCardFromCollectiveId; + HostCollectiveId = legacyMatchingTransaction.HostCollectiveId; + RefundTransactionId = legacyMatchingTransaction.RefundTransactionId; + } + ledgerFormattedTransactions.push( + parseLedgerTransactionToApiFormat(key, ledgerTransactions[key], { + AccountId: CollectiveId, + legacyUuid: uuid, + VirtualCardCollectiveId, + HostCollectiveId, + RefundTransactionId, + }), + ); + } + return ledgerFormattedTransactions; +} + +/** + * Parses "ledger" transactions into the "api" transactions format them + * to the api transactions + * @param {Number} legacyId - corresponding id from the opencollective-api Transactions table + * @param {Array} transactions - array of transactions representing one "legacy" transaction + * @param {Object} legacyInformation - extra information from corresponding legacy transaction + * @return {Object} returns a transaction with a similar format of the model Transaction + */ +export function parseLedgerTransactionToApiFormat(legacyId, transactions, legacyInformation) { + const { AccountId, legacyUuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId } = legacyInformation; + const creditTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.ACCOUNT || + t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && + t.type === 'CREDIT' + ); + }); + // setting up type, From and to accounts + const FromAccountId = parseInt(creditTransaction[0].FromAccountId); + const ToAccountId = parseInt(creditTransaction[0].ToAccountId); + const type = AccountId === FromAccountId ? 'DEBIT' : 'CREDIT'; + // finding fees, accounts and currency conversion transactions + // separately + const platformFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.PLATFORM || + t.category === `REFUND: ${ledgerTransactionCategories.PLATFORM}`) && + t.type === 'DEBIT' + ); + }); + const paymentFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.PAYMENT_PROVIDER || + t.category === `REFUND: ${ledgerTransactionCategories.PAYMENT_PROVIDER}`) && + t.type === 'DEBIT' + ); + }); + const hostFeeTransaction = transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.WALLET_PROVIDER || + t.category === `REFUND: ${ledgerTransactionCategories.WALLET_PROVIDER}`) && + t.type === 'DEBIT' + ); + }); + const accountTransaction = + type === 'CREDIT' + ? creditTransaction + : transactions.filter(t => { + return ( + (t.category === ledgerTransactionCategories.ACCOUNT || + t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && + t.type === type + ); + }); + // setting up currency and amount information + const hostFeeInHostCurrency = hostFeeTransaction.length > 0 ? hostFeeTransaction[0].amount : 0; + const platformFeeInHostCurrency = platformFeeTransaction.length > 0 ? platformFeeTransaction[0].amount : 0; + const paymentProcessorFeeInHostCurrency = paymentFeeTransaction.length > 0 ? paymentFeeTransaction[0].amount : 0; + let amount = accountTransaction[0].amount; + let netAmountInCollectiveCurrency = + amount + hostFeeInHostCurrency + platformFeeInHostCurrency + paymentProcessorFeeInHostCurrency; + // if type is DEBIT, amount and netAmount are calculated differently + if (type === 'DEBIT') { + amount = amount - hostFeeInHostCurrency - platformFeeInHostCurrency - paymentProcessorFeeInHostCurrency; + netAmountInCollectiveCurrency = accountTransaction[0].amount; + } + const currency = accountTransaction[0].forexRateSourceCoin; + const hostCurrency = accountTransaction[0].forexRateDestinationCoin; + const forexRate = accountTransaction[0].forexRate; + + const parsedTransaction = { + id: legacyId, + type, + amount, + currency, + HostCollectiveId, + hostCurrency, + hostFeeInHostCurrency, + platformFeeInHostCurrency, + paymentProcessorFeeInHostCurrency, + hostCurrencyFxRate: forexRate, + netAmountInCollectiveCurrency: parseInt(netAmountInCollectiveCurrency), + fromCollective: { id: type === 'DEBIT' ? ToAccountId : FromAccountId }, + collective: { id: type === 'DEBIT' ? FromAccountId : ToAccountId }, + description: accountTransaction[0].description, + createdAt: accountTransaction[0].createdAt, + updatedAt: accountTransaction[0].updatedAt, + uuid: legacyUuid, + UsingVirtualCardFromCollectiveId: VirtualCardCollectiveId, + // createdByUser: { type: UserType }, + // privateMessage: { type: GraphQLString }, + }; + // setting refund transaction + if (RefundTransactionId) { + parsedTransaction.refundTransaction = { + id: RefundTransactionId, + }; + } + // setting paymentMethod + if (get(creditTransaction[0], 'fromWallet.PaymentMethodId')) { + parsedTransaction.paymentMethod = { + id: parseInt(get(creditTransaction[0], 'fromWallet.PaymentMethodId')), + }; + } + return parsedTransaction; +} diff --git a/server/lib/transactions.js b/server/lib/transactions.js index 36d5b1f0e18..f7cdb67759e 100644 --- a/server/lib/transactions.js +++ b/server/lib/transactions.js @@ -1,4 +1,3 @@ -import { get } from 'lodash'; import models, { Op, sequelize } from '../models'; import errors from '../lib/errors'; import { TransactionTypes } from '../constants/transactions'; @@ -6,14 +5,6 @@ import { getFxRate } from '../lib/currency'; import { exportToCSV } from '../lib/utils'; import { toNegative } from '../lib/math'; -const ledgerTransactionCategories = Object.freeze({ - PLATFORM: 'Platform Fee', - PAYMENT_PROVIDER: 'Payment Provider Fee', - WALLET_PROVIDER: 'Wallet Provider Fee', - ACCOUNT: 'Account to Account', - CURRENCY_CONVERSION: 'Currency Conversion', -}); - /** * Export transactions as CSV * @param {*} transactions @@ -170,113 +161,6 @@ export async function createTransactionFromInKindDonation(expenseTransaction) { }); } -/** - * Gets "ledger"(from the ledger service) transactions and format them - * to the api transactions - * @param {Number} legacyId - corresponding id from the opencollective-api Transactions table - * @param {Array} transactions - array of transactions representing one "legacy" transaction - * @param {Object} legacyInformation - extra information from corresponding legacy transaction - * @return {Object} returns a transaction with a similar format of the model Transaction - */ -export function parseLedgerTransactionToApiFormat(legacyId, transactions, legacyInformation) { - const { AccountId, legacyUuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId } = legacyInformation; - const creditTransaction = transactions.filter(t => { - return ( - (t.category === ledgerTransactionCategories.ACCOUNT || - t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && - t.type === 'CREDIT' - ); - }); - // setting up type, From and to accounts - const FromAccountId = parseInt(creditTransaction[0].FromAccountId); - const ToAccountId = parseInt(creditTransaction[0].ToAccountId); - const type = AccountId === FromAccountId ? 'DEBIT' : 'CREDIT'; - // finding fees, accounts and currency conversion transactions - // separately - const platformFeeTransaction = transactions.filter(t => { - return ( - (t.category === ledgerTransactionCategories.PLATFORM || - t.category === `REFUND: ${ledgerTransactionCategories.PLATFORM}`) && - t.type === 'DEBIT' - ); - }); - const paymentFeeTransaction = transactions.filter(t => { - return ( - (t.category === ledgerTransactionCategories.PAYMENT_PROVIDER || - t.category === `REFUND: ${ledgerTransactionCategories.PAYMENT_PROVIDER}`) && - t.type === 'DEBIT' - ); - }); - const hostFeeTransaction = transactions.filter(t => { - return ( - (t.category === ledgerTransactionCategories.WALLET_PROVIDER || - t.category === `REFUND: ${ledgerTransactionCategories.WALLET_PROVIDER}`) && - t.type === 'DEBIT' - ); - }); - const accountTransaction = - type === 'CREDIT' - ? creditTransaction - : transactions.filter(t => { - return ( - (t.category === ledgerTransactionCategories.ACCOUNT || - t.category === `REFUND: ${ledgerTransactionCategories.ACCOUNT}`) && - t.type === type - ); - }); - // setting up currency and amount information - const hostFeeInHostCurrency = hostFeeTransaction.length > 0 ? hostFeeTransaction[0].amount : 0; - const platformFeeInHostCurrency = platformFeeTransaction.length > 0 ? platformFeeTransaction[0].amount : 0; - const paymentProcessorFeeInHostCurrency = paymentFeeTransaction.length > 0 ? paymentFeeTransaction[0].amount : 0; - let amount = accountTransaction[0].amount; - let netAmountInCollectiveCurrency = - amount + hostFeeInHostCurrency + platformFeeInHostCurrency + paymentProcessorFeeInHostCurrency; - // if type is DEBIT, amount and netAmount are calculated differently - if (type === 'DEBIT') { - amount = amount - hostFeeInHostCurrency - platformFeeInHostCurrency - paymentProcessorFeeInHostCurrency; - netAmountInCollectiveCurrency = accountTransaction[0].amount; - } - const currency = accountTransaction[0].forexRateSourceCoin; - const hostCurrency = accountTransaction[0].forexRateDestinationCoin; - const forexRate = accountTransaction[0].forexRate; - - const parsedTransaction = { - id: legacyId, - type, - amount, - currency, - HostCollectiveId, - hostCurrency, - hostFeeInHostCurrency, - platformFeeInHostCurrency, - paymentProcessorFeeInHostCurrency, - hostCurrencyFxRate: forexRate, - netAmountInCollectiveCurrency: parseInt(netAmountInCollectiveCurrency), - fromCollective: { id: type === 'DEBIT' ? ToAccountId : FromAccountId }, - collective: { id: type === 'DEBIT' ? FromAccountId : ToAccountId }, - description: accountTransaction[0].description, - createdAt: accountTransaction[0].createdAt, - updatedAt: accountTransaction[0].updatedAt, - uuid: legacyUuid, - UsingVirtualCardFromCollectiveId: VirtualCardCollectiveId, - // createdByUser: { type: UserType }, - // privateMessage: { type: GraphQLString }, - }; - // setting refund transaction - if (RefundTransactionId) { - parsedTransaction.refundTransaction = { - id: RefundTransactionId, - }; - } - // setting paymentMethod - if (get(creditTransaction[0], 'fromWallet.PaymentMethodId')) { - parsedTransaction.paymentMethod = { - id: parseInt(get(creditTransaction[0], 'fromWallet.PaymentMethodId')), - }; - } - return parsedTransaction; -} - /** * Calculate net amount of a transaction in the currency of the collective * Notes: From 90d3d7f980a6a7e08d68b6d25164a53188e40c5f Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Tue, 11 Dec 2018 08:43:09 -0500 Subject: [PATCH 06/10] feat(graphql/queries/allTransactions): use ledger lib and fix flag 'fetchDataFromLedger' logic --- server/graphql/v1/queries.js | 95 +++++++++++------------------------- 1 file changed, 29 insertions(+), 66 deletions(-) diff --git a/server/graphql/v1/queries.js b/server/graphql/v1/queries.js index fe696ed2c09..3d4079e1741 100644 --- a/server/graphql/v1/queries.js +++ b/server/graphql/v1/queries.js @@ -1,12 +1,8 @@ import Promise from 'bluebird'; -import { find, get, uniq, groupBy } from 'lodash'; -import axios from 'axios'; -import config from 'config'; -import queryString from 'query-string'; - +import { find, get, uniq } from 'lodash'; import algolia from '../../lib/algolia'; import errors from '../../lib/errors'; -import { parseLedgerTransactionToApiFormat } from '../../lib/transactions'; +import { fetchLedgerTransactionsGroupedByLegacyIds, parseLedgerTransactions } from '../../lib/ledger'; import { GraphQLList, GraphQLNonNull, GraphQLString, GraphQLInt, GraphQLBoolean } from 'graphql'; @@ -324,16 +320,9 @@ const queries = { fetchDataFromLedger: { type: GraphQLBoolean }, }, async resolve(_, args) { - const fetchDataFromLedger = args.fetchDataFromLedger || process.env.GET_TRANSACTIONS_FROM_LEDGER || false; - let attributes; - if (fetchDataFromLedger) { - attributes = [ - 'id', - 'uuid', // because the stored invoice pdf in aws uses the uuid as reference - 'UsingVirtualCardFromCollectiveId', // because virtual cards will only work for wallets and we're skipping wallets in the transactions details for now - 'HostCollectiveId', // because we're skipping wallets and using host on transactions details for now - 'RefundTransactionId', // because the ledger refundTransactionId refers to the ledger id and not the legacy one - ]; + let fetchDataFromLedger = process.env.GET_TRANSACTIONS_FROM_LEDGER || false; + if (args.fetchDataFromLedger) { + fetchDataFromLedger = args.fetchDataFromLedger; } // Load collective const { CollectiveId, collectiveSlug } = args; @@ -342,58 +331,32 @@ const queries = { const collective = await models.Collective.findOne({ where }); if (!collective) throw new Error('This collective does not exist'); - // Load transactions - const allTransactions = await collective.getTransactions({ - attributes: attributes, - order: [['createdAt', 'DESC']], - type: args.type, - limit: args.limit, - offset: args.offset, - startDate: args.dateFrom, - endDate: args.dateTo, - }); - if (!fetchDataFromLedger) return allTransactions; - - const ledgerQuery = { + // returns transactions straight from the api + if (!fetchDataFromLedger) { + return collective.getTransactions({ + order: [['createdAt', 'DESC']], + type: args.type, + limit: args.limit, + offset: args.offset, + startDate: args.dateFrom, + endDate: args.dateTo, + }); + } + // otherwise returns data from the ledger + const ledgerTransactions = await fetchLedgerTransactionsGroupedByLegacyIds(args); + const apiTransactions = await models.Transaction.findAll({ + attributes: [ + 'id', + 'uuid', // because the stored invoice pdf in aws uses the uuid as reference + 'UsingVirtualCardFromCollectiveId', // because virtual cards will only work for wallets and we're skipping wallets in the transactions details for now + 'HostCollectiveId', // because we're skipping wallets and using host on transactions details for now + 'RefundTransactionId', // because the ledger refundTransactionId refers to the ledger id and not the legacy one + ], where: { - ToAccountId: args.CollectiveId, + id: Object.keys(ledgerTransactions), }, - limit: args.limit, - offset: args.offset, - }; - ledgerQuery.where = JSON.stringify(ledgerQuery.where); - const transactionsEndpointResult = await axios.get( - `${config.ledger.transactionUrl}?${queryString.stringify(ledgerQuery)}`, - ); - const ledgerTransactions = groupBy(transactionsEndpointResult.data || [], 'LegacyCreditTransactionId'); - // sort keys of result by legacy id DESC as lodash groupBy changes the order - const ledgerFormattedTransactions = []; - for (const key of Object.keys(ledgerTransactions).sort((a, b) => b - a)) { - // mapping legacy info to parse inside ledger mapping - const legacyMatchingTransaction = allTransactions.filter(t => { - return ( - t.id === ledgerTransactions[key][0].LegacyDebitTransactionId || - t.id === ledgerTransactions[key][0].LegacyCreditTransactionId - ); - })[0]; - let uuid, VirtualCardCollectiveId, HostCollectiveId, RefundTransactionId; - if (legacyMatchingTransaction) { - uuid = legacyMatchingTransaction.uuid; - VirtualCardCollectiveId = legacyMatchingTransaction.UsingVirtualCardFromCollectiveId; - HostCollectiveId = legacyMatchingTransaction.HostCollectiveId; - RefundTransactionId = legacyMatchingTransaction.RefundTransactionId; - } - ledgerFormattedTransactions.push( - parseLedgerTransactionToApiFormat(key, ledgerTransactions[key], { - AccountId: args.CollectiveId, - legacyUuid: uuid, - VirtualCardCollectiveId, - HostCollectiveId, - RefundTransactionId, - }), - ); - } - return ledgerFormattedTransactions; + }); + return parseLedgerTransactions(args.CollectiveId, ledgerTransactions, apiTransactions); }, }, From ecffeb703b0a6aaf70e23f0020fb9a7d7d93b79a Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Tue, 11 Dec 2018 14:46:51 -0500 Subject: [PATCH 07/10] feat: include flag includeHostedCollectivesTransactions to check whether its going to return transactions from the collectives of that host or only the transactions of the host itself --- server/graphql/v1/queries.js | 6 +++++- server/lib/ledger.js | 1 + 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/server/graphql/v1/queries.js b/server/graphql/v1/queries.js index 3d4079e1741..d47a2f59b1e 100644 --- a/server/graphql/v1/queries.js +++ b/server/graphql/v1/queries.js @@ -317,7 +317,11 @@ const queries = { dateTo: { type: GraphQLString }, /** @deprecated since 2018-11-29: Virtual cards now included by default when necessary */ includeVirtualCards: { type: GraphQLBoolean }, - fetchDataFromLedger: { type: GraphQLBoolean }, + fetchDataFromLedger: { type: GraphQLBoolean }, // flag to go with either api or ledger transactions + includeHostedCollectivesTransactions: { + type: GraphQLBoolean, + } /** flag to determine + whether we should include the transactions of the collectives of that host(if it's a host collective) */, }, async resolve(_, args) { let fetchDataFromLedger = process.env.GET_TRANSACTIONS_FROM_LEDGER || false; diff --git a/server/lib/ledger.js b/server/lib/ledger.js index 0d9265f43d6..6c52e371d6c 100644 --- a/server/lib/ledger.js +++ b/server/lib/ledger.js @@ -24,6 +24,7 @@ export async function fetchLedgerTransactions(args) { }, limit: args.limit, offset: args.offset, + includeHostedCollectivesTransactions: args.includeHostedCollectivesTransactions, }; ledgerQuery.where = JSON.stringify(ledgerQuery.where); return axios.get(`${config.ledger.transactionUrl}?${queryString.stringify(ledgerQuery)}`); From 81782e4f44fec6e6adbada421cdfe48fecfcfd7c Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Wed, 12 Dec 2018 07:24:20 -0500 Subject: [PATCH 08/10] feat(lib/utils): add method to parse string or numbers to Boolean --- server/lib/utils.js | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/server/lib/utils.js b/server/lib/utils.js index 4e15bbc2460..c2a209e4831 100644 --- a/server/lib/utils.js +++ b/server/lib/utils.js @@ -589,3 +589,15 @@ export function promiseSeq(arr, predicate, consecutive = 100) { }); }, Promise.resolve([])); } + +export function parseToBoolean(value) { + let lowerValue = value; + // check whether it's string + if (lowerValue && (typeof lowerValue === 'string' || lowerValue instanceof String)) { + lowerValue = lowerValue.trim().toLowerCase(); + } + if (['on', 'enabled', '1', 'true', 'yes', 1].includes(lowerValue)) { + return true; + } + return false; +} From 73d15285c29025bf28ada216e476292dc7b99cbe Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Wed, 12 Dec 2018 07:25:37 -0500 Subject: [PATCH 09/10] fix(grapqhl/queries): check whether query has arg fetchDataFromLedger, if so consider it and not the defaults/env vars --- server/graphql/v1/queries.js | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/server/graphql/v1/queries.js b/server/graphql/v1/queries.js index d47a2f59b1e..d8365b2ea85 100644 --- a/server/graphql/v1/queries.js +++ b/server/graphql/v1/queries.js @@ -2,6 +2,7 @@ import Promise from 'bluebird'; import { find, get, uniq } from 'lodash'; import algolia from '../../lib/algolia'; import errors from '../../lib/errors'; +import { parseToBoolean } from '../../lib/utils'; import { fetchLedgerTransactionsGroupedByLegacyIds, parseLedgerTransactions } from '../../lib/ledger'; import { GraphQLList, GraphQLNonNull, GraphQLString, GraphQLInt, GraphQLBoolean } from 'graphql'; @@ -324,8 +325,8 @@ const queries = { whether we should include the transactions of the collectives of that host(if it's a host collective) */, }, async resolve(_, args) { - let fetchDataFromLedger = process.env.GET_TRANSACTIONS_FROM_LEDGER || false; - if (args.fetchDataFromLedger) { + let fetchDataFromLedger = parseToBoolean(process.env.GET_TRANSACTIONS_FROM_LEDGER); + if (args.hasOwnProperty('fetchDataFromLedger')) { fetchDataFromLedger = args.fetchDataFromLedger; } // Load collective From b8f4cbabc9c5bcd29ea55763da83b50320182785 Mon Sep 17 00:00:00 2001 From: marcogbarcellos Date: Wed, 12 Dec 2018 09:54:20 -0500 Subject: [PATCH 10/10] feat(grapqhl/transactionInterface): add comment to explain we validate in case transaction is not a sequelize model --- server/graphql/v1/TransactionInterface.js | 32 ++++++++++++++++++++++- 1 file changed, 31 insertions(+), 1 deletion(-) diff --git a/server/graphql/v1/TransactionInterface.js b/server/graphql/v1/TransactionInterface.js index 4da07757e5a..74f557441bb 100644 --- a/server/graphql/v1/TransactionInterface.js +++ b/server/graphql/v1/TransactionInterface.js @@ -67,6 +67,8 @@ const TransactionFields = () => { refundTransaction: { type: TransactionInterfaceType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getRefundTransaction + // otherwise we just null if (transaction && transaction.getRefundTransaction) { return transaction.getRefundTransaction(); } @@ -79,10 +81,12 @@ const TransactionFields = () => { if (!req.remoteUser) { return null; } + // If it's a sequelize model transaction, it means it has the method getDetailsForUser + // otherwise we return transaction.uuid if (transaction && transaction.getDetailsForUser) { return transaction.getDetailsForUser(req.remoteUser); } - return null; + return transaction.uuid; }, }, type: { @@ -171,6 +175,8 @@ const TransactionFields = () => { createdByUser: { type: UserType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getCreatedByUser + // otherwise we return null if (transaction && transaction.getCreatedByUser) { return transaction.getCreatedByUser(); } @@ -180,6 +186,8 @@ const TransactionFields = () => { fromCollective: { type: CollectiveInterfaceType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getFromCollective + // otherwise we check whether transaction has 'fromCollective.id', if not we return null if (transaction && transaction.getFromCollective) { return transaction.getFromCollective(); } @@ -192,6 +200,8 @@ const TransactionFields = () => { usingVirtualCardFromCollective: { type: CollectiveInterfaceType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getVirtualCardEmitterCollective + // otherwise we find the collective by id if transactions has UsingVirtualCardFromCollectiveId, if not we return null if (transaction && transaction.getVirtualCardEmitterCollective) { return transaction.getVirtualCardEmitterCollective(); } @@ -204,6 +214,8 @@ const TransactionFields = () => { collective: { type: CollectiveInterfaceType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getCollective + // otherwise we check whether transaction has 'collective.id', if not we return null if (transaction && transaction.getCollective) { return transaction.getCollective(); } @@ -246,6 +258,8 @@ export const TransactionExpenseType = new GraphQLObjectType({ description: { type: GraphQLString, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getExpense + // otherwise we return transaction.description , if not then return null const expense = transaction.getExpense ? transaction.getExpense().then(expense => expense && expense.description) : null; @@ -255,6 +269,8 @@ export const TransactionExpenseType = new GraphQLObjectType({ privateMessage: { type: GraphQLString, resolve(transaction, args, req) { + // If it's a sequelize model transaction, it means it has the method getExpenseFromViewer + // otherwise we return null return transaction.getExpenseForViewer ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.privateMessage) : null; @@ -263,6 +279,8 @@ export const TransactionExpenseType = new GraphQLObjectType({ category: { type: GraphQLString, resolve(transaction, args, req) { + // If it's a sequelize model transaction, it means it has the method getExpenseFromViewer + // otherwise we return null return transaction.getExpenseForViewer ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.category) : null; @@ -271,6 +289,8 @@ export const TransactionExpenseType = new GraphQLObjectType({ attachment: { type: GraphQLString, resolve(transaction, args, req) { + // If it's a sequelize model transaction, it means it has the method getExpenseFromViewer + // otherwise we return null return transaction.getExpenseForViewer ? transaction.getExpenseForViewer(req.remoteUser).then(expense => expense && expense.attachment) : null; @@ -290,6 +310,8 @@ export const TransactionOrderType = new GraphQLObjectType({ description: { type: GraphQLString, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getOrder + // otherwise we return either transaction.description or null const getOrder = transaction.getOrder ? transaction.getOrder().then(order => order && order.description) : null; @@ -299,24 +321,32 @@ export const TransactionOrderType = new GraphQLObjectType({ privateMessage: { type: GraphQLString, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getOrder + // otherwise we return null return transaction.getOrder ? transaction.getOrder().then(order => order && order.privateMessage) : null; }, }, publicMessage: { type: GraphQLString, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getOrder + // otherwise we return null return transaction.getOrder ? transaction.getOrder().then(order => order && order.publicMessage) : null; }, }, order: { type: OrderType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getOrder + // otherwise we return null return transaction.getOrder ? transaction.getOrder() : null; }, }, subscription: { type: SubscriptionType, resolve(transaction) { + // If it's a sequelize model transaction, it means it has the method getOrder + // otherwise we return null return transaction.getOrder ? transaction.getOrder().then(order => order && order.getSubscription()) : null; }, },