Generate DAX batch

.gitignore

@@ -4,3 +4,4 @@ helm/**/*.lock
 *.tgz
 .vscode
 .env
+*.csv

app/batching/allocate-to-batches.js

@@ -0,0 +1,98 @@
+const db = require('../data')
+const config = require('../config')
+
+const allocateToBatches = async (created = new Date()) => {
+  const transaction = await db.sequelize.transaction()
+  try {
+    const apPaymentRequests = await getPendingPaymentRequests('AP', transaction)
+    const arPaymentRequests = await getPendingPaymentRequests('AR', transaction)
+    if (apPaymentRequests.length) {
+      await allocateToBatch(apPaymentRequests, 'AP', created, transaction)
+    }
+    if (arPaymentRequests.length) {
+      await allocateToBatch(arPaymentRequests, 'AR', created, transaction)
+    }
+    await transaction.commit()
+  } catch (error) {
+    await transaction.rollback()
+    throw (error)
+  }
+}
+
+const getPendingPaymentRequests = async (ledger, transaction) => {
+  return db.scheme.findAll({
+    transaction,
+    include: [{
+      model: db.completedPaymentRequest,
+      as: 'completedPaymentRequests',
+      required: true,
+      include: [{
+        model: db.completedInvoiceLine,
+        as: 'invoiceLines',
+        required: true
+      }],
+      where: {
+        ledger,
+        batchId: null
+      },
+      order: ['completedPaymentRequestId'],
+      limit: config.batchSize
+    }]
+  })
+}
+
+const allocateToBatch = async (schemes, ledger, created, transaction) => {
+  for (const scheme of schemes) {
+    if (scheme.completedPaymentRequests.length) {
+      const sequence = await getAndIncrementSequence(scheme.schemeId, ledger, transaction)
+      const batch = await createNewBatch(scheme.schemeId, ledger, sequence, created, transaction)
+      await updatePaymentRequests(scheme, batch.batchId, transaction)
+    }
+  }
+}
+
+const getAndIncrementSequence = async (schemeId, ledger, transaction) => {
+  const sequence = await getSequence(schemeId, transaction)
+  let nextSequence
+  if (ledger === 'AP') {
+    nextSequence = sequence.nextAP
+    sequence.nextAP = sequence.nextAP + 1
+    await updateSequence(sequence, transaction)
+    return nextSequence
+  }
+  nextSequence = sequence.nextAR
+  sequence.nextAR = sequence.nextAR + 1
+  await updateSequence(sequence, transaction)
+  return nextSequence
+}
+
+const getSequence = async (schemeId, transaction) => {
+  return db.sequence.findByPk(schemeId, { transaction })
+}
+
+const updateSequence = async (sequence, transaction) => {
+  await db.sequence.update({
+    nextAP: sequence.nextAP,
+    nextAR: sequence.nextAR
+  }, {
+    where: { schemeId: sequence.schemeId },
+    transaction
+  })
+}
+
+const createNewBatch = async (schemeId, ledger, sequence, created, transaction) => {
+  return db.batch.create({ schemeId, ledger, sequence, created }, { transaction })
+}
+
+const updatePaymentRequests = async (scheme, batchId, transaction) => {
+  for (const paymentRequest of scheme.completedPaymentRequests) {
+    await db.completedPaymentRequest.update({ batchId }, {
+      where: {
+        completedPaymentRequestId: paymentRequest.completedPaymentRequestId
+      },
+      transaction
+    })
+  }
+}
+
+module.exports = allocateToBatches

app/batching/complete-batch.js

@@ -0,0 +1,18 @@
+const db = require('../data')
+
+const completeBatch = async (batchId) => {
+  const transaction = await db.sequelize.transaction()
+  try {
+    const batch = await db.batch.findByPk(batchId, { transaction })
+    // Check if completed already in case of duplicate processing
+    if (batch.published === null) {
+      await db.batch.update({ published: new Date() }, { where: { batchId }, transaction })
+    }
+    await transaction.commit()
+  } catch (error) {
+    await transaction.rollback()
+    throw (error)
+  }
+}
+
+module.exports = completeBatch

app/batching/generate-batches.js

@@ -0,0 +1,19 @@
+const allocateToBatches = require('./allocate-to-batches')
+const getBatches = require('./get-batches')
+const getFileName = require('./get-filename')
+const getContent = require('./get-content')
+const publishBatch = require('./publish-batch')
+const completeBatch = require('./complete-batch')
+
+const generateBatches = async () => {
+  await allocateToBatches()
+  const batches = await getBatches()
+  for (const batch of batches) {
+    const filename = getFileName(batch)
+    const content = getContent(batch)
+    await publishBatch(batch, filename, content)
+    await completeBatch(batch.batchId)
+  }
+}
+
+module.exports = generateBatches

app/batching/get-batches.js

@@ -0,0 +1,64 @@
+const db = require('../data')
+const moment = require('moment')
+const config = require('../config')
+
+const getBatches = async (started = new Date()) => {
+  const transaction = await db.sequelize.transaction()
+  try {
+    const batches = await getPendingBatches(started, transaction)
+    await updateStarted(batches, started, transaction)
+    await transaction.commit()
+    return batches
+  } catch (error) {
+    await transaction.rollback()
+    throw (error)
+  }
+}
+
+const getPendingBatches = async (started, transaction) => {
+  return db.batch.findAll({
+    transaction,
+    limit: config.batchCap,
+    order: ['sequence'],
+    include: [{
+      model: db.scheme,
+      as: 'scheme',
+      required: true,
+      include: [{
+        model: db.batchProperties,
+        as: 'batchProperties',
+        required: true
+      }]
+    }, {
+      model: db.completedPaymentRequest,
+      as: 'paymentRequests',
+      required: true,
+      include: [{
+        model: db.completedInvoiceLine,
+        as: 'invoiceLines',
+        required: true
+      }]
+    }],
+    where: {
+      published: null,
+      [db.Sequelize.Op.or]: [{
+        started: null
+      }, {
+        started: { [db.Sequelize.Op.lte]: moment(started).subtract(5, 'minutes').toDate() }
+      }]
+    }
+  })
+}
+
+const updateStarted = async (batches, started, transaction) => {
+  for (const batch of batches) {
+    await db.batch.update({ started }, {
+      where: {
+        batchId: batch.batchId
+      },
+      transaction
+    })
+  }
+}
+
+module.exports = getBatches

app/batching/get-content.js

@@ -0,0 +1,105 @@
+const moment = require('moment')
+const { convertToPounds } = require('../currency-convert')
+
+const getContent = (batch) => {
+  const rows = []
+  for (const paymentRequest of batch.paymentRequests) {
+    const vendorGroups = getVendorGroups(paymentRequest.invoiceLines)
+    for (const vendorGroup of vendorGroups) {
+      const vendor = getVendorLine(paymentRequest, vendorGroup, batch)
+      rows.push(vendor)
+      for (const invoiceLine of vendorGroup.invoiceLines) {
+        const ledger = getLedgerLine(invoiceLine, paymentRequest)
+        rows.push(ledger)
+      }
+    }
+  }
+
+  return rows
+}
+
+const getVendorGroups = (invoiceLines) => {
+  return [...invoiceLines.reduce((x, y) => {
+    // group by scheme and fund, so create key representing the combination
+    const key = `${y.schemeCode}-${y.fundCode}`
+
+    // if key doesn't exist then first instance so create new group
+    const item = x.get(key) || Object.assign({}, { fundCode: y.fundCode, schemeCode: y.schemeCode, value: 0, invoiceLines: [] })
+    item.value += Number(y.value)
+    item.invoiceLines.push(y)
+
+    return x.set(key, item)
+  }, new Map()).values()]
+}
+
+const getVendorLine = (paymentRequest, vendorGroup, batch) => {
+  return [
+    'Vendor',
+    paymentRequest.frn,
+    '',
+    vendorGroup.fundCode,
+    vendorGroup.schemeCode,
+    paymentRequest.marketingYear,
+    paymentRequest.deliveryBody,
+    paymentRequest.invoiceNumber,
+    convertToPounds(vendorGroup.value),
+    paymentRequest.currency,
+    'legacy',
+    '',
+    paymentRequest.contractNumber,
+    '',
+    1,
+    '',
+    '',
+    '',
+    '',
+    `BACS_${paymentRequest.currency}`,
+    batch.scheme.batchProperties.source,
+    '',
+    '',
+    '',
+    moment(paymentRequest.dueDate).format('DD[/]MM[/]YYYY'),
+    paymentRequest.currency,
+    '',
+    '',
+    paymentRequest.schedule,
+    'END'
+  ]
+}
+
+const getLedgerLine = (invoiceLine, paymentRequest) => {
+  return [
+    'Ledger',
+    invoiceLine.accountCode,
+    '',
+    invoiceLine.fundCode,
+    invoiceLine.schemeCode,
+    paymentRequest.marketingYear,
+    paymentRequest.deliveryBody,
+    paymentRequest.invoiceNumber,
+    convertToPounds(invoiceLine.value),
+    paymentRequest.currency,
+    'legacy',
+    '',
+    '',
+    '',
+    '',
+    '',
+    '',
+    invoiceLine.description,
+    '',
+    '',
+    '',
+    '',
+    '',
+    '',
+    '',
+    '',
+    '',
+    paymentRequest.agreementNumber,
+    '',
+    'END'
+  ]
+}
+
+module.exports = getContent

app/batching/get-filename.js

@@ -0,0 +1,7 @@
+const moment = require('moment')
+
+const getFileName = (batch) => {
+  return `${batch.scheme.batchProperties.prefix}${batch.sequence.toString().padStart(4, '0')}_${batch.ledger}_${moment().format('YYYYMMDDHHmmss')}${batch.scheme.batchProperties.suffix}.csv`
+}
+
+module.exports = getFileName

app/batching/index.js

@@ -0,0 +1,16 @@
+const generateBatches = require('./generate-batches')
+const config = require('../config')
+
+const start = async () => {
+  try {
+    await generateBatches()
+  } catch (err) {
+    console.error(err)
+  } finally {
+    setTimeout(start, config.batchGenerationInterval)
+  }
+}
+
+module.exports = {
+  start
+}

app/batching/publish-batch.js

@@ -0,0 +1,7 @@
+const publishBatch = async (batch, filename, content) => {
+  console.info(`Publishing ${batch.scheme.name} ${batch.ledger} batch ${batch.sequence}`)
+  console.log(`Filename: ${filename}`)
+  console.log(`content: ${JSON.stringify(content)}`)
+}
+
+module.exports = publishBatch

app/config/index.js

@@ -7,14 +7,20 @@ const { development, production, test } = require('./constants').environments
 const schema = Joi.object({
   env: Joi.string().valid(development, test, production).default(development),
   paymentProcessingInterval: Joi.number().default(1000),
-  processingBatchSize: Joi.number().default(1000)
+  processingCap: Joi.number().default(1000),
+  batchGenerationInterval: Joi.number().default(5000),
+  batchSize: Joi.number().default(10000),
+  batchCap: Joi.number().default(100)
 })
 
 // Build config
 const config = {
   env: process.env.NODE_ENV,
   paymentProcessingInterval: process.env.PROCESSING_INTERVAL,
-  processingBatchSize: process.env.PROCESSING_BATCH_SIZE
+  processingCap: process.env.PROCESSING_CAP,
+  batchGenerationInterval: process.env.BATCH_INTERVAL,
+  batchSize: process.env.BATCH_SIZE,
+  batchCap: process.env.BATCH_CAP
 }
 
 // Validate config

app/currency-convert/index.js

@@ -0,0 +1,15 @@
+const convertToPence = (valueInPounds) => {
+  const currencyArray = valueInPounds.toString().split('.')
+  const pounds = currencyArray[0]
+  const pence = (currencyArray[1] || '00').padEnd(2, '0')
+  return Number(pounds + pence)
+}
+
+const convertToPounds = (valueInPence) => {
+  return (valueInPence / 100).toFixed(2)
+}
+
+module.exports = {
+  convertToPence,
+  convertToPounds
+}