Improve transaction handling to accommodate large transaction quantity

README.md

@@ -4,3 +4,5 @@ Packages included in this repo:
 `config` - reads configuration from AWS SSM Parameter Store
 
 `parcs` - XML data handling for NetSuite-ParCS workflows
+
+`ssh` - SSH and SFTP functions

parcs/parcs.go

@@ -104,6 +104,7 @@ type SubsidiaryTransactions struct {
 	Subsidiary   string
 	TotalAmount  int
 	Transactions []Transaction
+	Document     ssh.Document
 }
 
 // PMISBatch is the definition of the top-level object in the XML file output.
@@ -289,29 +290,35 @@ func parseAmount(s string) int {
 	return int(math.Round(f * 100))
 }
 
-func GroupTransactions(transactions []Transaction) ([]SubsidiaryTransactions, error) {
-	groupedTransactions := map[string][]Transaction{}
-	totals := map[string]int{}
+// GroupTransactions segments the transaction list into blocks by subsidiary. No more than maxSize transactions will
+// be included in a single block.
+func GroupTransactions(transactions []Transaction, maxSize int) ([]SubsidiaryTransactions, error) {
+	// grouped is the finished list
+	grouped := make([]SubsidiaryTransactions, 0)
+
+	// subsidiaryLists are slice indexes of the current list for each subsidiary
+	subsidiaryLists := make(map[string]int)
+
 	for _, t := range transactions {
-		totals[t.SubsidiaryExternalID] = totals[t.SubsidiaryExternalID] + t.Amount
-		groupedTransactions[t.SubsidiaryExternalID] = append(groupedTransactions[t.SubsidiaryExternalID], t)
-	}
+		// if there's a list for this transaction's subsidiary, and it's still short enough, add to it
+		if idx, ok := subsidiaryLists[t.SubsidiaryExternalID]; ok {
+			if len(grouped[idx].Transactions) < maxSize {
+				grouped[idx].Transactions = append(grouped[idx].Transactions, t)
+				grouped[idx].TotalAmount += t.Amount
+				continue
+			}
+		}
 
-	totalTransactions := 0
-	t := make([]SubsidiaryTransactions, 0, len(groupedTransactions))
-	for subsidiary := range groupedTransactions {
-		t = append(t, SubsidiaryTransactions{
-			Subsidiary:   subsidiary,
-			TotalAmount:  totals[subsidiary],
-			Transactions: groupedTransactions[subsidiary],
+		// otherwise, make a new list and store its index in subsidiaryLists
+		grouped = append(grouped, SubsidiaryTransactions{
+			Subsidiary:   t.SubsidiaryExternalID,
+			TotalAmount:  t.Amount,
+			Transactions: []Transaction{t},
 		})
-		totalTransactions += len(groupedTransactions[subsidiary])
-	}
-	if len(transactions) != totalTransactions {
-		return nil, fmt.Errorf("total number of transactions in groups is not correct, expected %d, got %d",
-			len(transactions), totalTransactions)
+		subsidiaryLists[t.SubsidiaryExternalID] = len(grouped) - 1
 	}
-	return t, nil
+
+	return grouped, nil
 }
 
 func MarkTransactionsSent(ctx context.Context, transactions []Transaction, cfg Config) error {
@@ -464,29 +471,22 @@ func SendToWorkday(cfg Config, data []ssh.Document) error {
 	return nil
 }
 
-// CreateXMLDocuments converts a list of SubsidiaryTransactions to a map of Documents keyed by subsidiary
-func CreateXMLDocuments(st []SubsidiaryTransactions) (map[string]ssh.Document, error) {
+// CreateXMLDocuments creates an XML Document for each block of transactions in a list of SubsidiaryTransactions.
+func CreateXMLDocuments(st []SubsidiaryTransactions) error {
 	today := time.Now().Format(time.RFC3339)
 
-	docs := make(map[string]ssh.Document)
-	for _, t := range st {
+	for i, t := range st {
 		b, err := createXMLDocument(t)
 		if err != nil {
-			return nil, fmt.Errorf("XML error on %s: %w", t.Subsidiary, err)
+			return fmt.Errorf("XML error on %s: %w", t.Subsidiary, err)
 		}
 
-		doc := ssh.Document{
+		st[i].Document = ssh.Document{
 			Name:    fmt.Sprintf("%s_%s.xml", t.Subsidiary, today),
 			Content: string(b),
 		}
-
-		if _, ok := docs[t.Subsidiary]; ok {
-			return nil, fmt.Errorf("duplicate XML document: %s", t.Subsidiary)
-		}
-
-		docs[t.Subsidiary] = doc
 	}
-	return docs, nil
+	return nil
 }
 
 // createXMLDocument converts a SubsidiaryTransactions to an XMLDocument

parcs/parcs_test.go

@@ -6,6 +6,7 @@ import (
 	"net/http"
 	"net/http/httptest"
 	"reflect"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
@@ -280,38 +281,35 @@ func Test_createXMLDocuments(t *testing.T) {
 			TotalAmount:  cashSale.Amount + cashRefund.Amount,
 			Transactions: []Transaction{cashSale, cashRefund},
 		},
-	}
-
-	want := map[string]ssh.Document{
-		"ABC": {
-			Name:    "ABC",
-			Content: xmlDoc1,
-		},
-		"XYZ": {
-			Name:    "XYZ",
-			Content: xmlDoc2,
+		{
+			// one more transaction for XYZ
+			Subsidiary:   "XYZ",
+			TotalAmount:  cashSale.Amount,
+			Transactions: []Transaction{cashSale},
 		},
 	}
 
-	got, err := CreateXMLDocuments(st)
+	err := CreateXMLDocuments(st)
 	if err != nil {
 		t.Errorf("CreateXMLDocuments() error = %v", err)
 		return
 	}
-	if len(got) != 2 {
-		t.Errorf("expected 2 documents, got %d", len(got))
+	if len(st) != 3 {
+		t.Errorf("expected 3 documents, st %d", len(st))
 	}
-	if !strings.HasPrefix(got["ABC"].Name, "ABC") {
-		t.Errorf("incorrect XML document name, expected the subsidiary code ABC, got: %s", got["ABC"].Name)
+
+	if !strings.HasPrefix(st[0].Document.Name, "ABC") {
+		t.Errorf("incorrect XML document name, expected the subsidiary code ABC, got: %s", st[0].Document.Name)
 	}
-	if !strings.HasPrefix(got["XYZ"].Name, "XYZ") {
-		t.Errorf("incorrect XML document name, expected the subsidiary code XYZ, got: %s", got["XYZ"].Name)
+	if !strings.HasPrefix(st[1].Document.Name, "XYZ") {
+		t.Errorf("incorrect XML document name, expected the subsidiary code XYZ, got: %s", st[1].Document.Name)
 	}
-	if !cmp.Equal(got["ABC"], want["ABC"], cmpopts.IgnoreFields(ssh.Document{}, "Name")) {
-		t.Error("diff:", cmp.Diff(got["ABC"], want["ABC"]))
+
+	if !cmp.Equal(st[0].Document.Content, xmlDoc1, cmpopts.IgnoreFields(ssh.Document{}, "Name")) {
+		t.Error("diff:", cmp.Diff(st[0].Document.Content, xmlDoc1))
 	}
-	if !cmp.Equal(got["XYZ"], want["XYZ"], cmpopts.IgnoreFields(ssh.Document{}, "Name")) {
-		t.Error("diff:", cmp.Diff(got["XYZ"], want["XYZ"]))
+	if !cmp.Equal(st[1].Document.Content, xmlDoc2, cmpopts.IgnoreFields(ssh.Document{}, "Name")) {
+		t.Error("diff:", cmp.Diff(st[1].Document.Content, xmlDoc2))
 	}
 }
 
@@ -788,3 +786,44 @@ type roundTripFunc func(*http.Request) (*http.Response, error)
 func (f roundTripFunc) RoundTrip(r *http.Request) (*http.Response, error) {
 	return f(r)
 }
+
+func Test_GroupTransactions(t *testing.T) {
+	var transactions []Transaction
+	for i := 0; i < 5; i++ {
+		testTransaction := cashSale
+		testTransaction.TranID = strconv.Itoa(i)
+
+		// make two different subsidiary IDs: TS1 and TS2
+		testTransaction.SubsidiaryExternalID = "TS" + strconv.Itoa(i%2+1)
+
+		transactions = append(transactions, testTransaction)
+	}
+
+	// TS1 should be broken into two transaction groups, TS2 should be in one group
+	expected := []SubsidiaryTransactions{
+		{
+			Subsidiary:   "TS1",
+			TotalAmount:  2220,
+			Transactions: []Transaction{transactions[0], transactions[2]},
+		},
+		{
+			Subsidiary:   "TS2",
+			TotalAmount:  2220,
+			Transactions: []Transaction{transactions[1], transactions[3]},
+		},
+		{
+			Subsidiary:   "TS1",
+			TotalAmount:  1110,
+			Transactions: []Transaction{transactions[4]},
+		},
+	}
+
+	got, err := GroupTransactions(transactions, 2)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+
+	if !reflect.DeepEqual(expected, got) {
+		t.Fatalf("expected: %v, got: %v", expected, got)
+	}
+}