Optimize fetch performance, particularly for old zeroed accounts (#21)

idpaterson · web-flow · commit d34b10720887 · 2023-11-17T09:45:00.000-08:00
diff --git a/src/shared/lib/__tests__/accounts.test.ts b/src/shared/lib/__tests__/accounts.test.ts
@@ -1,11 +1,13 @@
 import { TEST_MINT_API_KEY } from '@root/src/shared/lib/constants';
 import {
+  calculateIntervalForAccountHistory,
   fetchAccounts,
   fetchDailyBalancesForAllAccounts,
   fetchMonthlyBalancesForAccount,
   fetchNetWorthBalances,
   formatBalancesAsCSV,
 } from '../accounts';
+import { DateTime } from 'luxon';
 
 describe('fetchMonthlyBalancesForAccount', () => {
   it('fetches balances by date for asset account', async () => {
@@ -77,6 +79,58 @@ describe('formatBalancesAsCSV', () => {
     ]);
     expect(result).toEqual(`"Date","Amount"
 "2020-01-01",""
+`);
+  });
+
+  it('trims trailing zero balances', () => {
+    const result = formatBalancesAsCSV([
+      {
+        amount: 123.45,
+        date: '2020-01-01',
+        type: '',
+      },
+      {
+        amount: 234.56,
+        date: '2020-01-02',
+        type: '',
+      },
+      {
+        amount: 0,
+        date: '2020-01-03',
+        type: '',
+      },
+      {
+        amount: 0,
+        date: '2020-01-04',
+        type: '',
+      },
+    ]);
+    expect(result).toEqual(`"Date","Amount"
+"2020-01-01","123.45"
+"2020-01-02","234.56"
+`);
+  });
+
+  it('leaves one row if all balances are zero', () => {
+    const result = formatBalancesAsCSV([
+      {
+        amount: 0,
+        date: '2020-01-01',
+        type: '',
+      },
+      {
+        amount: 0,
+        date: '2020-01-02',
+        type: '',
+      },
+      {
+        amount: 0,
+        date: '2020-01-03',
+        type: '',
+      },
+    ]);
+    expect(result).toEqual(`"Date","Amount"
+"2020-01-01","0"
 `);
   });
 });
@@ -90,3 +144,63 @@ describe('fetchDailyBalancesForAllAccounts', () => {
     expect(response.length).toBeGreaterThan(0);
   }, 60000);
 });
+
+describe('calculateIntervalForAccountHistory', () => {
+  it('starts at the first day of the first month with history', () => {
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 5, type: '' },
+      { date: '2023-02-28', amount: 10, type: '' },
+    ]);
+    expect(result.start.toISODate()).toBe('2023-01-01');
+  });
+
+  it('ends today for nonzero balances', () => {
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 5, type: '' },
+      { date: '2023-02-28', amount: 10, type: '' },
+    ]);
+    expect(result.end.toISODate()).toBe(DateTime.now().toISODate());
+  });
+
+  it('ends today even if the data goes beyond today', () => {
+    const nextMonth = DateTime.now().plus({ month: 1 }).endOf('month').toISODate();
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 5, type: '' },
+      { date: nextMonth, amount: 10, type: '' },
+    ]);
+    expect(result.end.toISODate()).toBe(DateTime.now().toISODate());
+  });
+
+  it('ends 1 month after the last historic nonzero monthly balance', () => {
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 5, type: '' },
+      { date: '2023-02-28', amount: 10, type: '' },
+      { date: '2023-03-31', amount: 0, type: '' },
+    ]);
+    expect(result.end.toISODate()).toBe('2023-03-31');
+  });
+
+  it('ends 1 month after the last historic nonzero monthly balance', () => {
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 5, type: '' },
+      { date: '2023-02-28', amount: 10, type: '' },
+      { date: '2023-03-31', amount: 0, type: '' },
+      { date: '2023-04-30', amount: 0, type: '' },
+      { date: '2023-05-31', amount: 0, type: '' },
+    ]);
+    expect(result.end.toISODate()).toBe('2023-03-31');
+  });
+
+  it('includes two full months for zero balances', () => {
+    // No need for a special case here, the interval is 2 months because we always add 1 month for
+    // safety to the last month worth including in the report.
+    const result = calculateIntervalForAccountHistory([
+      { date: '2023-01-31', amount: 0, type: '' },
+      { date: '2023-02-28', amount: 0, type: '' },
+      { date: '2023-03-31', amount: 0, type: '' },
+      { date: '2023-04-30', amount: 0, type: '' },
+    ]);
+    expect(result.start.toISODate()).toBe('2023-01-01');
+    expect(result.end.toISODate()).toBe('2023-02-28');
+  });
+});
diff --git a/src/shared/lib/accounts.ts b/src/shared/lib/accounts.ts
@@ -3,6 +3,7 @@ import { DateTime, Interval } from 'luxon';
 import { makeMintApiRequest } from '@root/src/shared/lib/auth';
 import {
   DATE_FILTER_ALL_TIME,
+  MINT_DAILY_TRENDS_MAX_DAYS,
   MINT_HEADERS,
   MINT_RATE_LIMIT_DELAY_MS,
 } from '@root/src/shared/lib/constants';
@@ -75,10 +76,40 @@ export const fetchMonthlyBalancesForAccount = async ({
   }
 };
 
+export const calculateIntervalForAccountHistory = (monthlyBalances: TrendEntry[]) => {
+  const startDate = monthlyBalances[0]?.date;
+
+  if (!startDate) {
+    throw new Error('Unable to determine start date for account history.');
+  }
+
+  // find the last month with a non-zero balance
+  let endDate: string;
+  let monthIndex = monthlyBalances.length - 1;
+  while (monthIndex > 0 && monthlyBalances[monthIndex].amount === 0) {
+    monthIndex -= 1;
+    endDate = monthlyBalances[monthIndex].date;
+  }
+
+  const now = DateTime.now();
+  const approximateRangeEnd = endDate
+    ? // Mint trend months are strange and daily balances may be present after the end of the reported
+      // month (anecodotally observed daily balances 10 days into the first month that showed a zero
+      // monthly balance).
+      DateTime.fromISO(endDate).plus({ month: 1 }).endOf('month')
+    : now;
+
+  // then fetch balances for each period in the range
+  return Interval.fromDateTimes(
+    DateTime.fromISO(startDate).startOf('month'),
+    (approximateRangeEnd < now ? approximateRangeEnd : now).endOf('day'),
+  );
+};
+
 /**
- * Determine earliest date for which account has balance history, and return monthly intervals from then to now.
+ * Determine earliest date for which account has balance history, and return 43 day intervals from then to now.
  */
-const fetchMonthlyIntervalsForAccountHistory = async ({
+const fetchIntervalsForAccountHistory = async ({
   accountId,
   overrideApiKey,
 }: {
@@ -95,35 +126,25 @@ const fetchMonthlyIntervalsForAccountHistory = async ({
   }
 
   const { balancesByDate: monthlyBalances, reportType } = balanceInfo;
-
-  const startDate = monthlyBalances[0]?.date;
-
-  if (!startDate) {
-    throw new Error('Unable to determine start date for account history.');
-  }
-
-  // then fetch balances for each month in range, since that's the only timeframe that the API will return a balance for each day
-  const months = Interval.fromDateTimes(
-    DateTime.fromISO(startDate).startOf('month'),
-    DateTime.local().endOf('month'),
-  ).splitBy({
-    months: 1,
+  const interval = calculateIntervalForAccountHistory(monthlyBalances);
+  const periods = interval.splitBy({
+    days: MINT_DAILY_TRENDS_MAX_DAYS,
   }) as Interval[];
 
-  return { months, reportType };
+  return { periods, reportType };
 };
 
 /**
  * Fetch balance history for each month for an account.
  */
-const fetchDailyBalancesForMonthIntervals = async ({
-  months,
+const fetchDailyBalancesForAccount = async ({
+  periods,
   accountId,
   reportType,
   overrideApiKey,
   onProgress,
 }: {
-  months: Interval[];
+  periods: Interval[];
   accountId: string;
   reportType: string;
   overrideApiKey?: string;
@@ -137,8 +158,8 @@ const fetchDailyBalancesForMonthIntervals = async ({
     count: 0,
   };
 
-  const dailyBalancesByMonth = await withRateLimit({ delayMs: MINT_RATE_LIMIT_DELAY_MS })(
-    months.map(
+  const dailyBalancesByPeriod = await withRateLimit({ delayMs: MINT_RATE_LIMIT_DELAY_MS })(
+    periods.map(
       ({ start, end }) =>
         () =>
           withRetry(() =>
@@ -167,13 +188,13 @@ const fetchDailyBalancesForMonthIntervals = async ({
               )
               .finally(() => {
                 counter.count += 1;
-                onProgress?.({ complete: counter.count, total: months.length });
+                onProgress?.({ complete: counter.count, total: periods.length });
               }),
           ),
     ),
   );
 
-  const balancesByDate = dailyBalancesByMonth.reduce((acc, balances) => acc.concat(balances), []);
+  const balancesByDate = dailyBalancesByPeriod.reduce((acc, balances) => acc.concat(balances), []);
 
   return balancesByDate;
 };
@@ -187,43 +208,43 @@ export const fetchDailyBalancesForAllAccounts = async ({
 }) => {
   const accounts = await withRetry(() => fetchAccounts({ overrideApiKey }));
 
-  // first, fetch the range of months we need to fetch for each account
-  const accountsWithMonthsToFetch = await Promise.all(
+  // first, fetch the range of dates we need to fetch for each account
+  const accountsWithPeriodsToFetch = await Promise.all(
     accounts.map(async ({ id: accountId, name: accountName }) => {
-      const { months, reportType } = await withDefaultOnError({ months: [], reportType: '' })(
-        fetchMonthlyIntervalsForAccountHistory({
+      const { periods, reportType } = await withDefaultOnError({ periods: [], reportType: '' })(
+        fetchIntervalsForAccountHistory({
           accountId,
           overrideApiKey,
         }),
       );
-      return { months, reportType, accountId, accountName };
+      return { periods, reportType, accountId, accountName };
     }),
   );
 
-  // one per account per month
-  const totalRequestsToFetch = accountsWithMonthsToFetch.reduce(
-    (acc, { months }) => acc + months.length,
+  // one per account per 43 day period
+  const totalRequestsToFetch = accountsWithPeriodsToFetch.reduce(
+    (acc, { periods }) => acc + periods.length,
     0,
   );
 
   // fetch one account at a time so we don't hit the rate limit
   const balancesByAccount = await resolveSequential(
-    accountsWithMonthsToFetch.map(
-      ({ accountId, accountName, months, reportType }, accountIndex) =>
+    accountsWithPeriodsToFetch.map(
+      ({ accountId, accountName, periods, reportType }, accountIndex) =>
         async () => {
           const balances = await withDefaultOnError<TrendEntry[]>([])(
-            fetchDailyBalancesForMonthIntervals({
+            fetchDailyBalancesForAccount({
               accountId,
-              months,
+              periods,
               reportType,
               overrideApiKey,
               onProgress: ({ complete }) => {
                 // this is the progress handler for *each* account, so we need to sum up the results before calling onProgress
 
-                const previousAccounts = accountsWithMonthsToFetch.slice(0, accountIndex);
+                const previousAccounts = accountsWithPeriodsToFetch.slice(0, accountIndex);
                 // since accounts are fetched sequentially, we can assume that all previous accounts have completed all their requests
                 const previousCompletedRequestCount = previousAccounts.reduce(
-                  (acc, { months }) => acc + months.length,
+                  (acc, { periods }) => acc + periods.length,
                   0,
                 );
                 const completedRequests = previousCompletedRequestCount + complete;
@@ -344,11 +365,17 @@ export const fetchAccounts = async ({
 
 export const formatBalancesAsCSV = (balances: TrendEntry[], accountName?: string) => {
   const header = ['Date', 'Amount', accountName && 'Account Name'].filter(Boolean);
-  const rows = balances.map(({ date, amount }) => [
-    date,
-    amount,
-    ...(accountName ? [accountName] : []),
-  ]);
+  const maybeAccountColumn: [string?] = accountName ? [accountName] : [];
+  // remove zero balances from the end of the report leaving just the first row if all are zero
+  const rows = balances.reduceRight(
+    (acc, { date, amount }, index) => {
+      if (acc.length || amount !== 0 || index === 0) {
+        acc.unshift([date, amount, ...maybeAccountColumn]);
+      }
+      return acc;
+    },
+    [] as [string, number, string?][],
+  );
 
   return formatCSV([header, ...rows]);
 };
diff --git a/src/shared/lib/constants.ts b/src/shared/lib/constants.ts
@@ -14,3 +14,6 @@ export const UTM_URL_PARAMETERS = {
 
 // we may need to increase this, need to test more
 export const MINT_RATE_LIMIT_DELAY_MS = 50;
+
+// The Mint API returns daily activity when the date range is 43 days or fewer.
+export const MINT_DAILY_TRENDS_MAX_DAYS = 43;
