[Feature] Create script to backload API data for a given date

.github/workflows/ci.yml

@@ -70,8 +70,8 @@ jobs:
       - lint
       - audit_dependencies
       - test
-    if: github.ref == 'refs/heads/develop'
-    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@develop
+    if: github.ref == 'refs/heads/bugfix/backload_ga4_api_data'
+    uses: 18F/analytics-reporter/.github/workflows/deploy.yml@bugfix/backload_ga4_api_data
     with:
       ANALYTICS_GA4_CALL_RETRY_COUNT: ${{ vars.ANALYTICS_GA4_CALL_RETRY_COUNT_DEV }}
       ANALYTICS_KEY_FILE_NAME: ${{ vars.ANALYTICS_KEY_FILE_NAME }}

deploy/backload.js

@@ -0,0 +1,92 @@
+#!/usr/bin/env node
+
+const fs = require("fs");
+const { program } = require("commander");
+const { formatISO, parseISO, differenceInDays, subDays } = require("date-fns");
+const apiReports = require("../reports/api.json");
+const { exec } = require("child_process");
+const logger = require("../src/logger").initialize();
+
+/**
+ * Script to backload DAP API data for a particular date.
+ *
+ * Example run command:
+ * dotenv -e .env.analytics node -- ./backload.js --date 2024-01-01
+ */
+
+program
+  .option("-d, --date <string>", "date to run the backload for in ISO format")
+  .option(
+    "-e, --env <string>",
+    "dotenv file to load for the process",
+    ".env.analytics",
+  );
+
+program.parse();
+
+const options = program.opts();
+require("dotenv").config({ path: options.env });
+
+const scriptRootPath = `${process.env.ANALYTICS_ROOT_PATH}/deploy`;
+
+const runScriptWithLogName = async (scriptPath, scriptLoggingName) => {
+  logger.info(`Beginning: ${scriptLoggingName}`);
+  logger.info(`File path: ${scriptPath}`);
+  const promise = new Promise((resolve) => {
+    const childProcess = exec(scriptPath);
+    childProcess.stdout.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.stderr.on("data", (data) => {
+      console.log(data.toString().trim());
+    });
+
+    childProcess.on("close", () => {
+      resolve();
+    });
+  });
+
+  await promise;
+  fs.unlinkSync("reports/api.json");
+  fs.renameSync("reports/api.original.json", "reports/api.json");
+};
+
+const api_run = () => {
+  return runScriptWithLogName(`${scriptRootPath}/api.sh`, "api.sh");
+};
+
+(async () => {
+  logger.info("===========================================");
+  logger.info("======= STARTING ANALYTICS-REPORTER =======");
+  logger.info(`Running /deploy/backload.js for: ${options.date}`);
+  logger.info("===========================================");
+
+  const scriptTargetDate = parseISO(options.date);
+  const today = new Date();
+  let totalDaysAgo = differenceInDays(today, scriptTargetDate);
+
+  while (totalDaysAgo >= 1) {
+    const iterationTargetDate = subDays(new Date(), totalDaysAgo);
+
+    const iterationDaysAgo = differenceInDays(today, iterationTargetDate) + 1;
+    const modifiedApiJsonString = JSON.stringify(apiReports).replaceAll(
+      '"yesterday"',
+      `"${iterationDaysAgo}daysAgo"`,
+    );
+
+    fs.unlinkSync("reports/api.json");
+    fs.writeFileSync("reports/api.original.json", JSON.stringify(apiReports));
+    fs.writeFileSync("reports/api.json", modifiedApiJsonString);
+    fs.writeFileSync("reports/api.new.json", modifiedApiJsonString);
+
+    logger.info(
+      `Running API reports for ${formatISO(iterationTargetDate, { representation: "date" })}...`,
+    );
+    await api_run();
+    totalDaysAgo = totalDaysAgo - 1;
+    logger.info(
+      `API reports for ${formatISO(iterationTargetDate, { representation: "date" })} complete`,
+    );
+  }
+})();

index.js

@@ -126,6 +126,7 @@ async function runQueuePublish(options = {}) {
 
   for (const agency of agencies) {
     for (const reportConfig of reportConfigs) {
+      agency.agencyName = agency.agencyName || "";
       process.env.AGENCY_NAME = agency.agencyName;
       const reportLogger = Logger.initialize({
         agencyName: appConfig.agencyLogName,
@@ -146,12 +147,12 @@ async function runQueuePublish(options = {}) {
             retryLimit: 2,
             retryDelay: 10,
             retryBackoff: true,
-            singletonKey: `${appConfig.scriptName}-${agency.agencyName}-${reportConfig.name}`,
+            singletonKey: `${appConfig.scriptName}-${agency.agencyName}-${reportConfig.name}-${reportConfig.query.dateRanges[0].startDate}`,
           },
         );
         if (jobId) {
           reportLogger.info(
-            `Created job in queue: ${queue} with job ID: ${jobId}`,
+            `Created job in queue: ${queue} with job ID: ${jobId} for ${reportConfig.query.dateRanges[0].startDate}`,
           );
         } else {
           reportLogger.info(`Found a duplicate job in queue: ${queue}`);
@@ -241,7 +242,9 @@ async function runQueueConsume() {
       queue,
       { newJobCheckIntervalSeconds: 1 },
       async (message) => {
-        appLogger.info("Queue message received");
+        appLogger.info(
+          `Queue message received for ${message.data.reportConfig.query.dateRanges[0].startDate}`,
+        );
         process.env.AGENCY_NAME = message.data.agencyName;
         process.env.ANALYTICS_REPORT_IDS = message.data.analyticsReportIds;
         process.env.AWS_BUCKET_PATH = message.data.awsBucketPath;

manifest.publisher.yml

@@ -9,7 +9,7 @@ applications:
   health-check-type: process
   buildpacks:
   - nodejs_buildpack
-  command: node deploy/cron.js
+  command: (node deploy/backload.js --date 2024-12-02 && echo SUCCESS || echo FAIL) && sleep infinity
   env:
     ANALYTICS_DEBUG: 'true'
     ANALYTICS_LOG_LEVEL: ${ANALYTICS_LOG_LEVEL}

package.json

@@ -73,20 +73,22 @@
     "@google-analytics/data": "^4.7.0",
     "@smithy/node-http-handler": "^3.0.0",
     "@snyk/protect": "^1.1269.0",
+    "dotenv": "^16.4.5",
     "fast-csv": "^4.3.6",
     "googleapis": "^140.0.0",
     "max-listeners-exceeded-warning": "^0.0.1",
     "minimist": "^1.2.8",
     "p-retry": "^6.2.0",
     "pg-boss": "^9.0.3",
     "proxy-agent": "^6.4.0",
-    "winston": "^3.11.0"
+    "winston": "^3.11.0",
+    "commander": "^12.1.0",
+    "date-fns": "^3.6.0"
   },
   "devDependencies": {
     "@cucumber/cucumber": "^10.3.1",
     "@eslint/js": "^8.57.0",
     "chai": "^4.4.0",
-    "dotenv": "^16.4.5",
     "dotenv-cli": "^7.4.3",
     "eslint": "^8.56.0",
     "eslint-config-prettier": "^9.1.0",

reports/api.json

@@ -19,7 +19,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -67,7 +67,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -115,7 +115,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -163,7 +163,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -211,7 +211,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -267,7 +267,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -318,7 +318,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -369,7 +369,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -425,7 +425,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -477,7 +477,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -525,7 +525,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -584,7 +584,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],
@@ -642,7 +642,7 @@
         ],
         "dateRanges": [
           {
-            "startDate": "3daysAgo",
+            "startDate": "yesterday",
             "endDate": "yesterday"
           }
         ],