Merge pull request #130 from GeneralMagicio/saveReportsInMongo

Save reports in mongo

config/example.env

@@ -300,4 +300,8 @@ ANKR_API_KEY_FOR_FUNDING_POT=
 # Sync donations with ankr
 ENABLE_ANKR_SYNC=
 ANKR_RPC_URL=
-ANKR_SYNC_CRONJOB_EXPRESSION=
+ANKR_SYNC_CRONJOB_EXPRESSION=
+
+# Reports database
+MONGO_DB_URI=
+MONGO_DB_REPORT_DB_NAME=

docker-compose-local.yml

@@ -93,10 +93,29 @@ services:
     ports:
       - '6379:6379'
 
+  qacc-mongo:
+    container_name: qacc-mongo
+    image: mongo:latest
+    profiles:
+      - server
+      - database
+      - local
+    restart: always
+    environment:
+      - MONGO_INITDB_ROOT_USERNAME=admin
+      - MONGO_INITDB_ROOT_PASSWORD=password
+    volumes:
+      - mongo-data:/data/db
+    networks:
+      - qacc
+    ports:
+      - '27017:27017'
+
 volumes:
   db-data:
   db-data-test:
   redis-data:
+  mongo-data:
 
 networks:
   qacc:

src/scripts/configs.ts

@@ -19,3 +19,5 @@ export function getReportsSubDir() {
   reportsSubDir += '/output';
   return reportsSubDir;
 }
+
+export const reportFilesDir = path.join(repoLocalDir, getReportsSubDir());

src/scripts/reportService.ts

@@ -0,0 +1,89 @@
+/* eslint-disable no-console */
+import fs from 'fs';
+import path from 'path';
+import { MongoClient } from 'mongodb';
+import config from '../config';
+
+const mongoUri = config.get('MONGO_DB_URI') as string;
+const dbName = config.get('MONGO_DB_REPORT_DB_NAME') as string;
+const collectionName = 'reports';
+
+// Function to save all reports in the MongoDB
+export async function saveReportsToDB(outputFolderPath: string) {
+  const client = new MongoClient(mongoUri);
+  console.info('Connecting to mongo db ...');
+  await client.connect();
+  console.info('Connected to mongo db successfully.');
+  const db = client.db(dbName);
+  const collection = db.collection(collectionName);
+
+  // Traverse the output directory and read files
+  const traverseDirectory = async (dir: string) => {
+    const files = await fs.promises.readdir(dir, { withFileTypes: true });
+
+    for (const file of files) {
+      const filePath = path.join(dir, file.name);
+
+      if (file.isDirectory()) {
+        // Recursively traverse subdirectories
+        await traverseDirectory(filePath);
+      } else if (file.isFile() && path.extname(file.name) === '.json') {
+        // Read the file content
+        const content = await fs.promises.readFile(filePath, 'utf8');
+        const projectPath = path.relative(outputFolderPath, filePath);
+
+        // Upsert file data and relative path to MongoDB
+        await collection.updateOne(
+          { projectPath }, // Query to check for existing record with the same projectPath
+          {
+            $set: {
+              projectPath,
+              content,
+              createdAt: new Date(),
+            },
+          },
+          { upsert: true }, // Insert if no document matches the query
+        );
+
+        console.info(
+          `Saved ${file.name} to MongoDB with path ${path.relative(outputFolderPath, filePath)}`,
+        );
+      }
+    }
+  };
+
+  await traverseDirectory(outputFolderPath);
+  await client.close();
+  console.info('All reports have been saved to MongoDB.');
+}
+
+// Function to retrieve all reports from MongoDB and recreate the folder structure
+export async function restoreReportsFromDB(outputFolderPath: string) {
+  const client = new MongoClient(mongoUri);
+  console.info('Connecting to mongo db ...');
+  await client.connect();
+  console.info('Connected to mongo db successfully.');
+  const db = client.db(dbName);
+  const collection = db.collection(collectionName);
+
+  // Retrieve all reports from MongoDB
+  const reportsCursor = collection.find();
+  const reports = await reportsCursor.toArray();
+
+  // Restore the file structure and save each report
+  for (const report of reports) {
+    const restoredFilePath = path.join(outputFolderPath, report.projectPath);
+
+    // Ensure the directory exists
+    await fs.promises.mkdir(path.dirname(restoredFilePath), {
+      recursive: true,
+    });
+
+    // Write the content to the file
+    await fs.promises.writeFile(restoredFilePath, report.content, 'utf8');
+    console.info(`Restored report to ${restoredFilePath}`);
+  }
+
+  await client.close();
+  console.info('All reports have been restored from MongoDB.');
+}

src/scripts/runFundingPotService.ts

@@ -3,7 +3,7 @@ import { spawn } from 'child_process';
 import path from 'path';
 import fs from 'fs-extra';
 import simpleGit from 'simple-git';
-import { repoLocalDir, repoUrl } from './configs';
+import { repoLocalDir, reportFilesDir, repoUrl } from './configs';
 import config from '../config';
 import { Project } from '../entities/project';
 import { AppDataSource } from '../orm';
@@ -17,6 +17,7 @@ import { EarlyAccessRound } from '../entities/earlyAccessRound';
 import { findAllEarlyAccessRounds } from '../repositories/earlyAccessRoundRepository';
 import { findQfRounds } from '../repositories/qfRoundRepository';
 import { updateRewardsForDonations } from './syncDataWithJsonReport';
+import { restoreReportsFromDB, saveReportsToDB } from './reportService';
 
 // Attention: the configs of batches should be saved in the funding pot repo
 // this script pulls the latest version of funding pot service,
@@ -223,7 +224,13 @@ async function installDependencies() {
 async function runFundingPotService(batchNumber: number) {
   const command = 'npm run all ' + batchNumber;
   console.info(`Running "${command}" in ${serviceDir}...`);
-  await execShellCommand(command, serviceDir);
+  try {
+    await execShellCommand(command, serviceDir);
+  } catch (e) {
+    console.error('Error in funding pot execution:', e);
+  }
+  console.info('Saving reports to the DB...');
+  await saveReportsToDB(reportFilesDir);
 }
 
 async function getFirstRoundThatNeedExecuteBatchMinting() {
@@ -341,16 +348,21 @@ async function main() {
     console.info('Env file created successfully.');
 
     // Step 5
+    console.info('Restoring previous report files...');
+    await restoreReportsFromDB(reportFilesDir);
+    console.info('Previous report files restored successfully!');
+
+    // Step 6
     console.info('Running funding pot service...');
     await runFundingPotService(batchNumber);
     console.info('Funding pot service executed successfully!');
 
-    // Step 6
+    // Step 7
     console.info('Setting batch minting execution flag in round data...');
     await setBatchMintingExecutionFlag(batchNumber);
     console.info('Batch minting execution flag set successfully.');
 
-    // Step 7
+    // Step 8
     console.info('Start Syncing reward data in donations...');
     await updateRewardsForDonations(batchNumber);
     console.info('Rewards data synced successfully.');

src/scripts/syncDataWithJsonReport.ts

@@ -7,7 +7,7 @@ import { Donation } from '../entities/donation';
 import { Project } from '../entities/project';
 import { AppDataSource } from '../orm';
 import { getStreamDetails } from './helpers';
-import { repoLocalDir, getReportsSubDir } from './configs';
+import { reportFilesDir } from './configs';
 
 async function loadReportFile(filePath: string) {
   try {
@@ -137,7 +137,6 @@ export async function updateRewardsForDonations(batchNumber: number) {
 
     const donationsByProjectId = _.groupBy(donations, 'projectId');
 
-    const reportFilesDir = path.join(repoLocalDir, getReportsSubDir());
     const allReportFiles = getAllReportFiles(reportFilesDir);
 
     for (const projectId of Object.keys(donationsByProjectId)) {