refactor: ♻️ 🔏 🔊 🧑‍💻 Add Environment Variables and Convert Logging to…

… Use Logwatch (#119) * dev: 🧑‍💻 🔐 add env variables for logwatch and validator * refactor: ♻️ 🔊 replace consola with logwatch * doc: 📝 update CHANGELOG
fairdataihub · Dec 12, 2024 · f10eae4 · f10eae4
1 parent b7f37c0
commit f10eae4
Show file tree

Hide file tree

Showing 15 changed files with 149 additions and 124 deletions.
diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml
@@ -26,6 +26,7 @@ jobs:
       KAMAL_SERVER_IP: ${{ secrets.KAMAL_SERVER_IP }}
       KAMAL_BOT_DOMAIN: ${{ secrets.KAMAL_BOT_DOMAIN }}
       DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      BOT_LOGWATCH_URL: ${{ secrets.BOT_LOGWATCH_URL }}
       APP_ID: ${{ secrets.APP_ID }}
       GH_APP_ID: ${{ secrets.GH_APP_ID }}
       GH_APP_NAME: ${{ secrets.GH_APP_NAME }}
@@ -36,6 +37,7 @@ jobs:
       CODEFAIR_APP_DOMAIN: ${{ secrets.CODEFAIR_APP_DOMAIN }}
       ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }}
       ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }}
+      VALIDATOR_URL: ${{ secrets.VALIDATOR_URL }}
 
     steps:
       - uses: actions/checkout@v4
@@ -99,6 +101,7 @@ jobs:
       GH_OAUTH_APP_ID: ${{ secrets.GH_OAUTH_APP_ID }}
       GH_OAUTH_CLIENT_ID: ${{ secrets.GH_OAUTH_CLIENT_ID }}
       GH_OAUTH_CLIENT_SECRET: ${{ secrets.GH_OAUTH_CLIENT_SECRET }}
+      UI_LOGWATCH_URL: ${{ secrets.UI_LOGWATCH_URL }}
       ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }}
       ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }}
       ZENODO_CLIENT_ID: ${{ secrets.ZENODO_CLIENT_ID }}

diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml
@@ -26,6 +26,7 @@ jobs:
       KAMAL_SERVER_IP: ${{ secrets.KAMAL_SERVER_IP }}
       KAMAL_BOT_DOMAIN: ${{ secrets.KAMAL_BOT_DOMAIN }}
       DATABASE_URL: ${{ secrets.DATABASE_URL }}
+      BOT_LOGWATCH_URL: ${{ secrets.BOT_LOGWATCH_URL }}
       APP_ID: ${{ secrets.APP_ID }}
       GH_APP_ID: ${{ secrets.GH_APP_ID }}
       GH_APP_NAME: ${{ secrets.GH_APP_NAME }}
@@ -36,6 +37,7 @@ jobs:
       CODEFAIR_APP_DOMAIN: ${{ secrets.CODEFAIR_APP_DOMAIN }}
       ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }}
       ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }}
+      VALIDATOR_URL: ${{ secrets.VALIDATOR_URL }}
 
     steps:
       - uses: actions/checkout@v4
@@ -99,6 +101,7 @@ jobs:
       GH_OAUTH_APP_ID: ${{ secrets.GH_OAUTH_APP_ID }}
       GH_OAUTH_CLIENT_ID: ${{ secrets.GH_OAUTH_CLIENT_ID }}
       GH_OAUTH_CLIENT_SECRET: ${{ secrets.GH_OAUTH_CLIENT_SECRET }}
+      UI_LOGWATCH_URL: ${{ secrets.UI_LOGWATCH_URL }}
       ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }}
       ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }}
       ZENODO_CLIENT_ID: ${{ secrets.ZENODO_CLIENT_ID }}

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -5,6 +5,17 @@ All notable changes the Codefair App will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## v3.2.1 - 12-12-2024
+
+### Added
+
+- Convert logging from 'consola' to 'logwatch' for improved log management and consistency across the application.
+
+### Fixed
+
+- Update CI and deployment workflows to include new environment variables 'BOT_LOGWATCH_URL' and 'VALIDATOR_URL'.
+- Patch to Zenodo workflow that was causing the user to be notified of a failed Zenodo upload when the upload was successful.
+
 ## v3.2.0 - 12-10-2024
 
 ### Added

diff --git a/bot/.kamal/secrets b/bot/.kamal/secrets
@@ -15,6 +15,7 @@ KAMAL_SERVER_IP=$KAMAL_SERVER_IP
 # bot secrets
 APP_ID=$APP_ID
 DATABASE_URL=$DATABASE_URL
+BOT_LOGWATCH_URL=$BOT_LOGWATCH_URL
 GH_APP_ID=$GH_APP_ID
 GH_APP_NAME=$GH_APP_NAME
 GH_APP_CLIENT_ID=$GH_APP_CLIENT_ID
@@ -24,6 +25,7 @@ WEBHOOK_SECRET=$WEBHOOK_SECRET
 CODEFAIR_APP_DOMAIN=$CODEFAIR_APP_DOMAIN
 ZENODO_API_ENDPOINT=$ZENODO_API_ENDPOINT
 ZENODO_ENDPOINT=$ZENODO_ENDPOINT
+VALIDATOR_URL=$VALIDATOR_URL
 
 # Option 2: Read secrets via a command
 # RAILS_MASTER_KEY=$(cat config/master.key)

diff --git a/bot/archival/index.js b/bot/archival/index.js
@@ -1,5 +1,6 @@
 import dbInstance from '../db.js';
 import { consola } from 'consola';
+import { logwatch } from '../utils/logwatch.js';
 import fs from 'fs';
 const licensesJson = JSON.parse(fs.readFileSync('./public/assets/data/licenses.json', 'utf8'));
 
@@ -20,7 +21,7 @@ export async function updateGitHubRelease(context, repositoryName, owner, releas
       release_id: releaseId,
       draft: false,
     });
-    consola.success("Updated release to not be a draft!");
+    logwatch.success("Updated release to not be a draft!");
   } catch (error) {
     throw new Error(`Error updating the GitHub release: ${error}`, { cause: error });
   }
@@ -33,7 +34,7 @@ export async function updateGitHubRelease(context, repositoryName, owner, releas
  */
 export async function publishZenodoDeposition(zenodoToken, depositionId) {
   try {
-    consola.start("Publishing the Zenodo deposition...", depositionId);
+    logwatch.start(`Publishing the Zenodo deposition: ${depositionId}`);
     const publishDeposition = await fetch(
       `${ZENODO_API_ENDPOINT}/deposit/depositions/${depositionId}/actions/publish`,
       {
@@ -50,7 +51,7 @@ export async function publishZenodoDeposition(zenodoToken, depositionId) {
     }
 
     const publishedDeposition = await publishDeposition.json();
-    consola.success("Zenodo deposition published successfully at:", publishedDeposition.links.latest_html);
+    logwatch.success(`Zenodo deposition published successfully at: ${publishedDeposition.links.latest_html}`);
   } catch (error) {
     throw new Error(`Error publishing the Zenodo deposition: ${error.message}`, { cause: error });
   }
@@ -251,7 +252,7 @@ export async function createNewVersionOfDeposition(zenodoToken, depositionId) {
       const errorText = await zenodoRecord.text();
       throw new Error(`Failed to create a new version of Zenodo deposition. Status: ${zenodoRecord.status}: ${zenodoRecord.statusText}.`, { cause: errorText});
     }
-    consola.success("New version of Zenodo deposition created successfully!");
+    logwatch.success("New version of Zenodo deposition created successfully!");
 
     const responseText = await zenodoRecord.json();
 
@@ -265,7 +266,7 @@ export async function createNewVersionOfDeposition(zenodoToken, depositionId) {
     });
 
     if (!draftZenodoRecord.ok) {
-      consola.error("Error fetching the latest draft of Zenodo deposition:", draftZenodoRecord);
+      logwatch.error({message: "Error fetching the latest draft of Zenodo deposition:", fetchReponse: draftZenodoRecord}, true);
       const errorText = await draftZenodoRecord.text();
       throw new Error(`Failed to fetch the latest draft of Zenodo deposition. Status: ${draftZenodoRecord.status}: ${draftZenodoRecord.statusText}. Error: ${errorText}`, { cause: errorText });
     }
@@ -295,7 +296,7 @@ export async function getZenodoDepositionInfo(
 
     if (zenodoDepositionInfo.submitted === false){
       // Delete the files in the draft
-      consola.start("Requested deposition is a draft. Deleting the files in the draft...");
+      logwatch.start("Requested deposition is a draft. Deleting the files in the draft...");
       for (const file of zenodoDepositionInfo.files) {
         await deleteFileFromZenodo(depositionId, zenodoToken, file.id);
       }
@@ -307,12 +308,12 @@ export async function getZenodoDepositionInfo(
 
     if (newZenodoVersion.files.length > 0) {
       for (const file of newZenodoVersion.files) {
-        consola.start("Deleting file from newly created draft:", file.links.download);
+        logwatch.start(`Deleting file from newly created draft: ${file.links.download}`);
         await deleteFileFromZenodo(newZenodoVersion.id, zenodoToken, file.id);
       }
     }
 
-    consola.success("New draft version of Zenodo deposition created successfully!");
+    logwatch.success("New draft version of Zenodo deposition created successfully!");
     return newZenodoVersion;
   }
 }
@@ -354,8 +355,8 @@ export async function createZenodoMetadata(codemetadata, repository) {
     });
     if (!codeMetaContent.license) {
       // fetch from the db
-      consola.warn(`No license found in the codemeta.json file. Fetching from the database...`);
-      consola.info(`License found in the database: ${existingLicense?.license_id}`);
+      logwatch.warn(`No license found in the codemeta.json file. Fetching from the database...`);
+      logwatch.info(`License found in the database: ${existingLicense?.license_id}`);
       codeMetaContent.license = `https://spdx.org/licenses/${existingLicense?.license_id}`;
     }
     const license = licensesJson.find((license) => license.detailsUrl === `${codeMetaContent.license}.json`);
@@ -372,7 +373,7 @@ export async function createZenodoMetadata(codemetadata, repository) {
     })
 
     if (!zenodoMetadata) {
-      consola.error("Zenodo metadata not found in the database. Please create a new Zenodo deposition.");
+      logwatch.error("Zenodo metadata not found in the database. Please create a new Zenodo deposition.");
       throw new Error("Zenodo metadata not found in the database. Please create a new Zenodo deposition.");
     }
 
@@ -440,7 +441,7 @@ export async function updateZenodoMetadata(depositionId, zenodoToken, metadata)
     );
 
     const updatedMetadataInfo = await updatedMetadata.json();
-    consola.success("Zenodo deposition metadata updated successfully!");
+    logwatch.success("Zenodo deposition metadata updated successfully!");
     return updatedMetadataInfo;
   } catch (error) {
     throw new Error(`Error updating Zenodo metadata: ${error}`, { cause: error });
@@ -480,7 +481,7 @@ export async function uploadReleaseAssetsToZenodo(
             accept: 'application/octet-stream'
           }
         });
-        consola.success(`Asset data fetched for ${asset.name}, for the release ${tagVersion}, from the GitHub repository: ${repository.name}`);
+        logwatch.success(`Asset data fetched for ${asset.name}, for the release ${tagVersion}, from the GitHub repository: ${repository.name}`);
 
         // Upload the file to Zenodo
         const uploadAsset = await fetch(`${bucket_url}/${asset.name}`,
@@ -493,9 +494,9 @@ export async function uploadReleaseAssetsToZenodo(
         });
 
         if (!uploadAsset.ok) {
-          consola.error(`Failed to upload ${asset.name}. Status: ${uploadAsset.statusText}. Error: ${uploadAsset}`);
+          logwatch.error(`Failed to upload ${asset.name}. Status: ${uploadAsset.statusText}. Error: ${uploadAsset}`);
         } else {
-          consola.success(`${asset.name} successfully uploaded to Zenodo!`);
+          logwatch.success(`${asset.name} successfully uploaded to Zenodo!`);
         }
       } catch (error) {
         throw new Error(`Error uploading assets to Zenodo: ${error}`, { cause: error });
@@ -516,13 +517,13 @@ export async function uploadReleaseAssetsToZenodo(
   );
 
   if (!uploadZip.ok) {
-    consola.error(`Failed to upload zip file. Status: ${uploadZip.statusText}`);
+    logwatch.error(`Failed to upload zip file. Status: ${uploadZip.statusText}`);
     throw new Error(`Failed to upload zip file. Status: ${uploadZip.statusText}`);
   }
 
   const endTime = performance.now();
-  consola.info(`Total duration to upload assets and zip to Zenodo deposition: ${(endTime - startTime) / 1000} seconds`);
-  consola.success("Zip file successfully uploaded to Zenodo!");
+  logwatch.info(`Total duration to upload assets and zip to Zenodo deposition: ${(endTime - startTime) / 1000} seconds`);
+  logwatch.success("Zip file successfully uploaded to Zenodo!");
 }
 
 /**
@@ -543,12 +544,11 @@ export async function deleteFileFromZenodo(depositionId, zenodoToken, fileId) {
     );
 
     if (!deleteFile.ok) {
-      consola.error(deleteFile);
       const errorText = await deleteFile.text();
       throw new Error(`Failed to delete file from Zenodo. Status: ${deleteFile.status}: ${deleteFile.statusText}. Error: ${errorText}`);
     }
 
-    consola.success("File successfully deleted from Zenodo!");
+    logwatch.success("File successfully deleted from Zenodo!");
   } catch (error) {
     throw new Error(`Error deleting file from Zenodo: ${error}`, { cause: error });
   }

diff --git a/bot/config/deploy.yml b/bot/config/deploy.yml
@@ -54,6 +54,8 @@ env:
     - CODEFAIR_APP_DOMAIN
     - ZENODO_API_ENDPOINT
     - ZENODO_ENDPOINT
+    - VALIDATOR_URL
+    - BOT_LOGWATCH_URL
   clear:
     NODE_ENV: production
 

diff --git a/bot/cwl/index.js b/bot/cwl/index.js
@@ -2,6 +2,7 @@
  * * This file contains the functions to interact with the CWL files in the repository
  */
 import { consola } from "consola";
+import { logwatch } from "../utils/logwatch.js";
 import {
   isRepoPrivate,
   createId,
@@ -10,6 +11,7 @@ import {
 import dbInstance from "../db.js";
 
 const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN;
+const { VALIDATOR_URL } = process.env;
 
 /**
  * * This function gets the CWL files in the repository
@@ -20,7 +22,7 @@ const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN;
  */
 export function getCWLFiles(context, owner, repository) {
   return new Promise((resolve, reject) => {
-    consola.info("Checking for CWL files in the repository...");
+    logwatch.info("Checking for CWL files in the repository...");
 
     const cwlFiles = [];
     const cwlObject = {
@@ -51,9 +53,12 @@ export function getCWLFiles(context, owner, repository) {
           resolve(cwlObject);
           return;
         }
-        consola.error(
-          "Error finding CWL files throughout the repository:",
-          error,
+        logwatch.error(
+          {
+            message: "Error finding CWL files throughout the repository:",
+            error,
+          },
+          true
         );
         reject(error);
       }
@@ -93,7 +98,7 @@ export function getCWLFiles(context, owner, repository) {
  */
 export async function validateCWLFile(downloadUrl) {
   try {
-    const response = await fetch("https://cwl-validate.codefair.io/validate-cwl", {
+    const response = await fetch(`${VALIDATOR_URL}/validate-cwl`, {
       body: JSON.stringify({
         file_path: downloadUrl,
       }),
@@ -108,15 +113,15 @@ export async function validateCWLFile(downloadUrl) {
       return [false, error.error];
     }
     if (!response.ok && response.status === 500) {
-      consola.error("Error validating CWL file:", response);
+      logwatch.error({message: "Error validating CWL file:", validation_response: response}, true);
       return [false, "Error validating CWL file"];
     }
     if (response.ok) {
       const data = await response.json();
       return [true, data.output];
     }
   } catch (e) {
-    consola.error("Error validating CWL file:", e);
+    logwatch.error({message: "Error validating CWL file:", error: e}, true);
     return [false, "Error validating CWL file"];
   }
 }
@@ -180,14 +185,14 @@ export async function applyCWLTemplate(
   });
 
   if (subjects.cwl.files.length === 0) {
-    consola.warn(
-      `No new/modified CWL files found in the repository, ${repository.name}`,
+    logwatch.warn(
+      `No CWL files found in the repository, ${repository.name}`,
     );
   }
 
-  consola.start("Validating CWL files for", repository.name);
+  logwatch.start("Validating CWL files for", repository.name);
   // Validate each CWL file from list\
-  consola.info(`Validating ${JSON.stringify(subjects.cwl)} CWL files`);
+  logwatch.info(`Validating ${JSON.stringify(subjects.cwl)} CWL files`);
   if (subjects.cwl.files.length > 0) {
     for (const file of subjects.cwl.files) {
       const fileSplit = file.name.split(".");
@@ -252,7 +257,7 @@ export async function applyCWLTemplate(
         // Add the file to the table content of the issue dashboard
         tableContent += `| ${file.path} | ${isValidCWL ? "✔️" : "❌"} |\n`;
 
-        consola.success(
+        logwatch.success(
           `File: ${file.path} is ${isValidCWL ? "valid" : "invalid"}`,
         );
       }
@@ -276,7 +281,7 @@ export async function applyCWLTemplate(
     });
 
     if (!cwlFiles.length > 0) {
-      consola.warn(
+      logwatch.warn(
         `No CWL files found in the repository, ${repository.name}, skipping CWL section`,
       );
       return baseTemplate;
@@ -317,15 +322,13 @@ export async function applyCWLTemplate(
 
     if (!newFiles.length > 0) {
       // All CWL files were removed from the repository
-      consola.warn(
-        "All CWL files were removed from:",
-        repository.name,
-        "skipping CWL section",
+      logwatch.warn(
+        `All CWL files were removed from: ${repository.name}, skipping CWL section`
       );
       return baseTemplate;
     } else {
       // Recreate the table content to include the new and old cwl files
-      consola.start(
+      logwatch.start(
         "Recreating the table content for the CWL section to include new and old files",
       );
       tableContent = "";
@@ -349,6 +352,6 @@ export async function applyCWLTemplate(
   const cwlBadge = `[![CWL](https://img.shields.io/badge/View_CWL_Report-0ea5e9.svg)](${url})`;
   baseTemplate += `${overallSection}\n\n### CWL Validations ${validOverall ? "✔️" : "❗"}\n\nCodefair has detected that you are following the Common Workflow Language (CWL) standard to describe your command line tool. Codefair ran the [cwltool validator](https://cwltool.readthedocs.io/en/latest/) and ${validOverall ? `all ***${subjects.cwl.files.length}*** CWL file(s) in your repository are valid.` : `***${failedCount}/${subjects.cwl.files.length}*** CWL file(s) in your repository are not valid.`}\n\n<details>\n<summary>Summary of the validation report</summary>\n\n| File | Validation result |\n| :---- | :----: |\n${tableContent}</details>\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`;
 
-  consola.success("CWL template section applied");
+  logwatch.success("CWL template section applied");
   return baseTemplate;
 }