diff --git a/.github/workflows/deploy-main.yml b/.github/workflows/deploy-main.yml index 998aa87..50fc659 100644 --- a/.github/workflows/deploy-main.yml +++ b/.github/workflows/deploy-main.yml @@ -26,6 +26,7 @@ jobs: KAMAL_SERVER_IP: ${{ secrets.KAMAL_SERVER_IP }} KAMAL_BOT_DOMAIN: ${{ secrets.KAMAL_BOT_DOMAIN }} DATABASE_URL: ${{ secrets.DATABASE_URL }} + BOT_LOGWATCH_URL: ${{ secrets.BOT_LOGWATCH_URL }} APP_ID: ${{ secrets.APP_ID }} GH_APP_ID: ${{ secrets.GH_APP_ID }} GH_APP_NAME: ${{ secrets.GH_APP_NAME }} @@ -36,6 +37,7 @@ jobs: CODEFAIR_APP_DOMAIN: ${{ secrets.CODEFAIR_APP_DOMAIN }} ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }} ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }} + VALIDATOR_URL: ${{ secrets.VALIDATOR_URL }} steps: - uses: actions/checkout@v4 @@ -99,6 +101,7 @@ jobs: GH_OAUTH_APP_ID: ${{ secrets.GH_OAUTH_APP_ID }} GH_OAUTH_CLIENT_ID: ${{ secrets.GH_OAUTH_CLIENT_ID }} GH_OAUTH_CLIENT_SECRET: ${{ secrets.GH_OAUTH_CLIENT_SECRET }} + UI_LOGWATCH_URL: ${{ secrets.UI_LOGWATCH_URL }} ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }} ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }} ZENODO_CLIENT_ID: ${{ secrets.ZENODO_CLIENT_ID }} diff --git a/.github/workflows/deploy-staging.yml b/.github/workflows/deploy-staging.yml index 352ec7d..ab2de0f 100644 --- a/.github/workflows/deploy-staging.yml +++ b/.github/workflows/deploy-staging.yml @@ -26,6 +26,7 @@ jobs: KAMAL_SERVER_IP: ${{ secrets.KAMAL_SERVER_IP }} KAMAL_BOT_DOMAIN: ${{ secrets.KAMAL_BOT_DOMAIN }} DATABASE_URL: ${{ secrets.DATABASE_URL }} + BOT_LOGWATCH_URL: ${{ secrets.BOT_LOGWATCH_URL }} APP_ID: ${{ secrets.APP_ID }} GH_APP_ID: ${{ secrets.GH_APP_ID }} GH_APP_NAME: ${{ secrets.GH_APP_NAME }} @@ -36,6 +37,7 @@ jobs: CODEFAIR_APP_DOMAIN: ${{ secrets.CODEFAIR_APP_DOMAIN }} ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }} ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }} + VALIDATOR_URL: ${{ secrets.VALIDATOR_URL }} steps: - uses: actions/checkout@v4 @@ -99,6 +101,7 @@ jobs: GH_OAUTH_APP_ID: ${{ secrets.GH_OAUTH_APP_ID }} GH_OAUTH_CLIENT_ID: ${{ secrets.GH_OAUTH_CLIENT_ID }} GH_OAUTH_CLIENT_SECRET: ${{ secrets.GH_OAUTH_CLIENT_SECRET }} + UI_LOGWATCH_URL: ${{ secrets.UI_LOGWATCH_URL }} ZENODO_API_ENDPOINT: ${{ secrets.ZENODO_API_ENDPOINT }} ZENODO_ENDPOINT: ${{ secrets.ZENODO_ENDPOINT }} ZENODO_CLIENT_ID: ${{ secrets.ZENODO_CLIENT_ID }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 065af84..5132d5a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,17 @@ All notable changes the Codefair App will be documented in this file. The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/) and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html). +## v3.2.1 - 12-12-2024 + +### Added + +- Convert logging from 'consola' to 'logwatch' for improved log management and consistency across the application. + +### Fixed + +- Update CI and deployment workflows to include new environment variables 'BOT_LOGWATCH_URL' and 'VALIDATOR_URL'. +- Patch to Zenodo workflow that was causing the user to be notified of a failed Zenodo upload when the upload was successful. + ## v3.2.0 - 12-10-2024 ### Added diff --git a/bot/.kamal/secrets b/bot/.kamal/secrets index f16a422..f1daf50 100644 --- a/bot/.kamal/secrets +++ b/bot/.kamal/secrets @@ -15,6 +15,7 @@ KAMAL_SERVER_IP=$KAMAL_SERVER_IP # bot secrets APP_ID=$APP_ID DATABASE_URL=$DATABASE_URL +BOT_LOGWATCH_URL=$BOT_LOGWATCH_URL GH_APP_ID=$GH_APP_ID GH_APP_NAME=$GH_APP_NAME GH_APP_CLIENT_ID=$GH_APP_CLIENT_ID @@ -24,6 +25,7 @@ WEBHOOK_SECRET=$WEBHOOK_SECRET CODEFAIR_APP_DOMAIN=$CODEFAIR_APP_DOMAIN ZENODO_API_ENDPOINT=$ZENODO_API_ENDPOINT ZENODO_ENDPOINT=$ZENODO_ENDPOINT +VALIDATOR_URL=$VALIDATOR_URL # Option 2: Read secrets via a command # RAILS_MASTER_KEY=$(cat config/master.key) diff --git a/bot/archival/index.js b/bot/archival/index.js index d51cc3f..5cb8d63 100644 --- a/bot/archival/index.js +++ b/bot/archival/index.js @@ -1,5 +1,6 @@ import dbInstance from '../db.js'; import { consola } from 'consola'; +import { logwatch } from '../utils/logwatch.js'; import fs from 'fs'; const licensesJson = JSON.parse(fs.readFileSync('./public/assets/data/licenses.json', 'utf8')); @@ -20,7 +21,7 @@ export async function updateGitHubRelease(context, repositoryName, owner, releas release_id: releaseId, draft: false, }); - consola.success("Updated release to not be a draft!"); + logwatch.success("Updated release to not be a draft!"); } catch (error) { throw new Error(`Error updating the GitHub release: ${error}`, { cause: error }); } @@ -33,7 +34,7 @@ export async function updateGitHubRelease(context, repositoryName, owner, releas */ export async function publishZenodoDeposition(zenodoToken, depositionId) { try { - consola.start("Publishing the Zenodo deposition...", depositionId); + logwatch.start(`Publishing the Zenodo deposition: ${depositionId}`); const publishDeposition = await fetch( `${ZENODO_API_ENDPOINT}/deposit/depositions/${depositionId}/actions/publish`, { @@ -50,7 +51,7 @@ export async function publishZenodoDeposition(zenodoToken, depositionId) { } const publishedDeposition = await publishDeposition.json(); - consola.success("Zenodo deposition published successfully at:", publishedDeposition.links.latest_html); + logwatch.success(`Zenodo deposition published successfully at: ${publishedDeposition.links.latest_html}`); } catch (error) { throw new Error(`Error publishing the Zenodo deposition: ${error.message}`, { cause: error }); } @@ -251,7 +252,7 @@ export async function createNewVersionOfDeposition(zenodoToken, depositionId) { const errorText = await zenodoRecord.text(); throw new Error(`Failed to create a new version of Zenodo deposition. Status: ${zenodoRecord.status}: ${zenodoRecord.statusText}.`, { cause: errorText}); } - consola.success("New version of Zenodo deposition created successfully!"); + logwatch.success("New version of Zenodo deposition created successfully!"); const responseText = await zenodoRecord.json(); @@ -265,7 +266,7 @@ export async function createNewVersionOfDeposition(zenodoToken, depositionId) { }); if (!draftZenodoRecord.ok) { - consola.error("Error fetching the latest draft of Zenodo deposition:", draftZenodoRecord); + logwatch.error({message: "Error fetching the latest draft of Zenodo deposition:", fetchReponse: draftZenodoRecord}, true); const errorText = await draftZenodoRecord.text(); throw new Error(`Failed to fetch the latest draft of Zenodo deposition. Status: ${draftZenodoRecord.status}: ${draftZenodoRecord.statusText}. Error: ${errorText}`, { cause: errorText }); } @@ -295,7 +296,7 @@ export async function getZenodoDepositionInfo( if (zenodoDepositionInfo.submitted === false){ // Delete the files in the draft - consola.start("Requested deposition is a draft. Deleting the files in the draft..."); + logwatch.start("Requested deposition is a draft. Deleting the files in the draft..."); for (const file of zenodoDepositionInfo.files) { await deleteFileFromZenodo(depositionId, zenodoToken, file.id); } @@ -307,12 +308,12 @@ export async function getZenodoDepositionInfo( if (newZenodoVersion.files.length > 0) { for (const file of newZenodoVersion.files) { - consola.start("Deleting file from newly created draft:", file.links.download); + logwatch.start(`Deleting file from newly created draft: ${file.links.download}`); await deleteFileFromZenodo(newZenodoVersion.id, zenodoToken, file.id); } } - consola.success("New draft version of Zenodo deposition created successfully!"); + logwatch.success("New draft version of Zenodo deposition created successfully!"); return newZenodoVersion; } } @@ -354,8 +355,8 @@ export async function createZenodoMetadata(codemetadata, repository) { }); if (!codeMetaContent.license) { // fetch from the db - consola.warn(`No license found in the codemeta.json file. Fetching from the database...`); - consola.info(`License found in the database: ${existingLicense?.license_id}`); + logwatch.warn(`No license found in the codemeta.json file. Fetching from the database...`); + logwatch.info(`License found in the database: ${existingLicense?.license_id}`); codeMetaContent.license = `https://spdx.org/licenses/${existingLicense?.license_id}`; } const license = licensesJson.find((license) => license.detailsUrl === `${codeMetaContent.license}.json`); @@ -372,7 +373,7 @@ export async function createZenodoMetadata(codemetadata, repository) { }) if (!zenodoMetadata) { - consola.error("Zenodo metadata not found in the database. Please create a new Zenodo deposition."); + logwatch.error("Zenodo metadata not found in the database. Please create a new Zenodo deposition."); throw new Error("Zenodo metadata not found in the database. Please create a new Zenodo deposition."); } @@ -440,7 +441,7 @@ export async function updateZenodoMetadata(depositionId, zenodoToken, metadata) ); const updatedMetadataInfo = await updatedMetadata.json(); - consola.success("Zenodo deposition metadata updated successfully!"); + logwatch.success("Zenodo deposition metadata updated successfully!"); return updatedMetadataInfo; } catch (error) { throw new Error(`Error updating Zenodo metadata: ${error}`, { cause: error }); @@ -480,7 +481,7 @@ export async function uploadReleaseAssetsToZenodo( accept: 'application/octet-stream' } }); - consola.success(`Asset data fetched for ${asset.name}, for the release ${tagVersion}, from the GitHub repository: ${repository.name}`); + logwatch.success(`Asset data fetched for ${asset.name}, for the release ${tagVersion}, from the GitHub repository: ${repository.name}`); // Upload the file to Zenodo const uploadAsset = await fetch(`${bucket_url}/${asset.name}`, @@ -493,9 +494,9 @@ export async function uploadReleaseAssetsToZenodo( }); if (!uploadAsset.ok) { - consola.error(`Failed to upload ${asset.name}. Status: ${uploadAsset.statusText}. Error: ${uploadAsset}`); + logwatch.error(`Failed to upload ${asset.name}. Status: ${uploadAsset.statusText}. Error: ${uploadAsset}`); } else { - consola.success(`${asset.name} successfully uploaded to Zenodo!`); + logwatch.success(`${asset.name} successfully uploaded to Zenodo!`); } } catch (error) { throw new Error(`Error uploading assets to Zenodo: ${error}`, { cause: error }); @@ -516,13 +517,13 @@ export async function uploadReleaseAssetsToZenodo( ); if (!uploadZip.ok) { - consola.error(`Failed to upload zip file. Status: ${uploadZip.statusText}`); + logwatch.error(`Failed to upload zip file. Status: ${uploadZip.statusText}`); throw new Error(`Failed to upload zip file. Status: ${uploadZip.statusText}`); } const endTime = performance.now(); - consola.info(`Total duration to upload assets and zip to Zenodo deposition: ${(endTime - startTime) / 1000} seconds`); - consola.success("Zip file successfully uploaded to Zenodo!"); + logwatch.info(`Total duration to upload assets and zip to Zenodo deposition: ${(endTime - startTime) / 1000} seconds`); + logwatch.success("Zip file successfully uploaded to Zenodo!"); } /** @@ -543,12 +544,11 @@ export async function deleteFileFromZenodo(depositionId, zenodoToken, fileId) { ); if (!deleteFile.ok) { - consola.error(deleteFile); const errorText = await deleteFile.text(); throw new Error(`Failed to delete file from Zenodo. Status: ${deleteFile.status}: ${deleteFile.statusText}. Error: ${errorText}`); } - consola.success("File successfully deleted from Zenodo!"); + logwatch.success("File successfully deleted from Zenodo!"); } catch (error) { throw new Error(`Error deleting file from Zenodo: ${error}`, { cause: error }); } diff --git a/bot/config/deploy.yml b/bot/config/deploy.yml index 40867bc..b4cd87e 100644 --- a/bot/config/deploy.yml +++ b/bot/config/deploy.yml @@ -54,6 +54,8 @@ env: - CODEFAIR_APP_DOMAIN - ZENODO_API_ENDPOINT - ZENODO_ENDPOINT + - VALIDATOR_URL + - BOT_LOGWATCH_URL clear: NODE_ENV: production diff --git a/bot/cwl/index.js b/bot/cwl/index.js index 7359ec7..f6ba557 100644 --- a/bot/cwl/index.js +++ b/bot/cwl/index.js @@ -2,6 +2,7 @@ * * This file contains the functions to interact with the CWL files in the repository */ import { consola } from "consola"; +import { logwatch } from "../utils/logwatch.js"; import { isRepoPrivate, createId, @@ -10,6 +11,7 @@ import { import dbInstance from "../db.js"; const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN; +const { VALIDATOR_URL } = process.env; /** * * This function gets the CWL files in the repository @@ -20,7 +22,7 @@ const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN; */ export function getCWLFiles(context, owner, repository) { return new Promise((resolve, reject) => { - consola.info("Checking for CWL files in the repository..."); + logwatch.info("Checking for CWL files in the repository..."); const cwlFiles = []; const cwlObject = { @@ -51,9 +53,12 @@ export function getCWLFiles(context, owner, repository) { resolve(cwlObject); return; } - consola.error( - "Error finding CWL files throughout the repository:", - error, + logwatch.error( + { + message: "Error finding CWL files throughout the repository:", + error, + }, + true ); reject(error); } @@ -93,7 +98,7 @@ export function getCWLFiles(context, owner, repository) { */ export async function validateCWLFile(downloadUrl) { try { - const response = await fetch("https://cwl-validate.codefair.io/validate-cwl", { + const response = await fetch(`${VALIDATOR_URL}/validate-cwl`, { body: JSON.stringify({ file_path: downloadUrl, }), @@ -108,7 +113,7 @@ export async function validateCWLFile(downloadUrl) { return [false, error.error]; } if (!response.ok && response.status === 500) { - consola.error("Error validating CWL file:", response); + logwatch.error({message: "Error validating CWL file:", validation_response: response}, true); return [false, "Error validating CWL file"]; } if (response.ok) { @@ -116,7 +121,7 @@ export async function validateCWLFile(downloadUrl) { return [true, data.output]; } } catch (e) { - consola.error("Error validating CWL file:", e); + logwatch.error({message: "Error validating CWL file:", error: e}, true); return [false, "Error validating CWL file"]; } } @@ -180,14 +185,14 @@ export async function applyCWLTemplate( }); if (subjects.cwl.files.length === 0) { - consola.warn( - `No new/modified CWL files found in the repository, ${repository.name}`, + logwatch.warn( + `No CWL files found in the repository, ${repository.name}`, ); } - consola.start("Validating CWL files for", repository.name); + logwatch.start("Validating CWL files for", repository.name); // Validate each CWL file from list\ - consola.info(`Validating ${JSON.stringify(subjects.cwl)} CWL files`); + logwatch.info(`Validating ${JSON.stringify(subjects.cwl)} CWL files`); if (subjects.cwl.files.length > 0) { for (const file of subjects.cwl.files) { const fileSplit = file.name.split("."); @@ -252,7 +257,7 @@ export async function applyCWLTemplate( // Add the file to the table content of the issue dashboard tableContent += `| ${file.path} | ${isValidCWL ? "✔️" : "❌"} |\n`; - consola.success( + logwatch.success( `File: ${file.path} is ${isValidCWL ? "valid" : "invalid"}`, ); } @@ -276,7 +281,7 @@ export async function applyCWLTemplate( }); if (!cwlFiles.length > 0) { - consola.warn( + logwatch.warn( `No CWL files found in the repository, ${repository.name}, skipping CWL section`, ); return baseTemplate; @@ -317,15 +322,13 @@ export async function applyCWLTemplate( if (!newFiles.length > 0) { // All CWL files were removed from the repository - consola.warn( - "All CWL files were removed from:", - repository.name, - "skipping CWL section", + logwatch.warn( + `All CWL files were removed from: ${repository.name}, skipping CWL section` ); return baseTemplate; } else { // Recreate the table content to include the new and old cwl files - consola.start( + logwatch.start( "Recreating the table content for the CWL section to include new and old files", ); tableContent = ""; @@ -349,6 +352,6 @@ export async function applyCWLTemplate( const cwlBadge = `[![CWL](https://img.shields.io/badge/View_CWL_Report-0ea5e9.svg)](${url})`; baseTemplate += `${overallSection}\n\n### CWL Validations ${validOverall ? "✔️" : "❗"}\n\nCodefair has detected that you are following the Common Workflow Language (CWL) standard to describe your command line tool. Codefair ran the [cwltool validator](https://cwltool.readthedocs.io/en/latest/) and ${validOverall ? `all ***${subjects.cwl.files.length}*** CWL file(s) in your repository are valid.` : `***${failedCount}/${subjects.cwl.files.length}*** CWL file(s) in your repository are not valid.`}\n\n
\nSummary of the validation report\n\n| File | Validation result |\n| :---- | :----: |\n${tableContent}
\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`; - consola.success("CWL template section applied"); + logwatch.success("CWL template section applied"); return baseTemplate; } diff --git a/bot/index.js b/bot/index.js index 9490ccd..f6fa163 100644 --- a/bot/index.js +++ b/bot/index.js @@ -50,13 +50,13 @@ export default async (app, { getRouter }) => { router.use(express.static("public")); router.get("/healthcheck", (req, res) => { - consola.log('Requested healthcheck'); + logwatch.info('Requested healthcheck'); res.status(200).send("Health check passed"); }); // for kamal router.get("/up", (req, res) => { - consola.log('Requested healthcheck'); + logwatch.info('Requested healthcheck'); res.status(200).send("Health check passed"); }); @@ -76,7 +76,7 @@ export default async (app, { getRouter }) => { repoCount++; if (repoCount > 5) { - consola.info(`Applying action limit to ${repository.name}`); + logwatch.info(`Applying action limit to ${repository.name}`); applyActionLimit = true; actionCount = 5; } @@ -247,7 +247,7 @@ export default async (app, { getRouter }) => { }); } - consola.info("Repository uninstalled:", repository.name); + logwatch.info(`Repository uninstalled: ${repository.name}`); } }, ); @@ -263,7 +263,7 @@ export default async (app, { getRouter }) => { context.payload.ref !== `refs/heads/${context.payload.repository.default_branch}` ) { - consola.warn("Not pushing to default branch, ignoring..."); + logwatch.info("Not pushing to default branch, ignoring..."); return; } @@ -518,7 +518,7 @@ export default async (app, { getRouter }) => { }); if (installation && installation?.action_count > 0) { - consola.info("pull_request.opened: Action limit still applied, ignoring..."); + logwatch.info(`pull_request.opened: Action limit is at ${installation.action_count} still applied, ignoring...`); return; } @@ -534,7 +534,7 @@ export default async (app, { getRouter }) => { const dashboardIssue = issues.data.find(issue => issue.title === "FAIR Compliance Dashboard"); if (!dashboardIssue) { - consola.error("FAIR Compliance Dashboard issue not found"); + logwatch.error("FAIR Compliance Dashboard issue not found"); return; } @@ -553,7 +553,7 @@ export default async (app, { getRouter }) => { }); if (!response) { - consola.error("Error updating the license request PR URL"); + logwatch.error("Error updating the license request PR URL"); return; } @@ -578,7 +578,7 @@ export default async (app, { getRouter }) => { }); if (!response) { - consola.error("Error updating the code metadata PR URL"); + logwatch.error("Error updating the code metadata PR URL"); return; } @@ -716,7 +716,7 @@ export default async (app, { getRouter }) => { const lastModified = await applyLastModifiedTemplate(issueBodyRemovedCommand); await createIssue(context, owner, repository, ISSUE_TITLE, lastModified); if (error.cause) { - logwatch.error(error.cause); + logwatch.error({message: "Error.cause message for CWL Validation", error: error.cause}, true); } throw new Error("Error rerunning full repo validation", error); } @@ -773,7 +773,7 @@ export default async (app, { getRouter }) => { const lastModified = await applyLastModifiedTemplate(issueBodyRemovedCommand); await createIssue(context, owner, repository, ISSUE_TITLE, lastModified); if (error.cause) { - logwatch.error(error.cause); + logwatch.error({message: "Error.cause message for Full Repo Validation", error: error.cause}, true); } throw new Error("Error rerunning full repo validation", error); } @@ -802,7 +802,7 @@ export default async (app, { getRouter }) => { const { licenseId, licenseContent, licenseContentEmpty } = validateLicense(licenseRequest, existingLicense); - logwatch.info("License validation complete:", licenseId, licenseContent, licenseContentEmpty); + logwatch.info(`License validation complete: ${licenseId}, ${licenseContent}, ${licenseContentEmpty}`); // Update the database with the license information if (existingLicense) { @@ -839,7 +839,7 @@ export default async (app, { getRouter }) => { const lastModified = await applyLastModifiedTemplate(issueBodyRemovedCommand); await createIssue(context, owner, repository, ISSUE_TITLE, lastModified); if (error.cause) { - logwatch.error(error.cause); + logwatch.error({message: "Error.cause message for License Validation", error: error.cause}, true); } throw new Error("Error rerunning license validation", error); } @@ -862,7 +862,6 @@ export default async (app, { getRouter }) => { if (existingMetadataEntry?.metadata) { // Update the metadata variable - consola.info("Existing metadata in db found"); containsCitation = existingMetadataEntry.contains_citation; containsCodemeta = existingMetadataEntry.contains_codemeta; metadata = applyDbMetadata(existingMetadataEntry, metadata); @@ -933,7 +932,7 @@ export default async (app, { getRouter }) => { const lastModified = await applyLastModifiedTemplate(issueBodyRemovedCommand); await createIssue(context, owner, repository, ISSUE_TITLE, lastModified); if (error.cause) { - logwatch.error(error.cause); + logwatch.error({message: "Error.cause message for Metadata Validation", error: error.cause}, true); } throw new Error("Error rerunning metadata validation", error); } @@ -946,7 +945,7 @@ export default async (app, { getRouter }) => { const badgeURL = `${CODEFAIR_DOMAIN}/dashboard/${owner}/${repository.name}/release/zenodo`; const releaseBadge = `[![Create Release](https://img.shields.io/badge/Create_Release-00bcd4.svg)](${badgeURL})` const { depositionId, releaseId, tagVersion, userWhoSubmitted } = parseZenodoInfo(issueBody); - logwatch.info("Parsed Zenodo info:", depositionId, releaseId, tagVersion, userWhoSubmitted); + logwatch.info(`Parsed Zenodo info: ${depositionId}, ${releaseId}, ${tagVersion}, ${userWhoSubmitted}`); try { // 1. Get the metadata from the repository @@ -1037,7 +1036,7 @@ export default async (app, { getRouter }) => { } }); - consola.success("Updated the analytics in the database!"); + logwatch.success("Updated the analytics in the database!"); } catch (error) { // Update the issue with the new body // Update the GitHub issue with a status report @@ -1053,7 +1052,7 @@ export default async (app, { getRouter }) => { } }); if (error.cause) { - logwatch.error(`Error causes:`, { cause: error.cause }); + logwatch.error({message: "Error.cause message for Zenodo Publishing", error: error.cause}, true); } throw new Error(`Error publishing to Zenodo: ${error.message}`, { cause: error }); } @@ -1252,7 +1251,7 @@ export default async (app, { getRouter }) => { const dashboardIssue = issues.data.find(issue => issue.title === "FAIR Compliance Dashboard"); if (!dashboardIssue) { - consola.error("FAIR Compliance Dashboard issue not found"); + logwatch.error("FAIR Compliance Dashboard issue not found"); return; } @@ -1270,7 +1269,7 @@ export default async (app, { getRouter }) => { }); if (!response) { - consola.error("Error updating the license request PR URL"); + logwatch.error("Error updating the license request PR URL"); return; } diff --git a/bot/license/index.js b/bot/license/index.js index 7d95b0f..b162e84 100644 --- a/bot/license/index.js +++ b/bot/license/index.js @@ -2,6 +2,7 @@ * @fileoverview This file contains utility functions for the license bot */ import { consola } from "consola"; +import { logwatch } from "../utils/logwatch.js"; import dbInstance from "../db.js"; import { createId } from "../utils/tools/index.js"; @@ -22,10 +23,10 @@ export async function checkForLicense(context, owner, repo) { repo, }); - consola.success("License found in the repository!"); + logwatch.success("License found in the repository!"); return true; } catch (error) { - consola.warn("No license found in the repository"); + logwatch.warn("No license found in the repository"); // Errors when no License is found in the repo return false; } @@ -97,12 +98,12 @@ export async function createLicense(context, owner, repo, license) { }); defaultBranchName = defaultBranch.data.name; } catch (error) { - consola.error("Error getting default branch:", error); + logwatch.error({message: "Error getting default branch:", error}, true); return; } // Create a new branch base off the default branch - consola.info("Creating branch..."); + logwatch.info("Creating branch..."); await context.octokit.git.createRef({ owner, ref: `refs/heads/${branch}`, @@ -111,7 +112,7 @@ export async function createLicense(context, owner, repo, license) { }); // Create a new file - consola.info("Creating file..."); + logwatch.info("Creating file..."); await context.octokit.repos.createOrUpdateFileContents({ branch, content: Buffer.from(responseData.licenseText).toString("base64"), @@ -122,7 +123,7 @@ export async function createLicense(context, owner, repo, license) { }); // Create a PR from that branch with the commit of our added file - consola.info("Creating PR..."); + logwatch.info("Creating PR..."); await context.octokit.pulls.create({ title: "feat: ✨ LICENSE file added", base: defaultBranchName, @@ -134,7 +135,7 @@ export async function createLicense(context, owner, repo, license) { }); // Comment on issue to notify user that license has been added - consola.info("Commenting on issue..."); + logwatch.info("Commenting on issue..."); await context.octokit.issues.createComment({ body: `A LICENSE file with ${license} license terms has been added to a new branch and a pull request is awaiting approval. I will close this issue automatically once the pull request is approved.`, issue_number: context.payload.issue.number, @@ -142,7 +143,7 @@ export async function createLicense(context, owner, repo, license) { repo, }); } catch (error) { - consola.error("Error fetching license file:", error); + logwatch.error({message: "Error fetching license file:", error}, true); } } else { // License not found, comment on issue to notify user @@ -177,22 +178,21 @@ export function validateLicense(licenseRequest, existingLicense) { // console.log("Existing License:", existingLicense?.license_id); // consola.warn(existingLicense?.license_content.trim()); - // consola.info("dfl;aksjdfl;ksajl;dfkjas;ldfjk") // consola.warn(licenseContent.trim()); if (licenseId === "NOASSERTION") { if (licenseContent === "") { // No assertion and no content indicates no valid license - consola.info("No assertion and no content indicates no valid license"); + logwatch.info("No assertion and no content indicates no valid license"); licenseId = null; } else { // Custom license with content provided licenseContentEmpty = false; if (existingLicense?.license_content.trim() !== licenseContent.trim()) { - consola.info("Custom license with new content provided"); + logwatch.info("No assertion ID with different content from db provided"); licenseId = "Custom"; // New custom license } else if (existingLicense?.license_id) { - consola.info("Custom license with existing content provided"); + logwatch.info("Custom license with existing content provided"); licenseId = existingLicense.license_id; // Use existing custom license ID if it matches } } @@ -240,13 +240,12 @@ export async function applyLicenseTemplate( ({ licenseId, licenseContent, licenseContentEmpty } = validateLicense(licenseRequest, existingLicense)); - consola.info("License ID:", licenseId); - // consola.info("License Content:", licenseContent); - consola.info("License Content Empty:", licenseContentEmpty); + // logwatch.info("License ID:", licenseId); + // logwatch.info("License Content Empty:", licenseContentEmpty); } if (existingLicense) { - consola.info("Updating existing license request..."); + logwatch.info("Updating existing license request..."); await dbInstance.licenseRequest.update({ data: { contains_license: subjects.license, @@ -260,7 +259,7 @@ export async function applyLicenseTemplate( where: { repository_id: repository.id }, }); } else { - consola.info("Creating new license request..."); + logwatch.info("Creating new license request..."); await dbInstance.licenseRequest.create({ data: { contains_license: subjects.license, diff --git a/bot/main.js b/bot/main.js index 485ad2f..a52c891 100644 --- a/bot/main.js +++ b/bot/main.js @@ -6,8 +6,6 @@ const privateKey = process.env.GH_APP_PRIVATE_KEY.replace(/\\n/g, "\n"); async function startServer() { logwatch.info("Starting server..."); - logwatch.info({ prop1: "value1", prop2: "value2" }); - logwatch.debug({ prop3: "value1", prop4: "value2" }, true); const server = new Server({ port: process.env.PORT || 3000, diff --git a/bot/metadata/index.js b/bot/metadata/index.js index ea4fe80..fd7036c 100644 --- a/bot/metadata/index.js +++ b/bot/metadata/index.js @@ -10,7 +10,7 @@ import dbInstance from "../db.js"; import { logwatch } from "../utils/logwatch.js"; const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN; -const { GH_APP_NAME } = process.env; +const { GH_APP_NAME, VALIDATOR_URL } = process.env; /** * * Converts the date to a Unix timestamp @@ -254,7 +254,7 @@ export async function convertCitationForDB(citationContent, repository) { * @returns {object} - An object containing the metadata for the repository */ export async function gatherMetadata(context, owner, repo) { - consola.start("Gathering initial metadata from GitHub API..."); + logwatch.start("Gathering initial metadata from GitHub API..."); // Get the metadata of the repo const repoData = await context.octokit.repos.get({ @@ -410,10 +410,10 @@ export async function validateMetadata(metadataInfo, fileType, repository) { return false; } - consola.start("Sending content to metadata validator"); + logwatch.start("Sending content to metadata validator"); try { const response = await fetch( - "https://staging-validator.codefair.io/validate-codemeta", + `${VALIDATOR_URL}/validate-codemeta`, { method: "POST", headers: { @@ -441,7 +441,7 @@ export async function validateMetadata(metadataInfo, fileType, repository) { ); } const data = await response.json(); - consola.info("Codemeta validation response", data); + logwatch.info({message: "Codemeta validation response", data}, true); let validationMessage = `The codemeta.json file is valid according to the ${data.version} codemeta.json schema.`; if (data.message !== "valid") { @@ -461,8 +461,6 @@ export async function validateMetadata(metadataInfo, fileType, repository) { return data.message === "valid"; } catch (error) { logwatch.error(`error parsing the codemeta.json file: ${error}`); - consola.error("Error validating the codemeta.json file", error); - return false; } } catch (error) { @@ -487,16 +485,15 @@ export async function validateMetadata(metadataInfo, fileType, repository) { return false; } const loaded_file = yaml.load(metadataInfo.content); - consola.start("Validating the CITATION.cff file"); + logwatch.start("Validating the CITATION.cff file"); // Verify the required fields are present if (!loaded_file.title || !loaded_file.authors) { return false; } try { - // TODO: CHANGE THIS BEFORE DEPLOYING TO MAIN const response = await fetch( - "https://staging-validator.codefair.io/validate-citation", + `${VALIDATOR_URL}/validate-citation`, { method: "POST", headers: { @@ -514,7 +511,7 @@ export async function validateMetadata(metadataInfo, fileType, repository) { const data = await response.json(); - consola.info("Citation validation response", data); + logwatch.info({message: "Citation validation response", data}, true); let validationMessage = ""; if (data.message === "valid") { validationMessage = data.output; @@ -534,7 +531,7 @@ export async function validateMetadata(metadataInfo, fileType, repository) { return data.message === "valid"; } catch (error) { - consola.error("Error validating the CITATION.cff file", error); + logwatch.error({message: "Error validating the CITATION.cff file", error}, true); return false; } } catch (error) { @@ -615,7 +612,7 @@ export async function updateMetadataIdentifier( sha: citationSha, }); - consola.success("CITATION.cff file updated with Zenodo identifier"); + logwatch.success("CITATION.cff file updated with Zenodo identifier"); // Update the codemeta file await context.octokit.repos.createOrUpdateFileContents({ @@ -629,7 +626,7 @@ export async function updateMetadataIdentifier( sha: codeMetaSha, }); - consola.success("codemeta.json file updated with Zenodo identifier"); + logwatch.success("codemeta.json file updated with Zenodo identifier"); // Get the codemetadata content from the database const existingCodemeta = await dbInstance.codeMetadata.findUnique({ @@ -752,14 +749,14 @@ export function applyDbMetadata(existingMetadataEntry, metadata) { } export async function applyCodemetaMetadata(codemeta, metadata, repository) { - consola.info("Codemeta found"); + logwatch.info("Codemeta found"); try { // consola.warn("codemeta", codemeta.content.trim()); let codemetaContent; try { codemetaContent = JSON.parse(codemeta.content.trim()); } catch (error) { - consola.error("Error parsing codemeta content", error); + logwatch.error({message: "Error parsing codemeta content", error}, true); return; } const convertedCodemeta = await convertCodemetaForDB( @@ -953,13 +950,13 @@ export async function applyCodemetaMetadata(codemeta, metadata, repository) { return metadata; } catch (error) { - consola.error("Error applying codemeta metadata", JSON.stringify(error)); + logwatch.error({message: "Error applying codemeta metadata", error}, true); throw new Error("Error applying codemeta metadata", { cause: error }); } } export async function applyCitationMetadata(citation, metadata, repository) { - consola.info("Citation found"); + logwatch.info("Citation found"); const citationContent = yaml.load(citation.content); const convertedCitation = await convertCitationForDB( citationContent, @@ -1062,7 +1059,7 @@ export async function applyMetadataTemplate( if (githubAction && githubAction !== `${GH_APP_NAME}[bot]`) { // Push event was made, only update the metadata if the pusher updated the codemeta.json or citation.cff - consola.info("Push event detected"); + logwatch.info("Push event detected, checking for metadata or license changes..."); const updatedFiles = context.payload.head_commit.modified; const addedFiles = context.payload.head_commit.added; revalidate = false; @@ -1171,8 +1168,7 @@ export async function applyMetadataTemplate( return baseTemplate; } catch (error) { if (error.cause) { - consola.error("Error applying metadata template", error.cause); - // throw new Error("Error applying metadata template", { cause: error.cause }); + logwatch.error({message: "Error applying metadata template", error: error.cause}, true); } throw new Error("Error applying metadata template", { cause: error }); } diff --git a/bot/utils/logwatch.js b/bot/utils/logwatch.js index f37227f..da513e2 100644 --- a/bot/utils/logwatch.js +++ b/bot/utils/logwatch.js @@ -1,4 +1,5 @@ import consola from "consola"; +const { BOT_LOGWATCH_URL } = process.env; class Logwatch { /** @@ -7,7 +8,7 @@ class Logwatch { */ constructor(endpoint) { const BOT_ENDPOINT = - "https://logwatch.fairdataihub.org/api/log/cm4hkn79200027r01ya9gij7r"; + BOT_LOGWATCH_URL; if (!endpoint) { this.endpoint = BOT_ENDPOINT; @@ -97,6 +98,16 @@ class Logwatch { this._sendLog("info", message, isJson ? "json" : "text"); } + /** + * Success level logging + * @param {string|object} message - Log message + * @param {boolean} [isJson=false] - Whether the message is a JSON object + */ + success(message, isJson = false) { + consola.success(message); + this._sendLog("info", message, isJson ? "json" : "text"); + } + /** * Warning level logging * @param {string|object} message - Log message diff --git a/bot/utils/renderer/index.js b/bot/utils/renderer/index.js index a5aca7d..738d7b6 100644 --- a/bot/utils/renderer/index.js +++ b/bot/utils/renderer/index.js @@ -1,4 +1,5 @@ import { consola } from "consola"; +import { logwatch } from "../logwatch.js"; import { applyGitHubIssueToDatabase, applyLastModifiedTemplate, @@ -33,9 +34,8 @@ export async function renderIssues( ) { try { if (emptyRepo) { - consola.success( - "Applying empty repo template for repository:", - repository.name, + logwatch.success( + `Applying empty repo template for repository: ${repository.name}` ); let emptyTemplate = `# Check the FAIRness of your software\n\nThis issue is your repository's dashboard for all things FAIR. Keep it open as making and keeping software FAIR is a continuous process that evolves along with the software. You can read the [documentation](https://docs.codefair.io/docs/dashboard.html) to learn more.\n\n> [!WARNING]\n> Currently your repository is empty and will not be checked until content is detected within your repository.\n\n## LICENSE\n\nTo make your software reusable a license file is expected at the root level of your repository. Codefair will check for a license file after you add content to your repository.\n\n![License](https://img.shields.io/badge/License_Not_Checked-fbbf24)\n\n## Metadata\n\nTo make your software FAIR a CITATION.cff and codemeta.json metadata files are expected at the root level of your repository. Codefair will check for these files after a license file is detected.\n\n![Metadata](https://img.shields.io/badge/Metadata_Not_Checked-fbbf24)`; @@ -83,7 +83,7 @@ export async function renderIssues( }); } } catch (error) { - consola.error("Error fetching pull request:", error); + logwatch.error({message: "Error fetching pull request:", error}, true); } } @@ -117,7 +117,7 @@ export async function renderIssues( }); } } catch (error) { - consola.error("Error fetching metadata pull request:", error); + logwatch.error({message: "Error fetching metadata pull request:", error}, true); } } @@ -176,7 +176,7 @@ export async function createIssue(context, owner, repository, title, body) { } if (!noIssue) { - consola.info("Creating an issue since no open issue was found"); + logwatch.info("Creating an issue since no open issue was found"); // Issue has not been created so we create one const response = await context.octokit.issues.create({ title, @@ -188,7 +188,7 @@ export async function createIssue(context, owner, repository, title, body) { await applyGitHubIssueToDatabase(response.data.number, repository.id); } else { // Update the issue with the new body - consola.info("Updating existing issue: " + issueNumber); + logwatch.info(`Updating existing issue: ${issueNumber}`); await context.octokit.issues.update({ title, body, @@ -210,7 +210,7 @@ export async function createIssue(context, owner, repository, title, body) { repo: repository.name, }); - consola.info("Creating an issue since none exist"); + logwatch.info("Creating an issue since none exist"); await applyGitHubIssueToDatabase(response.data.number, repository.id); } diff --git a/bot/utils/tools/index.js b/bot/utils/tools/index.js index c58457a..989a5ae 100644 --- a/bot/utils/tools/index.js +++ b/bot/utils/tools/index.js @@ -2,6 +2,7 @@ * @fileoverview Utility functions for the bot */ import { consola } from "consola"; +import { logwatch } from "../logwatch.js"; import { init } from "@paralleldrive/cuid2"; import human from "humanparser"; import dayjs from "dayjs"; @@ -18,12 +19,12 @@ dayjs.extend(timezone); */ export async function intializeDatabase() { try { - consola.start("Connecting to database..."); + logwatch.start("Connecting to database..."); await dbInstance; - consola.success("Connected to database!"); + logwatch.success("Connected to database!"); return true; } catch (error) { - consola.error("Error connecting to database:", error); + logwatch.error({message: "Error connecting to database:", error}, true); } } @@ -42,7 +43,7 @@ export const createId = init({ */ export function checkEnvVariable(varName) { if (!process.env[varName]) { - consola.error(`Please set the ${varName} environment variable`); + logwatch.error(`Please set the ${varName} environment variable`); process.exit(1); } } @@ -63,7 +64,7 @@ export async function getDefaultBranch(context, owner, repositoryName) { return defaultBranch.data.default_branch; } catch (error) { - consola.error("Error getting the default branch:", error); + logwatch.error({message: "Error getting the default branch:", error}, true); } } @@ -270,7 +271,7 @@ export async function verifyRepoName( collection, ) { if (dbRepoName !== repository.name) { - consola.info( + logwatch.info( `Repository name for ${owner} has changed from ${dbRepoName} to ${repository.name}`, ); @@ -305,7 +306,7 @@ export async function isRepoEmpty(context, owner, repoName) { if (error.status === 404) { return true; } - consola.error("Error checking if the repository is empty:", error); + logwatch.error({message: "Error checking if the repository is empty:", error}, true); } } @@ -375,7 +376,7 @@ export async function verifyInstallationAnalytics( } if (installation.action_count === 0) { - consola.info("Action limit reached, no longer limiting actions"); + logwatch.info(`Action limit reached for ${installation.repo}, no longer limiting actions`); await dbInstance.installation.update({ data: { action_count: 0, @@ -419,12 +420,12 @@ export async function isRepoPrivate(context, owner, repoName) { repo: repoName, }); - consola.info( + logwatch.info( `Repository ${repoName} is private: ${repoDetails.data.private}`, ); return repoDetails.data.private; } catch (error) { - consola.error("Error verifying if the repository is private:", error); + logwatch.error({message: "Error verifying if the repository is private:", error}, true); } } @@ -484,10 +485,6 @@ export function applyLastModifiedTemplate(baseTemplate) { .tz("America/Los_Angeles") .format("MMM D YYYY, HH:mm:ss"); - consola.info( - `GitHub Issue updated at: ${lastModified} (timezone: America/Los_Angeles)`, - ); - return `${baseTemplate}\n\nLast updated ${lastModified} (timezone: America/Los_Angeles)`; } @@ -506,7 +503,7 @@ export async function getReleaseById(context, repositoryName, owner, releaseId) release_id: releaseId, }); - consola.success(`Fetched the draft release for: ${repositoryName}`); + logwatch.success(`Fetched the draft release for: ${repositoryName}`); return draftRelease; } catch (error) { @@ -532,7 +529,7 @@ export async function downloadRepositoryZip(context, owner, repositoryName, bran ref: branch }); - consola.success(`Downloaded the repository archive successfully for: ${repositoryName}`); + logwatch.success(`Downloaded the repository archive successfully for: ${repositoryName}`); return data; } catch (error) { throw new Error(`Error download the repository archive for ${repositoryName}: ${error}`, { cause: error }); diff --git a/ui/.kamal/secrets b/ui/.kamal/secrets index 0db1130..873ffc3 100644 --- a/ui/.kamal/secrets +++ b/ui/.kamal/secrets @@ -23,6 +23,7 @@ ZENODO_ENDPOINT=$ZENODO_ENDPOINT ZENODO_CLIENT_ID=$ZENODO_CLIENT_ID ZENODO_CLIENT_SECRET=$ZENODO_CLIENT_SECRET ZENODO_REDIRECT_URI=$ZENODO_REDIRECT_URI +UI_LOGWATCH_URL=$UI_LOGWATCH_URL # Option 2: Read secrets via a command # RAILS_MASTER_KEY=$(cat config/master.key)