Skip to content

Commit

Permalink
fix: 🐛 bot patches for cwl and action queue features (#48)
Browse files Browse the repository at this point in the history 
* feat: ✨ remove deleted cwl files from db

* dev: 📝 better comments

* fix: 🐛 add await to db search, set to false if gatherd cwl is undefind

* refactor: ♻️ convert getcwlFiles fn to a promise

* refactor: ♻️ remove token from url string in priv repos, update emojis on validation report

* refator: ♻️ sourcery notes

* refactor: ♻️ sourcery notes (push fn out function block
  • Loading branch information
@slugb0t
slugb0t authored Aug 8, 2024
1 parent 8a59388 commit d44ae7b
Show file tree
Hide file tree
Showing 3 changed files with 132 additions and 57 deletions.
57 changes: 44 additions & 13 deletions bot/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,17 @@ export default async (app, { getRouter }) => {
);
const cwl = await getCWLFiles(context, owner, repository.name);
const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

// If existing cwl validation exists, update the contains_cwl value
const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

if (cwlExists) {
if (cwlExists?.contains_cwl_files) {
cwlObject.contains_cwl = cwlExists.contains_cwl_files;
}

Expand Down Expand Up @@ -237,7 +238,7 @@ export default async (app, { getRouter }) => {
return;
}

if (installation?.action === false && installation?.action_count > 4) {
if (installation?.action && installation?.action_count >= 4) {
installationCollection.updateOne(
{ repositoryId: repository.id },
{
Expand Down Expand Up @@ -265,9 +266,11 @@ export default async (app, { getRouter }) => {

// Check if any of the commits added a LICENSE, CITATION, or codemeta file
const gatheredCWLFiles = [];
const removedCWLFiles = [];
if (commits.length > 0) {
for (let i = 0; i < commits.length; i++) {
if (commits[i]?.added?.length > 0) {
// Iterate through the added files
for (let j = 0; j < commits[i]?.added.length; j++) {
if (commits[i].added[j] === "LICENSE") {
license = true;
Expand All @@ -288,7 +291,7 @@ export default async (app, { getRouter }) => {
}
}
}
// TODO: This will only return the file name so request the file name and gather the file metadata
// Iterate through the modified files
if (commits[i]?.modified?.length > 0) {
for (let j = 0; j < commits[i]?.modified.length; j++) {
const fileSplit = commits[i]?.modified[j].split(".");
Expand All @@ -298,6 +301,29 @@ export default async (app, { getRouter }) => {
}
}
}

// Iterate through the remove files
if (commits[i]?.removed?.length > 0) {
for (let j = 0; j < commits[i]?.removed.length; j++) {
const fileSplit = commits[i]?.removed[j].split(".");
if (fileSplit.includes("cwl")) {
removedCWLFiles.push(commits[i].removed[j]);
continue;
}
if (commits[i]?.removed[j] === "LICENSE") {
license = false;
continue;
}
if (commits[i]?.removed[j] === "CITATION.cff") {
citation = false;
continue;
}
if (commits[i]?.removed[j] === "codemeta.json") {
codemeta = false;
continue;
}
}
}
}
}

Expand All @@ -315,8 +341,9 @@ export default async (app, { getRouter }) => {
}

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: removedCWLFiles,
};

const cwlExists = await db.collection("cwlValidation").findOne({
Expand Down Expand Up @@ -381,11 +408,12 @@ export default async (app, { getRouter }) => {
const cwl = await getCWLFiles(context, owner, repository.name); // This variable is an array of cwl files

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down Expand Up @@ -471,11 +499,12 @@ export default async (app, { getRouter }) => {
);

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down Expand Up @@ -506,7 +535,7 @@ export default async (app, { getRouter }) => {
}
});

// When an issue is deleted
// When an issue is deleted or closed
app.on(["issues.deleted", "issues.closed"], async (context) => {
const repository = context.payload.repository;
const issueTitle = context.payload.issue.title;
Expand Down Expand Up @@ -540,6 +569,7 @@ export default async (app, { getRouter }) => {
}
});

// When an issue is reopened
app.on("issues.reopened", async (context) => {
const repository = context.payload.repository;
const owner = context.payload.repository.owner.login;
Expand All @@ -558,11 +588,12 @@ export default async (app, { getRouter }) => {
const cwl = await getCWLFiles(context, owner, repository.name); // This variable is an array of cwl files

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down
66 changes: 34 additions & 32 deletions bot/utils/cwl/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,46 @@
* @param {String} repoName - Repository name
* @returns {Array} - Array of CWL files in the repository
*/
export async function getCWLFiles(context, owner, repoName) {
const cwlFiles = [];
console.log("Checking for CWL files in the repository");
export function getCWLFiles(context, owner, repoName) {
return new Promise((resolve, reject) => {
console.log("Checking for CWL files in the repository");

async function searchDirectory(path) {
try {
const repoContent = await context.octokit.repos.getContent({
owner,
path,
repo: repoName,
});
const cwlFiles = [];

for (const file of repoContent.data) {
const fileSplit = file.name.split(".");
if (file.type === "file" && fileSplit.includes("cwl")) {
cwlFiles.push(file);
const searchDirectory = async function (path) {
try {
const repoContent = await context.octokit.repos.getContent({
owner,
path,
repo: repoName,
});

for (const file of repoContent.data) {
const fileSplit = file.name.split(".");
if (file.type === "file" && fileSplit.includes("cwl")) {
cwlFiles.push(file);
}
if (file.type === "dir") {
await searchDirectory(file.path);
}
}
if (file.type === "dir") {
await searchDirectory(file.path);
} catch (error) {
if (error.status === 404) {
// Repository is empty
resolve(cwlFiles);
return;
}
console.log("Error finding CWL files throughout the repository");
console.log(error);
reject(error);
}
} catch (error) {
console.log("Error finding CWL files throughout the repository");
console.log(error);
if (error.status === 404) {
// Repository is empty
return cwlFiles;
}
}
}
};

try {
await searchDirectory("");
return cwlFiles;
} catch (error) {
console.log("Error checking for CWL file");
console.log(error);
}
// Call the async function and handle its promise
searchDirectory("")
.then(() => resolve(cwlFiles))
.catch(reject);
});
}

export async function validateCWLFile(downloadUrl) {
Expand Down
66 changes: 54 additions & 12 deletions bot/utils/renderer/index.js
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,23 @@ import dbInstance from "../../db.js";
const GITHUB_APP_NAME = process.env.GITHUB_APP_NAME;
const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN;

/**
* * Removes the token from the URL in the validation message
* @param {String} inputString - The string to remove the token from
* @returns {String} - The string with the token removed
*/
function removeTokenFromUrlInString(inputString) {
// Regex to find the GitHub raw URL with an optional token
const urlRegex =
/https:\/\/raw\.githubusercontent\.com\/[^\s:]+(\?token=[^:\s]+)?/g;

// Replace each found URL in the string after removing the token
return inputString.replace(urlRegex, (url) => {
console.log(url);
return url.replace(/\?token=[^:]+/, "");
});
}

/**
* * Applies the metadata template to the base template (CITATION.cff and codemeta.json)
*
Expand Down Expand Up @@ -255,12 +272,12 @@ export async function applyCWLTemplate(
) {
const privateRepo = await isRepoPrivate(context, owner, repository.name);
// If the repository is private and contains CWL files, we cannot validate them
if (privateRepo && subjects.cwl.contains_cwl) {
baseTemplate += `\n\n## CWL Validations ❌\n\n> [!WARNING]\n> Your repository is private. Codefair will not be able to validate any CWL files for you. You can check the CWL file yourself using the [cwltool validator](https://cwltool.readthedocs.io/en/latest/)`;
return baseTemplate;
}
// if (privateRepo && subjects.cwl.contains_cwl) {
// baseTemplate += `\n\n## CWL Validations ❌\n\n> [!WARNING]\n> Your repository is private. Codefair will not be able to validate any CWL files for you. You can check the CWL file yourself using the [cwltool validator](https://cwltool.readthedocs.io/en/latest/)`;
// return baseTemplate;
// }

let url = `${CODEFAIR_DOMAIN}/add/cwl/`;
let url = `${CODEFAIR_DOMAIN}/view/cwl-validation/`;
const identifier = createId();
const cwlCollection = dbInstance.getDb().collection("cwlValidation");
const existingCWL = await cwlCollection.findOne({
Expand Down Expand Up @@ -294,7 +311,7 @@ export async function applyCWLTemplate(

// TODO: Create a table of the cwl files that were validated
for (const file of existingCWL.files) {
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "✔️" : "❌"} |\n`;
}
}

Expand All @@ -319,7 +336,7 @@ export async function applyCWLTemplate(
for (const file of subjects.cwl.files) {
const fileSplit = file.name.split(".");
if (fileSplit.includes("cwl")) {
const [isValidCWL, validationMessage] = await validateCWLFile(
let [isValidCWL, validationMessage] = await validateCWLFile(
file.download_url,
);

Expand All @@ -331,6 +348,12 @@ export async function applyCWLTemplate(
failedCount += 1;
}

if (isRepoPrivate) {
console.log("Private repo, removing token from validation message");
validationMessage = removeTokenFromUrlInString(validationMessage);
console.log(validationMessage);
}

const newDate = Date.now();
cwlFiles.push({
href: file.html_url,
Expand All @@ -341,10 +364,10 @@ export async function applyCWLTemplate(
validation_status: isValidCWL ? "valid" : "invalid",
});

tableContent += `| ${file.path} | ${isValidCWL ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${isValidCWL ? "✔️" : "❌"} |\n`;
}
}
url = `${CODEFAIR_DOMAIN}/add/cwl/${identifier}`;
url = `${CODEFAIR_DOMAIN}/view/cwl-validation/${identifier}`;
if (!existingCWL) {
// Entry does not exist in the db, create a new one
const newDate = Date.now();
Expand Down Expand Up @@ -372,11 +395,11 @@ export async function applyCWLTemplate(
},
},
);
url = `${CODEFAIR_DOMAIN}/add/cwl/${existingCWL.identifier}`;
url = `${CODEFAIR_DOMAIN}/view/cwl-validation/${existingCWL.identifier}`;

// TODO: Create a table of the cwl files that were validated
for (const file of existingCWL.files) {
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "✔️" : "❌"} |\n`;
subjects.cwl.files.push(file);

if (file.validation_status === "invalid") {
Expand All @@ -390,7 +413,26 @@ export async function applyCWLTemplate(
}

const cwlBadge = `[![CWL](https://img.shields.io/badge/View_CWL_Report-0ea5e9.svg)](${url})`;
baseTemplate += `\n\n## CWL Validations ${validOverall ? "✔️" : "❌"}\n\nCWL files were found in the repository and ***${subjects.cwl.files.length - failedCount}/${subjects.cwl.files.length}*** are considered valid by the [cwltool validator](https://cwltool.readthedocs.io/en/latest/).\n\n<details>\n<summary>Summary of the validation report</summary>\n\n| File | Status |\n| :---- | :----: |\n${tableContent}</details>\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`;
baseTemplate += `\n\n## CWL Validations ${validOverall ? "✔️" : "❗"}\n\nCWL files were found in the repository and ***${subjects.cwl.files.length - failedCount}/${subjects.cwl.files.length}*** are considered valid by the [cwltool validator](https://cwltool.readthedocs.io/en/latest/).\n\n<details>\n<summary>Summary of the validation report</summary>\n\n| File | Status |\n| :---- | :----: |\n${tableContent}</details>\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`;
}

if (subjects.cwl.removed_files.length > 0) {
// Remove the files from the database
const cwlCollection = dbInstance.getDb().collection("cwlValidation");
const existingCWL = await cwlCollection.findOne({
repositoryId: repository.id,
});

if (existingCWL) {
const newFiles = existingCWL.files.filter((file) => {
return !subjects.cwl.removed_files.includes(file.path);
});

await cwlCollection.updateOne(
{ repositoryId: repository.id },
{ $set: { files: newFiles } },
);
}
}

return baseTemplate;
Expand Down

0 comments on commit d44ae7b

Please sign in to comment.