Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: 🐛 bot patches for cwl and action queue features #48

Merged
merged 7 commits into from
Aug 8, 2024
Merged

fix: 🐛 bot patches for cwl and action queue features #48

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
7 commits
Select commit Hold shift + click to select a range
ceca6fa
feat: :sparkles: remove deleted cwl files from db
slugb0t Aug 7, 2024
50b9eb0
dev: :memo: better comments
slugb0t Aug 8, 2024
aebe7f5
fix: :bug: add await to db search, set to false if gatherd cwl is und…
slugb0t Aug 8, 2024
3ae1f56
refactor: :recycle: convert getcwlFiles fn to a promise
slugb0t Aug 8, 2024
6d7a9ed
refactor: :recycle: remove token from url string in priv repos, updat…
slugb0t Aug 8, 2024
c101063
refator: :recycle: sourcery notes
slugb0t Aug 8, 2024
89d38a5
refactor: :recycle: sourcery notes (push fn out function block
slugb0t Aug 8, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
57 changes: 44 additions & 13 deletions bot/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -103,16 +103,17 @@ export default async (app, { getRouter }) => {
);
const cwl = await getCWLFiles(context, owner, repository.name);
const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

// If existing cwl validation exists, update the contains_cwl value
const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

if (cwlExists) {
if (cwlExists?.contains_cwl_files) {
cwlObject.contains_cwl = cwlExists.contains_cwl_files;
}

Expand Down Expand Up @@ -237,7 +238,7 @@ export default async (app, { getRouter }) => {
return;
}

if (installation?.action === false && installation?.action_count > 4) {
if (installation?.action && installation?.action_count >= 4) {
installationCollection.updateOne(
{ repositoryId: repository.id },
{
Expand Down Expand Up @@ -265,9 +266,11 @@ export default async (app, { getRouter }) => {

// Check if any of the commits added a LICENSE, CITATION, or codemeta file
const gatheredCWLFiles = [];
const removedCWLFiles = [];
if (commits.length > 0) {
for (let i = 0; i < commits.length; i++) {
if (commits[i]?.added?.length > 0) {
// Iterate through the added files
for (let j = 0; j < commits[i]?.added.length; j++) {
if (commits[i].added[j] === "LICENSE") {
license = true;
Expand All @@ -288,7 +291,7 @@ export default async (app, { getRouter }) => {
}
}
}
// TODO: This will only return the file name so request the file name and gather the file metadata
// Iterate through the modified files
if (commits[i]?.modified?.length > 0) {
for (let j = 0; j < commits[i]?.modified.length; j++) {
const fileSplit = commits[i]?.modified[j].split(".");
Expand All @@ -298,6 +301,29 @@ export default async (app, { getRouter }) => {
}
}
}

// Iterate through the remove files
if (commits[i]?.removed?.length > 0) {
for (let j = 0; j < commits[i]?.removed.length; j++) {
const fileSplit = commits[i]?.removed[j].split(".");
if (fileSplit.includes("cwl")) {
removedCWLFiles.push(commits[i].removed[j]);
continue;
}
if (commits[i]?.removed[j] === "LICENSE") {
license = false;
continue;
}
if (commits[i]?.removed[j] === "CITATION.cff") {
citation = false;
continue;
}
if (commits[i]?.removed[j] === "codemeta.json") {
codemeta = false;
continue;
}
}
}
}
}

Expand All @@ -315,8 +341,9 @@ export default async (app, { getRouter }) => {
}

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: removedCWLFiles,
};

const cwlExists = await db.collection("cwlValidation").findOne({
Expand Down Expand Up @@ -381,11 +408,12 @@ export default async (app, { getRouter }) => {
const cwl = await getCWLFiles(context, owner, repository.name); // This variable is an array of cwl files

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down Expand Up @@ -471,11 +499,12 @@ export default async (app, { getRouter }) => {
);

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down Expand Up @@ -506,7 +535,7 @@ export default async (app, { getRouter }) => {
}
});

// When an issue is deleted
// When an issue is deleted or closed
app.on(["issues.deleted", "issues.closed"], async (context) => {
const repository = context.payload.repository;
const issueTitle = context.payload.issue.title;
Expand Down Expand Up @@ -540,6 +569,7 @@ export default async (app, { getRouter }) => {
}
});

// When an issue is reopened
app.on("issues.reopened", async (context) => {
const repository = context.payload.repository;
const owner = context.payload.repository.owner.login;
Expand All @@ -558,11 +588,12 @@ export default async (app, { getRouter }) => {
const cwl = await getCWLFiles(context, owner, repository.name); // This variable is an array of cwl files

const cwlObject = {
contains_cwl: cwl.length > 0,
contains_cwl: cwl.length > 0 || false,
files: cwl,
removed_files: [],
};

const cwlExists = db.collection("cwlValidation").findOne({
const cwlExists = await db.collection("cwlValidation").findOne({
repositoryId: repository.id,
});

Expand Down
66 changes: 34 additions & 32 deletions bot/utils/cwl/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,44 +9,46 @@
* @param {String} repoName - Repository name
* @returns {Array} - Array of CWL files in the repository
*/
export async function getCWLFiles(context, owner, repoName) {
const cwlFiles = [];
console.log("Checking for CWL files in the repository");
export function getCWLFiles(context, owner, repoName) {
return new Promise((resolve, reject) => {
console.log("Checking for CWL files in the repository");

async function searchDirectory(path) {
try {
const repoContent = await context.octokit.repos.getContent({
owner,
path,
repo: repoName,
});
const cwlFiles = [];

for (const file of repoContent.data) {
const fileSplit = file.name.split(".");
if (file.type === "file" && fileSplit.includes("cwl")) {
cwlFiles.push(file);
const searchDirectory = async function (path) {
try {
const repoContent = await context.octokit.repos.getContent({
owner,
path,
repo: repoName,
});

for (const file of repoContent.data) {
const fileSplit = file.name.split(".");
if (file.type === "file" && fileSplit.includes("cwl")) {
cwlFiles.push(file);
}
if (file.type === "dir") {
await searchDirectory(file.path);
}
}
if (file.type === "dir") {
await searchDirectory(file.path);
} catch (error) {
if (error.status === 404) {
// Repository is empty
resolve(cwlFiles);
return;
}
console.log("Error finding CWL files throughout the repository");
console.log(error);
reject(error);
}
} catch (error) {
console.log("Error finding CWL files throughout the repository");
console.log(error);
if (error.status === 404) {
// Repository is empty
return cwlFiles;
}
}
}
};

try {
await searchDirectory("");
return cwlFiles;
} catch (error) {
console.log("Error checking for CWL file");
console.log(error);
}
// Call the async function and handle its promise
searchDirectory("")
.then(() => resolve(cwlFiles))
.catch(reject);
});
}

export async function validateCWLFile(downloadUrl) {
Expand Down
66 changes: 54 additions & 12 deletions bot/utils/renderer/index.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,23 @@ import dbInstance from "../../db.js";
const GITHUB_APP_NAME = process.env.GITHUB_APP_NAME;
const CODEFAIR_DOMAIN = process.env.CODEFAIR_APP_DOMAIN;

/**
* * Removes the token from the URL in the validation message
* @param {String} inputString - The string to remove the token from
* @returns {String} - The string with the token removed
*/
function removeTokenFromUrlInString(inputString) {
// Regex to find the GitHub raw URL with an optional token
const urlRegex =
/https:\/\/raw\.githubusercontent\.com\/[^\s:]+(\?token=[^:\s]+)?/g;

// Replace each found URL in the string after removing the token
return inputString.replace(urlRegex, (url) => {
console.log(url);
return url.replace(/\?token=[^:]+/, "");
});
}

/**
* * Applies the metadata template to the base template (CITATION.cff and codemeta.json)
*
Expand Down Expand Up @@ -255,12 +272,12 @@ export async function applyCWLTemplate(
) {
const privateRepo = await isRepoPrivate(context, owner, repository.name);
// If the repository is private and contains CWL files, we cannot validate them
if (privateRepo && subjects.cwl.contains_cwl) {
baseTemplate += `\n\n## CWL Validations ❌\n\n> [!WARNING]\n> Your repository is private. Codefair will not be able to validate any CWL files for you. You can check the CWL file yourself using the [cwltool validator](https://cwltool.readthedocs.io/en/latest/)`;
return baseTemplate;
}
// if (privateRepo && subjects.cwl.contains_cwl) {
// baseTemplate += `\n\n## CWL Validations ❌\n\n> [!WARNING]\n> Your repository is private. Codefair will not be able to validate any CWL files for you. You can check the CWL file yourself using the [cwltool validator](https://cwltool.readthedocs.io/en/latest/)`;
// return baseTemplate;
// }

let url = `${CODEFAIR_DOMAIN}/add/cwl/`;
let url = `${CODEFAIR_DOMAIN}/view/cwl-validation/`;
const identifier = createId();
const cwlCollection = dbInstance.getDb().collection("cwlValidation");
const existingCWL = await cwlCollection.findOne({
Expand Down Expand Up @@ -294,7 +311,7 @@ export async function applyCWLTemplate(

// TODO: Create a table of the cwl files that were validated
for (const file of existingCWL.files) {
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "✔️" : "❌"} |\n`;
}
}

Expand All @@ -319,7 +336,7 @@ export async function applyCWLTemplate(
for (const file of subjects.cwl.files) {
const fileSplit = file.name.split(".");
if (fileSplit.includes("cwl")) {
const [isValidCWL, validationMessage] = await validateCWLFile(
let [isValidCWL, validationMessage] = await validateCWLFile(
file.download_url,
);

Expand All @@ -331,6 +348,12 @@ export async function applyCWLTemplate(
failedCount += 1;
}

if (isRepoPrivate) {
console.log("Private repo, removing token from validation message");
validationMessage = removeTokenFromUrlInString(validationMessage);
console.log(validationMessage);
}

const newDate = Date.now();
cwlFiles.push({
href: file.html_url,
Expand All @@ -341,10 +364,10 @@ export async function applyCWLTemplate(
validation_status: isValidCWL ? "valid" : "invalid",
});

tableContent += `| ${file.path} | ${isValidCWL ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${isValidCWL ? "✔️" : "❌"} |\n`;
}
}
url = `${CODEFAIR_DOMAIN}/add/cwl/${identifier}`;
url = `${CODEFAIR_DOMAIN}/view/cwl-validation/${identifier}`;
if (!existingCWL) {
// Entry does not exist in the db, create a new one
const newDate = Date.now();
Expand Down Expand Up @@ -372,11 +395,11 @@ export async function applyCWLTemplate(
},
},
);
url = `${CODEFAIR_DOMAIN}/add/cwl/${existingCWL.identifier}`;
url = `${CODEFAIR_DOMAIN}/view/cwl-validation/${existingCWL.identifier}`;

// TODO: Create a table of the cwl files that were validated
for (const file of existingCWL.files) {
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "❗" : "❌"} |\n`;
tableContent += `| ${file.path} | ${file.validation_status === "valid" ? "✔️" : "❌"} |\n`;
subjects.cwl.files.push(file);

if (file.validation_status === "invalid") {
Expand All @@ -390,7 +413,26 @@ export async function applyCWLTemplate(
}

const cwlBadge = `[![CWL](https://img.shields.io/badge/View_CWL_Report-0ea5e9.svg)](${url})`;
baseTemplate += `\n\n## CWL Validations ${validOverall ? "✔️" : "❌"}\n\nCWL files were found in the repository and ***${subjects.cwl.files.length - failedCount}/${subjects.cwl.files.length}*** are considered valid by the [cwltool validator](https://cwltool.readthedocs.io/en/latest/).\n\n<details>\n<summary>Summary of the validation report</summary>\n\n| File | Status |\n| :---- | :----: |\n${tableContent}</details>\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`;
baseTemplate += `\n\n## CWL Validations ${validOverall ? "✔️" : "❗"}\n\nCWL files were found in the repository and ***${subjects.cwl.files.length - failedCount}/${subjects.cwl.files.length}*** are considered valid by the [cwltool validator](https://cwltool.readthedocs.io/en/latest/).\n\n<details>\n<summary>Summary of the validation report</summary>\n\n| File | Status |\n| :---- | :----: |\n${tableContent}</details>\n\nTo view the full report of each CWL file or to rerun the validation, click the "View CWL Report" button below.\n\n${cwlBadge}`;
}

if (subjects.cwl.removed_files.length > 0) {
// Remove the files from the database
const cwlCollection = dbInstance.getDb().collection("cwlValidation");
const existingCWL = await cwlCollection.findOne({
repositoryId: repository.id,
});

if (existingCWL) {
const newFiles = existingCWL.files.filter((file) => {
return !subjects.cwl.removed_files.includes(file.path);
});

await cwlCollection.updateOne(
{ repositoryId: repository.id },
{ $set: { files: newFiles } },
);
}
}

return baseTemplate;
Expand Down