From 3e9270b7ebc438ba15497a946ac0b5105d7e5cc3 Mon Sep 17 00:00:00 2001 From: rory Date: Fri, 7 Jul 2023 15:36:01 -0700 Subject: [PATCH 1/8] Make the same files without crazy git diff --- package-lock.json | 13 +++ package.json | 1 + scripts/AggregateGitHubDataFromUpwork.js | 105 +++++++++++++++++++++++ 3 files changed, 119 insertions(+) create mode 100644 scripts/AggregateGitHubDataFromUpwork.js diff --git a/package-lock.json b/package-lock.json index bc8aade90b76..cfe6e401c585 100644 --- a/package-lock.json +++ b/package-lock.json @@ -142,6 +142,7 @@ "concurrently": "^5.3.0", "copy-webpack-plugin": "^6.4.1", "css-loader": "^6.7.2", + "csv-writer": "^1.6.0", "diff-so-fancy": "^1.3.0", "dotenv": "^16.0.3", "electron": "22.3.14", @@ -21382,6 +21383,12 @@ "version": "3.1.1", "license": "MIT" }, + "node_modules/csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==", + "dev": true + }, "node_modules/currently-unhandled": { "version": "0.4.1", "dev": true, @@ -58148,6 +58155,12 @@ "csstype": { "version": "3.1.1" }, + "csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==", + "dev": true + }, "currently-unhandled": { "version": "0.4.1", "dev": true, diff --git a/package.json b/package.json index 9ccdca875ddc..42c8f197dac1 100644 --- a/package.json +++ b/package.json @@ -179,6 +179,7 @@ "concurrently": "^5.3.0", "copy-webpack-plugin": "^6.4.1", "css-loader": "^6.7.2", + "csv-writer": "^1.6.0", "diff-so-fancy": "^1.3.0", "dotenv": "^16.0.3", "electron": "22.3.14", diff --git a/scripts/AggregateGitHubDataFromUpwork.js b/scripts/AggregateGitHubDataFromUpwork.js new file mode 100644 index 000000000000..21d991e1a545 --- /dev/null +++ b/scripts/AggregateGitHubDataFromUpwork.js @@ -0,0 +1,105 @@ +/* + * To run this script from the root of E/App: + * + * node ./scripts/AggregateGitHubDataFromUpwork.js + */ + +/* eslint-disable no-console, @lwc/lwc/no-async-await, no-restricted-syntax, no-await-in-loop */ +const _ = require('underscore'); +const fs = require('fs'); +const {GitHub, getOctokitOptions} = require('@actions/github/lib/utils'); +const {throttling} = require('@octokit/plugin-throttling'); +const {paginateRest} = require('@octokit/plugin-paginate-rest'); +const createCsvWriter = require('csv-writer').createObjectCsvWriter; + +const csvWriter = createCsvWriter({ + path: 'output.csv', + header: [ + {id: 'number', title: 'number'}, + {id: 'title', title: 'title'}, + {id: 'labels', title: 'labels'}, + {id: 'type', title: 'type'}, + ], +}); + +if (process.argv.length < 3) { + throw new Error('Error: must provide filepath for CSV data'); +} + +if (process.argv.length < 4) { + throw new Error('Error: must provide GitHub token'); +} + +// Get filepath for csv +const filepath = process.argv[2]; + +// Get data from csv +let issues = _.filter(fs.readFileSync(filepath).toString().split('\n'), (issue) => !_.isEmpty(issue)); + +// Skip header row +issues = issues.slice(1); + +// Get GitHub token +const token = process.argv[3].trim(); +const Octokit = GitHub.plugin(throttling, paginateRest); +const octokit = new Octokit( + getOctokitOptions(token, { + throttle: { + onRateLimit: (retryAfter, options) => { + console.warn(`Request quota exhausted for request ${options.method} ${options.url}`); + + // Retry once after hitting a rate limit error, then give up + if (options.request.retryCount <= 1) { + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + } + }, + onAbuseLimit: (retryAfter, options) => { + // does not retry, only logs a warning + console.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, + }), +).rest; + +function getType(labels) { + if (_.contains(labels, 'Bug')) { + return 'bug'; + } + if (_.contains(labels, 'NewFeature')) { + return 'feature'; + } + return 'other'; +} + +async function getGitHubData() { + const gitHubData = []; + for (const issueNumber of issues) { + const num = issueNumber.trim(); + console.info(`Fetching ${num}`); + const result = await octokit.issues + .get({ + owner: 'Expensify', + repo: 'App', + issue_number: num, + }) + .catch(() => { + console.warn(`Error getting issue ${num}`); + }); + if (result) { + const issue = result.data; + const labels = _.map(issue.labels, (label) => label.name); + gitHubData.push({ + number: issue.number, + title: issue.title, + labels, + type: getType(labels), + }); + } + } + return gitHubData; +} + +getGitHubData() + .then((gitHubData) => csvWriter.writeRecords(gitHubData)) + .then(() => console.info('Done ✅ Wrote file to output.csv')); From cb1be523c0d5e8f139a29bd482add784decac732 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 9 Apr 2024 11:42:57 -0700 Subject: [PATCH 2/8] Add project check --- scripts/AggregateGitHubDataFromUpwork.js | 28 ++++++++++++++++++++++-- 1 file changed, 26 insertions(+), 2 deletions(-) diff --git a/scripts/AggregateGitHubDataFromUpwork.js b/scripts/AggregateGitHubDataFromUpwork.js index 21d991e1a545..9409e430c5ac 100644 --- a/scripts/AggregateGitHubDataFromUpwork.js +++ b/scripts/AggregateGitHubDataFromUpwork.js @@ -60,7 +60,7 @@ const octokit = new Octokit( }, }, }), -).rest; +); function getType(labels) { if (_.contains(labels, 'Bug')) { @@ -77,7 +77,7 @@ async function getGitHubData() { for (const issueNumber of issues) { const num = issueNumber.trim(); console.info(`Fetching ${num}`); - const result = await octokit.issues + const result = await octokit.rest.issues .get({ owner: 'Expensify', repo: 'App', @@ -89,11 +89,35 @@ async function getGitHubData() { if (result) { const issue = result.data; const labels = _.map(issue.labels, (label) => label.name); + const type = getType(labels); + let capSWProjects = []; + if (type === 'NewFeature') { + // eslint-disable-next-line rulesdir/prefer-underscore-method + capSWProjects = await octokit + .graphql( + ` + { + repository(owner: "Expensify", name: "App") { + issue(number: 39322) { + projectsV2(last: 30) { + nodes { + title + } + } + } + } + } + `, + ) + .repository.issue.projectsV2.nodes.map((node) => node.title) + .join(','); + } gitHubData.push({ number: issue.number, title: issue.title, labels, type: getType(labels), + capSWProjects, }); } } From 97b6c82050ab3abc1e0372ececfb37b2b2893c44 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 9 Jul 2024 09:47:07 -0700 Subject: [PATCH 3/8] Update package-lock.json --- package-lock.json | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/package-lock.json b/package-lock.json index 36cb2639ec4f..276408dcbee1 100644 --- a/package-lock.json +++ b/package-lock.json @@ -209,6 +209,7 @@ "copy-webpack-plugin": "^10.1.0", "css-loader": "^6.7.2", "csv-parse": "^5.5.5", + "csv-writer": "^1.6.0", "diff-so-fancy": "^1.3.0", "dotenv": "^16.0.3", "electron": "^29.4.1", @@ -23198,6 +23199,12 @@ "integrity": "sha512-erCk7tyU3yLWAhk6wvKxnyPtftuy/6Ak622gOO7BCJ05+TYffnPCJF905wmOQm+BpkX54OdAl8pveJwUdpnCXQ==", "dev": true }, + "node_modules/csv-writer": { + "version": "1.6.0", + "resolved": "https://registry.npmjs.org/csv-writer/-/csv-writer-1.6.0.tgz", + "integrity": "sha512-NOx7YDFWEsM/fTRAJjRpPp8t+MKRVvniAg9wQlUKx20MFrPs73WLJhFf5iteqrxNYnsy924K3Iroh3yNHeYd2g==", + "dev": true + }, "node_modules/dag-map": { "version": "1.0.2", "license": "MIT" From 46a6c2028386599089db2a63bafc02d9df872f29 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 1 Oct 2024 13:22:27 -0700 Subject: [PATCH 4/8] Convert script to TS --- scripts/aggregateGitHubDataFromUpwork.ts | 171 +++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 scripts/aggregateGitHubDataFromUpwork.ts diff --git a/scripts/aggregateGitHubDataFromUpwork.ts b/scripts/aggregateGitHubDataFromUpwork.ts new file mode 100644 index 000000000000..f516ee16f33e --- /dev/null +++ b/scripts/aggregateGitHubDataFromUpwork.ts @@ -0,0 +1,171 @@ +/** + * This script is used for categorizing upwork costs into cost buckets for accounting purposes. + * + * To run this script from the root of E/App: + * + * ts-node ./scripts/aggregateGitHubDataFromUpwork.js + * + * The input file must be a CSV with a single column containing just the GitHub issue number. The CSV must have a single header row. + */ +import {getOctokitOptions, GitHub} from '@actions/github/lib/utils'; +import {paginateRest} from '@octokit/plugin-paginate-rest'; +import {throttling} from '@octokit/plugin-throttling'; +import {createObjectCsvWriter} from 'csv-writer'; +import fs from 'fs'; +import CONST from '../.github/libs/CONST'; + +type OctokitOptions = {method: string; url: string; request: {retryCount: number}}; +type IssueType = 'bug' | 'feature' | 'other'; + +if (process.argv.length < 3) { + throw new Error('Error: must provide filepath for CSV data'); +} + +if (process.argv.length < 4) { + throw new Error('Error: must provide GitHub token'); +} + +if (process.argv.length < 5) { + throw new Error('Error: must provide output file path'); +} + +// Get filepath for csv +const inputFilepath = process.argv.at(2); +if (!inputFilepath) { + throw new Error('Error: must provide filepath for CSV data'); +} + +// Get GitHub token +const token = (process.argv.at(3) ?? '').trim(); +if (!token) { + throw new Error('Error: must provide GitHub token'); +} + +const Octokit = GitHub.plugin(throttling, paginateRest); +const octokit = new Octokit( + getOctokitOptions(token, { + throttle: { + onRateLimit: (retryAfter: number, options: OctokitOptions) => { + console.warn(`Request quota exhausted for request ${options.method} ${options.url}`); + + // Retry once after hitting a rate limit error, then give up + if (options.request.retryCount <= 1) { + console.log(`Retrying after ${retryAfter} seconds!`); + return true; + } + }, + onAbuseLimit: (retryAfter: number, options: OctokitOptions) => { + // does not retry, only logs a warning + console.warn(`Abuse detected for request ${options.method} ${options.url}`); + }, + }, + }), +); + +// Get output filepath +const outputFilepath = process.argv.at(4); +if (!outputFilepath) { + throw new Error('Error: must provide output file path'); +} + +// Get data from csv +const issues = fs + .readFileSync(inputFilepath) + .toString() + .split('\n') + .reduce((acc, issue) => { + if (!issue) { + return acc; + } + acc.push(Number(issue.trim())); + return acc; + }, [] as number[]); + +const csvWriter = createObjectCsvWriter({ + path: outputFilepath, + header: [ + {id: 'number', title: 'number'}, + {id: 'title', title: 'title'}, + {id: 'labels', title: 'labels'}, + {id: 'type', title: 'type'}, + ], +}); + +function getIssueTypeFromLabels(labels: string[]): IssueType { + if (labels.includes('NewFeature')) { + return 'feature'; + } + if (labels.includes('Bug')) { + return 'bug'; + } + return 'other'; +} + +/** + * Returns a comma-delimited string with all projects associated with the given issue. + */ +async function getProjectsForIssue(issueNumber: number): Promise { + const response = await octokit.graphql( + ` + { + repository(owner: "Expensify", name: "App") { + issue(number: ${issueNumber}) { + projectsV2(last: 30) { + nodes { + title + } + } + } + } + } + `, + ); + return (response as {repository: {issue: {projectsV2: {nodes: Array<{title: string}>}}}}).repository.issue.projectsV2.nodes.map((node) => node.title).join(','); +} + +async function getGitHubData() { + const gitHubData = []; + // Note: we fetch issues in a loop rather than in parallel to help address rate limiting issues with a PAT + for (const issueNumber of issues) { + console.info(`Fetching ${issueNumber}`); + const result = await octokit.rest.issues + .get({ + owner: CONST.GITHUB_OWNER, + repo: CONST.APP_REPO, + // eslint-disable-next-line @typescript-eslint/naming-convention + issue_number: issueNumber, + }) + .catch(() => { + console.warn(`Error getting issue ${issueNumber}`); + }); + if (result) { + const issue = result.data; + const labels = issue.labels.reduce((acc, label) => { + if (typeof label === 'string') { + acc.push(label); + } else if (label.name) { + acc.push(label.name); + } + return acc; + }, [] as string[]); + const type = getIssueTypeFromLabels(labels); + let capSWProjects = ''; + if (type === 'feature') { + // eslint-disable-next-line @typescript-eslint/no-unsafe-call + capSWProjects = await getProjectsForIssue(issueNumber); + } + gitHubData.push({ + number: issue.number, + title: issue.title, + labels, + type, + capSWProjects, + }); + } + } + return gitHubData; +} + +getGitHubData() + .then((gitHubData) => csvWriter.writeRecords(gitHubData)) + .then(() => console.info(`Done ✅ Wrote file to ${outputFilepath}`)); From 5a29fa333b59fc475f4393c81b2b1fa477aefad4 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 1 Oct 2024 13:22:53 -0700 Subject: [PATCH 5/8] Remove JS script --- scripts/AggregateGitHubDataFromUpwork.js | 129 ----------------------- 1 file changed, 129 deletions(-) delete mode 100644 scripts/AggregateGitHubDataFromUpwork.js diff --git a/scripts/AggregateGitHubDataFromUpwork.js b/scripts/AggregateGitHubDataFromUpwork.js deleted file mode 100644 index 9409e430c5ac..000000000000 --- a/scripts/AggregateGitHubDataFromUpwork.js +++ /dev/null @@ -1,129 +0,0 @@ -/* - * To run this script from the root of E/App: - * - * node ./scripts/AggregateGitHubDataFromUpwork.js - */ - -/* eslint-disable no-console, @lwc/lwc/no-async-await, no-restricted-syntax, no-await-in-loop */ -const _ = require('underscore'); -const fs = require('fs'); -const {GitHub, getOctokitOptions} = require('@actions/github/lib/utils'); -const {throttling} = require('@octokit/plugin-throttling'); -const {paginateRest} = require('@octokit/plugin-paginate-rest'); -const createCsvWriter = require('csv-writer').createObjectCsvWriter; - -const csvWriter = createCsvWriter({ - path: 'output.csv', - header: [ - {id: 'number', title: 'number'}, - {id: 'title', title: 'title'}, - {id: 'labels', title: 'labels'}, - {id: 'type', title: 'type'}, - ], -}); - -if (process.argv.length < 3) { - throw new Error('Error: must provide filepath for CSV data'); -} - -if (process.argv.length < 4) { - throw new Error('Error: must provide GitHub token'); -} - -// Get filepath for csv -const filepath = process.argv[2]; - -// Get data from csv -let issues = _.filter(fs.readFileSync(filepath).toString().split('\n'), (issue) => !_.isEmpty(issue)); - -// Skip header row -issues = issues.slice(1); - -// Get GitHub token -const token = process.argv[3].trim(); -const Octokit = GitHub.plugin(throttling, paginateRest); -const octokit = new Octokit( - getOctokitOptions(token, { - throttle: { - onRateLimit: (retryAfter, options) => { - console.warn(`Request quota exhausted for request ${options.method} ${options.url}`); - - // Retry once after hitting a rate limit error, then give up - if (options.request.retryCount <= 1) { - console.log(`Retrying after ${retryAfter} seconds!`); - return true; - } - }, - onAbuseLimit: (retryAfter, options) => { - // does not retry, only logs a warning - console.warn(`Abuse detected for request ${options.method} ${options.url}`); - }, - }, - }), -); - -function getType(labels) { - if (_.contains(labels, 'Bug')) { - return 'bug'; - } - if (_.contains(labels, 'NewFeature')) { - return 'feature'; - } - return 'other'; -} - -async function getGitHubData() { - const gitHubData = []; - for (const issueNumber of issues) { - const num = issueNumber.trim(); - console.info(`Fetching ${num}`); - const result = await octokit.rest.issues - .get({ - owner: 'Expensify', - repo: 'App', - issue_number: num, - }) - .catch(() => { - console.warn(`Error getting issue ${num}`); - }); - if (result) { - const issue = result.data; - const labels = _.map(issue.labels, (label) => label.name); - const type = getType(labels); - let capSWProjects = []; - if (type === 'NewFeature') { - // eslint-disable-next-line rulesdir/prefer-underscore-method - capSWProjects = await octokit - .graphql( - ` - { - repository(owner: "Expensify", name: "App") { - issue(number: 39322) { - projectsV2(last: 30) { - nodes { - title - } - } - } - } - } - `, - ) - .repository.issue.projectsV2.nodes.map((node) => node.title) - .join(','); - } - gitHubData.push({ - number: issue.number, - title: issue.title, - labels, - type: getType(labels), - capSWProjects, - }); - } - } - return gitHubData; -} - -getGitHubData() - .then((gitHubData) => csvWriter.writeRecords(gitHubData)) - .then(() => console.info('Done ✅ Wrote file to output.csv')); From c5356b1d3603a8a25091493d2dabb909dd676452 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 1 Oct 2024 13:37:35 -0700 Subject: [PATCH 6/8] Include capSWProjects in CSV writer --- scripts/aggregateGitHubDataFromUpwork.ts | 1 + 1 file changed, 1 insertion(+) diff --git a/scripts/aggregateGitHubDataFromUpwork.ts b/scripts/aggregateGitHubDataFromUpwork.ts index f516ee16f33e..348362abe83c 100644 --- a/scripts/aggregateGitHubDataFromUpwork.ts +++ b/scripts/aggregateGitHubDataFromUpwork.ts @@ -88,6 +88,7 @@ const csvWriter = createObjectCsvWriter({ {id: 'title', title: 'title'}, {id: 'labels', title: 'labels'}, {id: 'type', title: 'type'}, + {id: 'capSWProjects', title: 'capSWProjects'}, ], }); From eff80cb24fca4294dd7f2f7c12e05401cffba791 Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 1 Oct 2024 13:40:17 -0700 Subject: [PATCH 7/8] Remove CONST which isn't working for some reason --- scripts/aggregateGitHubDataFromUpwork.ts | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/scripts/aggregateGitHubDataFromUpwork.ts b/scripts/aggregateGitHubDataFromUpwork.ts index 348362abe83c..dd82dcb299ac 100644 --- a/scripts/aggregateGitHubDataFromUpwork.ts +++ b/scripts/aggregateGitHubDataFromUpwork.ts @@ -12,7 +12,6 @@ import {paginateRest} from '@octokit/plugin-paginate-rest'; import {throttling} from '@octokit/plugin-throttling'; import {createObjectCsvWriter} from 'csv-writer'; import fs from 'fs'; -import CONST from '../.github/libs/CONST'; type OctokitOptions = {method: string; url: string; request: {retryCount: number}}; type IssueType = 'bug' | 'feature' | 'other'; @@ -131,8 +130,8 @@ async function getGitHubData() { console.info(`Fetching ${issueNumber}`); const result = await octokit.rest.issues .get({ - owner: CONST.GITHUB_OWNER, - repo: CONST.APP_REPO, + owner: 'Expensify', + repo: 'App', // eslint-disable-next-line @typescript-eslint/naming-convention issue_number: issueNumber, }) From b645d20b1d6ed59feb18a0d7e3e1b806665d079d Mon Sep 17 00:00:00 2001 From: rory Date: Tue, 1 Oct 2024 21:35:34 -0700 Subject: [PATCH 8/8] Handle NaN from header row --- scripts/aggregateGitHubDataFromUpwork.ts | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/scripts/aggregateGitHubDataFromUpwork.ts b/scripts/aggregateGitHubDataFromUpwork.ts index dd82dcb299ac..f47b2b43e5cc 100644 --- a/scripts/aggregateGitHubDataFromUpwork.ts +++ b/scripts/aggregateGitHubDataFromUpwork.ts @@ -76,7 +76,11 @@ const issues = fs if (!issue) { return acc; } - acc.push(Number(issue.trim())); + const issueNum = Number(issue.trim()); + if (!issueNum) { + return acc; + } + acc.push(issueNum); return acc; }, [] as number[]);