diff --git a/lib/config.js b/lib/config.js index 82cfee17..8f39a25c 100644 --- a/lib/config.js +++ b/lib/config.js @@ -32,6 +32,52 @@ const DEFAULT_CONFIG = { 'swift', 'typescript' ], + tags: [ + 'array', + 'backtracking', + 'binary-indexed-tree', + 'binary-search', + 'binary-search-tree', + 'bit-manipulation', + 'brainteaser', + 'breadth-first-search', + 'depth-first-search', + 'design', + 'divide-and-conquer', + 'dynamic-programming', + 'geometry', + 'graph', + 'greedy', + 'hash-table', + 'heap', + 'line-sweep', + 'linked-list', + 'math', + 'memoization', + 'minimax', + 'ordered-map', + 'queue', + 'random', + 'recursion', + 'rejection-sampling', + 'reservoir-sampling', + 'segment-tree', + 'sliding-window', + 'sort', + 'stack', + 'string', + 'topological-sort', + 'tree', + 'trie', + 'two-pointers', + 'union-find', + + // TODO: these two tags below are included on leetcode.com but not on leetcode-cn.com + // Currently comment out for simplicity + + // 'rolling-hash', + // 'suffix-array', + ], urls: { // base urls base: 'https://leetcode.com', @@ -51,6 +97,7 @@ const DEFAULT_CONFIG = { linkedin_login_request: 'https://www.linkedin.com/login', linkedin_session_request: 'https://www.linkedin.com/checkpoint/lg/login-submit', // questions urls + tag: 'https://leetcode.com/tag/$tag/', problems: 'https://leetcode.com/api/problems/$category/', problem: 'https://leetcode.com/problems/$slug/description/', test: 'https://leetcode.com/problems/$slug/interpret_solution/', diff --git a/lib/plugins/company.js b/lib/plugins/company.js index 7959363b..52a641a4 100644 --- a/lib/plugins/company.js +++ b/lib/plugins/company.js @@ -1520,9 +1520,6 @@ plugin.getProblems = function(cb) { if (id in COMPONIES) { problem.companies = (problem.companies || []).concat(COMPONIES[id]); } - if (id in TAGS) { - problem.tags = (problem.tags || []).concat(TAGS[id]); - } }); return cb(null, problems); }); diff --git a/lib/plugins/leetcode.js b/lib/plugins/leetcode.js index c403582d..4afc2a3e 100644 --- a/lib/plugins/leetcode.js +++ b/lib/plugins/leetcode.js @@ -3,6 +3,7 @@ var util = require('util'); var _ = require('underscore'); var request = require('request'); +var { orderBy } = require('natural-orderby'); var prompt = require('prompt'); var config = require('../config'); @@ -54,15 +55,14 @@ plugin.init = function() { config.app = 'leetcode'; }; -plugin.getProblems = function(cb) { - log.debug('running leetcode.getProblems'); +plugin.getProblemsWithoutTags = function(cb) { let problems = []; const getCategory = function(category, queue, cb) { plugin.getCategoryProblems(category, function(e, _problems) { if (e) { - log.debug(category + ': failed to getProblems: ' + e.msg); + log.debug(category + ': failed to getCategory: ' + e.msg); } else { - log.debug(category + ': getProblems got ' + _problems.length + ' problems'); + log.debug(category + ': getCategory got ' + _problems.length + ' problems'); problems = problems.concat(_problems); } return cb(e); @@ -77,6 +77,68 @@ plugin.getProblems = function(cb) { }); }; +plugin.getProblems = function(cb) { + log.debug('running leetcode.getProblems'); + plugin.getProblemsWithoutTags(function(e, problems) { + if (e) return cb(e); + problems = new Map(problems.map(p => [p.id, p])); + const getTag = function (tag, queue, cb) { + plugin.getTagProblems(tag, function(e, _problems) { + if (e) { + log.debug(tag + ': failed to getTag: ' + JSON.stringify(e)); + } else if (!_problems) { + log.debug(tag + ': retrieve empty tag'); + } else { + log.debug(tag + ': getTag got ' + _problems.length + ' problems'); + _problems.forEach(function (p) { + let id = parseInt(p.questionId); + if (problems.has(id)) { + problems.get(id).tags.push(tag); + } + }) + } + return cb(e); + }); + }; + spin = h.spin('Downloading tags'); + const q = new Queue(config.sys.tags, {}, getTag); + q.run(null, function (e) { + spin.stop(); + problems = orderBy(Array.from(problems.values()), [p => p.fid], ['asc']); + return cb(e, problems); + }); + }); +} + +plugin.getTagProblems = function(tag, cb) { + log.debug('running leetcode.getTagProblems: ' + tag); + const opts = plugin.makeOpts(config.sys.urls.graphql); + opts.headers.Origin = config.sys.urls.base; + opts.headers.Referer = config.sys.urls.tag.replace('$tag', tag); + opts.json = true; + opts.body = { + query: [ + 'query getTopicTag($slug: String!) {', + ' topicTag(slug: $slug) {', + ' slug', + ' questions {', + ' questionId', + ' }', + ' }', + '}' + ].join('\n'), + variables: {slug: tag}, + operationName: 'getTopicTag' + }; + spin.text = 'Downloading tag ' + tag; + request.post(opts, function(e, resp, body) { + e = plugin.checkError(e, resp, 200); + if (e) return cb(e); + if(!body || !body.data || !body.data.topicTag) return cb('receive invalid response body'); + return cb(null, body.data.topicTag.questions); + }); +} + plugin.getCategoryProblems = function(category, cb) { log.debug('running leetcode.getCategoryProblems: ' + category); const opts = plugin.makeOpts(config.sys.urls.problems.replace('$category', category)); @@ -109,7 +171,8 @@ plugin.getCategoryProblems = function(category, cb) { percent: p.stat.total_acs * 100 / p.stat.total_submitted, level: h.levelToName(p.difficulty.level), starred: p.is_favor, - category: json.category_slug + category: json.category_slug, + tags: [] }; }); diff --git a/package-lock.json b/package-lock.json index 607ea460..23028703 100644 --- a/package-lock.json +++ b/package-lock.json @@ -2871,6 +2871,11 @@ "integrity": "sha1-Sr6/7tdUHywnrPspvbvRXI1bpPc=", "dev": true }, + "natural-orderby": { + "version": "2.0.3", + "resolved": "https://registry.npmjs.org/natural-orderby/-/natural-orderby-2.0.3.tgz", + "integrity": "sha512-p7KTHxU0CUrcOXe62Zfrb5Z13nLvPhSWR/so3kFulUQU0sgUll2Z0LwpsLN351eOOD+hRGu/F1g+6xDfPeD++Q==" + }, "nconf": { "version": "0.10.0", "resolved": "https://registry.npmjs.org/nconf/-/nconf-0.10.0.tgz", diff --git a/package.json b/package.json index 71aff1a8..ea1c6b76 100644 --- a/package.json +++ b/package.json @@ -54,6 +54,7 @@ "he": "1.2.0", "mkdirp": "^1.0.4", "moment": "^2.20.1", + "natural-orderby": "^2.0.3", "nconf": "0.10.0", "ora": "3.0.0", "prompt": "1.0.0", diff --git a/test/mock/tags.json.20200911 b/test/mock/tags.json.20200911 new file mode 100644 index 00000000..b68e4b98 --- /dev/null +++ b/test/mock/tags.json.20200911 @@ -0,0 +1 @@ +{"data":{"topicTag":{"slug":"array","questions":[{"questionId":"1"},{"questionId":"4"},{"questionId":"11"},{"questionId":"15"},{"questionId":"16"},{"questionId":"18"},{"questionId":"26"},{"questionId":"27"},{"questionId":"31"},{"questionId":"33"},{"questionId":"34"},{"questionId":"35"},{"questionId":"39"},{"questionId":"40"},{"questionId":"41"},{"questionId":"42"},{"questionId":"45"},{"questionId":"48"},{"questionId":"53"},{"questionId":"54"},{"questionId":"55"},{"questionId":"56"},{"questionId":"57"},{"questionId":"59"},{"questionId":"62"},{"questionId":"63"},{"questionId":"64"},{"questionId":"66"},{"questionId":"73"},{"questionId":"74"},{"questionId":"75"},{"questionId":"78"},{"questionId":"79"},{"questionId":"80"},{"questionId":"81"},{"questionId":"84"},{"questionId":"85"},{"questionId":"88"},{"questionId":"90"},{"questionId":"105"},{"questionId":"106"},{"questionId":"118"},{"questionId":"119"},{"questionId":"120"},{"questionId":"121"},{"questionId":"122"},{"questionId":"123"},{"questionId":"126"},{"questionId":"128"},{"questionId":"152"},{"questionId":"153"},{"questionId":"154"},{"questionId":"162"},{"questionId":"163"},{"questionId":"167"},{"questionId":"169"},{"questionId":"189"},{"questionId":"209"},{"questionId":"216"},{"questionId":"217"},{"questionId":"219"},{"questionId":"228"},{"questionId":"229"},{"questionId":"238"},{"questionId":"243"},{"questionId":"245"},{"questionId":"259"},{"questionId":"268"},{"questionId":"277"},{"questionId":"280"},{"questionId":"283"},{"questionId":"287"},{"questionId":"289"},{"questionId":"370"},{"questionId":"380"},{"questionId":"381"},{"questionId":"414"},{"questionId":"442"},{"questionId":"448"},{"questionId":"457"},{"questionId":"485"},{"questionId":"495"},{"questionId":"531"},{"questionId":"532"},{"questionId":"533"},{"questionId":"548"},{"questionId":"560"},{"questionId":"561"},{"questionId":"562"},{"questionId":"565"},{"questionId":"566"},{"questionId":"581"},{"questionId":"605"},{"questionId":"611"},{"questionId":"621"},{"questionId":"624"},{"questionId":"628"},{"questionId":"643"},{"questionId":"644"},{"questionId":"661"},{"questionId":"665"},{"questionId":"667"},{"questionId":"670"},{"questionId":"674"},{"questionId":"689"},{"questionId":"695"},{"questionId":"697"},{"questionId":"713"},{"questionId":"714"},{"questionId":"717"},{"questionId":"718"},{"questionId":"719"},{"questionId":"723"},{"questionId":"724"},{"questionId":"729"},{"questionId":"747"},{"questionId":"748"},{"questionId":"756"},{"questionId":"777"},{"questionId":"779"},{"questionId":"780"},{"questionId":"790"},{"questionId":"798"},{"questionId":"808"},{"questionId":"811"},{"questionId":"852"},{"questionId":"857"},{"questionId":"861"},{"questionId":"864"},{"questionId":"870"},{"questionId":"879"},{"questionId":"898"},{"questionId":"901"},{"questionId":"905"},{"questionId":"924"},{"questionId":"927"},{"questionId":"932"},{"questionId":"936"},{"questionId":"941"},{"questionId":"943"},{"questionId":"950"},{"questionId":"951"},{"questionId":"954"},{"questionId":"958"},{"questionId":"962"},{"questionId":"978"},{"questionId":"982"},{"questionId":"987"},{"questionId":"991"},{"questionId":"1002"},{"questionId":"1009"},{"questionId":"1013"},{"questionId":"1016"},{"questionId":"1019"},{"questionId":"1020"},{"questionId":"1027"},{"questionId":"1031"},{"questionId":"1041"},{"questionId":"1044"},{"questionId":"1049"},{"questionId":"1055"},{"questionId":"1056"},{"questionId":"1062"},{"questionId":"1063"},{"questionId":"1066"},{"questionId":"1071"},{"questionId":"1074"},{"questionId":"1082"},{"questionId":"1083"},{"questionId":"1096"},{"questionId":"1098"},{"questionId":"1102"},{"questionId":"1105"},{"questionId":"1107"},{"questionId":"1108"},{"questionId":"1112"},{"questionId":"1113"},{"questionId":"1137"},{"questionId":"1138"},{"questionId":"1139"},{"questionId":"1145"},{"questionId":"1168"},{"questionId":"1175"},{"questionId":"1206"},{"questionId":"1217"},{"questionId":"1221"},{"questionId":"1227"},{"questionId":"1231"},{"questionId":"1232"},{"questionId":"1241"},{"questionId":"1247"},{"questionId":"1249"},{"questionId":"1253"},{"questionId":"1255"},{"questionId":"1256"},{"questionId":"1262"},{"questionId":"1272"},{"questionId":"1273"},{"questionId":"1280"},{"questionId":"1281"},{"questionId":"1287"},{"questionId":"1289"},{"questionId":"1293"},{"questionId":"1306"},{"questionId":"1308"},{"questionId":"1321"},{"questionId":"1329"},{"questionId":"1342"},{"questionId":"1345"},{"questionId":"1349"},{"questionId":"1350"},{"questionId":"1374"},{"questionId":"1378"},{"questionId":"1386"},{"questionId":"1391"},{"questionId":"1395"},{"questionId":"1396"},{"questionId":"1400"},{"questionId":"1402"},{"questionId":"1413"},{"questionId":"1421"},{"questionId":"1422"},{"questionId":"1426"},{"questionId":"1445"},{"questionId":"1455"},{"questionId":"1463"},{"questionId":"1464"},{"questionId":"1468"},{"questionId":"1476"},{"questionId":"1477"},{"questionId":"1482"},{"questionId":"1483"},{"questionId":"1486"},{"questionId":"1487"},{"questionId":"1491"},{"questionId":"1496"},{"questionId":"1500"},{"questionId":"1505"},{"questionId":"1510"},{"questionId":"1511"},{"questionId":"1514"},{"questionId":"1515"},{"questionId":"1525"},{"questionId":"1528"},{"questionId":"1538"},{"questionId":"1539"},{"questionId":"1548"},{"questionId":"1549"},{"questionId":"1553"},{"questionId":"1556"},{"questionId":"1560"},{"questionId":"1570"},{"questionId":"1572"},{"questionId":"1574"},{"questionId":"1575"},{"questionId":"1580"},{"questionId":"1581"},{"questionId":"1584"},{"questionId":"1586"},{"questionId":"1603"},{"questionId":"1604"},{"questionId":"1605"},{"questionId":"1610"},{"questionId":"1612"},{"questionId":"1615"},{"questionId":"1616"},{"questionId":"1620"},{"questionId":"1622"},{"questionId":"1626"},{"questionId":"1627"},{"questionId":"1631"},{"questionId":"1635"},{"questionId":"1640"},{"questionId":"1646"},{"questionId":"1656"},{"questionId":"1657"},{"questionId":"1675"},{"questionId":"1677"},{"questionId":"1679"},{"questionId":"1682"},{"questionId":"1689"},{"questionId":"1713"}]}}} \ No newline at end of file diff --git a/test/plugins/test_leetcode.js b/test/plugins/test_leetcode.js index 39a1431e..3f896efb 100644 --- a/test/plugins/test_leetcode.js +++ b/test/plugins/test_leetcode.js @@ -124,10 +124,15 @@ describe('plugin:leetcode', function() { nock('https://leetcode.com') .get('/api/problems/concurrency/') .replyWithFile(200, './test/mock/problems.json.20160911'); - + + nock('https://leetcode.com') + .post('/graphql') + .times(config.sys.tags.length) + .replyWithFile(200, './test/mock/tags.json.20200911'); + plugin.getProblems(function(e, problems) { assert.equal(e, null); - assert.equal(problems.length, 377 * 4); + assert.equal(problems.length, 377); done(); }); });