feat(route): add nber route: NBER working paper

docs/en/journal.md

@@ -313,6 +313,20 @@ Only some journals are supported.
 
 <RouteEn author="TonyRL" example="/nature/siteindex" path="/nature/siteindex"/>
 
+## National Bureau of Economic Research (NBER) Working Papers
+
+### All Papers
+
+<Route author="5upernova-heng" example="/nber/papers/20" path="/nber/papers/:perPage" :paramsDesc=['paper numbers per page']/>
+
+-   Page size should be one of 20, 50, 100. By default the number is 50.
+
+### New Papers
+
+<Route author="5upernova-heng" example="/nber/news" path="/nber/news"/>
+
+Papers that are labeled by "new" at the website.
+
 ## Network and Distributed System Security (NDSS) Symposium
 
 ### Accepted papers

docs/journal.md

@@ -284,6 +284,20 @@ You can get all short name of a journal from <https://www.nature.com/siteindex>
 
 <Route author="TonyRL" example="/nature/siteindex" path="/nature/siteindex"/>
 
+## National Bureau of Economic Research (NBER) Working Papers
+
+### 全部论文
+
+<Route author="5upernova-heng" example="/nber/papers/20" path="/nber/papers/:perPage" :paramsDesc=['paper numbers per page']/>
+
+- 页面文章数量必须为以下值之一：20，50，100. 默认为 50
+
+### 新论文
+
+<Route author="5upernova-heng" example="/nber/news" path="/nber/news"/>
+
+在网站上被标记为 "new" 的论文
+
 ## Network and Distributed System Security (NDSS) Symposium
 
 ### Accepted papers

lib/v2/nber/maintainer.js

@@ -0,0 +1,4 @@
+module.exports = {
+    '/articles/:perPage?': ['5upernova-heng'],
+    '/news': ['5upernova-heng'],
+};

lib/v2/nber/news.js

@@ -0,0 +1,39 @@
+const { get_html, get_elements } = require('./utils');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    url = `https://www.nber.org/papers?page=1&perPage=20&sortBy=public_date`;
+    const html = await get_html(url);
+    const elements = await get_elements(html, '.digest-card.is-new .digest-card__title a');
+
+    // Get Author and Abstarct
+    const baseUrl = 'https://www.nber.org';
+    const items = await Promise.all(
+        elements.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: `${baseUrl}${item.link}`,
+                });
+                const content = cheerio.load(detailResponse.data);
+                const authors = [];
+                content('.page-header__author-item a').each((index, element) => {
+                    const text = content(element).text();
+                    const link = content(element).attr('href');
+                    authors.push({ name: text, link });
+                });
+                item.authors = authors;
+                item.abstract = content('.page-header__intro-inner p').text();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'NBER Working Paper News',
+        link: url,
+        item: items,
+        description: 'National Bureau of Economic Research Working Papers -- News',
+    };
+};

lib/v2/nber/papers.js

@@ -0,0 +1,47 @@
+const { get_html, get_elements } = require('./utils');
+const got = require('@/utils/got');
+const cheerio = require('cheerio');
+
+module.exports = async (ctx) => {
+    let { perPage = '50' } = ctx.params;
+    perPage = parseInt(perPage);
+    // perPage has to be one of 20, 50, 100
+    if (perPage <= 35) {perPage = 20;}
+    if (35 < perPage && perPage <= 70) {perPage = 50;}
+    if (70 < perPage) {perPage = 100;}
+
+    // Get title and link
+    url = `https://www.nber.org/papers?page=1&perPage=${perPage}&sortBy=public_date`;
+    const html = await get_html(url);
+    const elements = await get_elements(html, '.digest-card .digest-card__title a');
+
+    // Get Author and Abstarct
+    const baseUrl = 'https://www.nber.org';
+    const items = await Promise.all(
+        elements.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const detailResponse = await got({
+                    method: 'get',
+                    url: `${baseUrl}${item.link}`,
+                });
+                const content = cheerio.load(detailResponse.data);
+                const authors = [];
+                content('.page-header__author-item a').each((index, element) => {
+                    const text = content(element).text();
+                    const link = content(element).attr('href');
+                    authors.push({ name: text, link });
+                });
+                item.authors = authors;
+                item.abstract = content('.page-header__intro-inner p').text();
+                return item;
+            })
+        )
+    );
+
+    ctx.state.data = {
+        title: 'NBER Working Paper',
+        link: url,
+        item: items,
+        description: `National Bureau of Economic Research Working Papers -- ${perPage} articles`,
+    };
+};

lib/v2/nber/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'nber.org': {
+        __name: 'NBER',
+        '.': [
+            {
+                title: 'New working paper',
+                docs: 'todo',
+                source: ['/news'],
+                target: '/nber/news',
+            },
+            {
+                title: 'All working paper',
+                docs: 'todo',
+                source: ['/papers'],
+                target: '/nber/papers',
+            },
+        ],
+    },
+};

lib/v2/nber/router.js

@@ -0,0 +1,4 @@
+module.exports = (router) => {
+    router.get('/papers/:perPage?', require('./papers'));
+    router.get('/news', require('./news'));
+};

lib/v2/nber/utils.js

@@ -0,0 +1,29 @@
+const cheerio = require('cheerio');
+
+async function get_html(url) {
+    const browser = await require('@/utils/puppeteer')();
+    const page = await browser.newPage();
+    await page.goto(url);
+    await page.waitForSelector('.promo-grid');
+    const html = await page.content();
+    await browser.close();
+
+    return html;
+    // Get title and link
+}
+
+async function get_elements(html, selector) {
+    const elements = [];
+    const $ = cheerio.load(html);
+    $(selector).each((index, element) => {
+        const text = $(element).text();
+        const href = $(element).attr('href');
+        elements.push({ title: text, link: href });
+    });
+    return elements;
+}
+
+module.exports = {
+    get_html,
+    get_elements,
+};