Merge pull request #1031 from DIYgod/master

[pull] master from diygod:master

.github/workflows/docs-search-index.yml

@@ -6,6 +6,7 @@ on:
       - master
     paths:
       - '.github/workflows/docs-search-index.yml'
+      - 'scripts/docs-scraper/docs.rsshub.app.json'
       - 'website/**'
   workflow_dispatch: ~
   schedule:

lib/config.js

@@ -10,6 +10,7 @@ const calculateValue = () => {
     const email_config = {};
     const discuz_cookies = {};
     const medium_cookies = {};
+    const discourse_config = {};
 
     for (const name in envs) {
         if (name.startsWith('BILIBILI_COOKIE_')) {
@@ -27,6 +28,9 @@ const calculateValue = () => {
         } else if (name.startsWith('MEDIUM_COOKIE_')) {
             const username = name.slice(14).toLowerCase();
             medium_cookies[username] = envs[name];
+        } else if (name.startsWith('DISCOURSE_CONFIG_')) {
+            const id = name.slice('DISCOURSE_CONFIG_'.length);
+            discourse_config[id] = JSON.parse(envs[name]);
         }
     }
 
@@ -136,6 +140,9 @@ const calculateValue = () => {
         discord: {
             authorization: envs.DISCORD_AUTHORIZATION,
         },
+        discourse: {
+            config: discourse_config,
+        },
         discuz: {
             cookies: discuz_cookies,
         },
@@ -275,6 +282,9 @@ const calculateValue = () => {
         telegram: {
             token: envs.TELEGRAM_TOKEN,
         },
+        tophub: {
+            cookie: envs.TOPHUB_COOKIE,
+        },
         twitter: {
             consumer_key: envs.TWITTER_CONSUMER_KEY,
             consumer_secret: envs.TWITTER_CONSUMER_SECRET,

lib/radar.js

@@ -3,7 +3,8 @@ const fs = require('fs');
 const toSource = require('tosource');
 const { join } = require('path');
 
-const allowNamespace = ['ehentai', 'test'];
+// Namespaces that do not require radar.js
+const allowNamespace = ['discourse', 'ehentai', 'test'];
 // Check if a radar.js file is exist under each folder of dirname
 for (const dir of fs.readdirSync(dirname)) {
     const dirPath = join(dirname, dir);

lib/router.js

@@ -770,8 +770,8 @@ router.get('/blogs/wordpress/:domain/:https?', lazyloadRouteHandler('./routes/bl
 // 西祠胡同
 router.get('/xici/:id?', lazyloadRouteHandler('./routes/xici'));
 
-// 今日热榜
-router.get('/tophub/:id', lazyloadRouteHandler('./routes/tophub'));
+// 今日热榜 migrated to v2
+// router.get('/tophub/:id', lazyloadRouteHandler('./routes/tophub'));
 
 // 游戏时光
 router.get('/vgtime/news', lazyloadRouteHandler('./routes/vgtime/news.js'));

lib/v2/baidu/maintainer.js

@@ -1,7 +1,7 @@
 module.exports = {
     '/gushitong/index': ['CaoMeiYouRen'],
-    '/tieba/forum/:kw': ['u3u'],
-    '/tieba/forum/good/:kw/:cid?': ['u3u'],
+    '/tieba/forum/good/:kw/:cid?/:sortBy?': ['u3u'],
+    '/tieba/forum/:kw/:sortBy?': ['u3u'],
     '/tieba/post/:id': ['u3u'],
     '/tieba/post/lz/:id': ['u3u'],
     '/tieba/user/:uid': ['igxlin', 'nczitzk'],

lib/v2/baidu/router.js

@@ -1,7 +1,7 @@
 module.exports = (router) => {
     router.get('/gushitong/index', require('./gushitong'));
-    router.get('/tieba/forum/:kw', require('./tieba/forum'));
-    router.get('/tieba/forum/good/:kw/:cid?', require('./tieba/forum'));
+    router.get('/tieba/forum/good/:kw/:cid?/:sortBy?', require('./tieba/forum'));
+    router.get('/tieba/forum/:kw/:sortBy?', require('./tieba/forum'));
     router.get('/tieba/post/:id', require('./tieba/post'));
     router.get('/tieba/post/lz/:id', require('./tieba/post'));
     router.get('/tieba/user/:uid', require('./tieba/user'));

lib/v2/baidu/tieba/forum.js

@@ -6,7 +6,8 @@ const { art } = require('@/utils/render');
 const path = require('path');
 
 module.exports = async (ctx) => {
-    const { kw, cid } = ctx.params;
+    // sortBy: created, replied
+    const { kw, cid = '0', sortBy = 'created' } = ctx.params;
 
     // PC端：https://tieba.baidu.com/f?kw=${encodeURIComponent(kw)}
     // 移动端接口：https://tieba.baidu.com/mo/q/m?kw=${encodeURIComponent(kw)}&lp=5024&forum_recommend=1&lm=0&cid=0&has_url_param=1&pn=0&is_ajax=1
@@ -31,7 +32,7 @@ module.exports = async (ctx) => {
         .map((element) => {
             const item = $(element);
             const { id, author_name } = item.data('field');
-            const time = item.find('.threadlist_reply_date').text().trim();
+            const time = sortBy === 'created' ? item.find('.is_show_create_time').text().trim() : item.find('.threadlist_reply_date').text().trim();
             const title = item.find('a.j_th_tit').text().trim();
             const details = item.find('.threadlist_abs').text().trim();
             const medias = item
@@ -50,7 +51,7 @@ module.exports = async (ctx) => {
                     medias,
                     author_name,
                 }),
-                pubDate: timezone(parseDate(time, ['HH:mm', 'M-D'], true), +8),
+                pubDate: timezone(parseDate(time, ['HH:mm', 'M-D', 'YYYY-MM'], true), +8),
                 link: `https://tieba.baidu.com/p/${id}`,
             };
         });

lib/v2/discourse/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:configId/posts': ['dzx-dzx'],
+};

lib/v2/discourse/posts.js

@@ -0,0 +1,28 @@
+const config = require('@/config').value;
+const got = require('@/utils/got');
+const RSSParser = require('@/utils/rss-parser');
+
+module.exports = async (ctx) => {
+    if (!config.discourse.config[ctx.params.configId]) {
+        throw Error('Discourse RSS is disabled due to the lack of <a href="https://docs.rsshub.app/install">relevant config</a>');
+    }
+    const { link, key } = config.discourse.config[ctx.params.configId];
+
+    const feed = await RSSParser.parseString(
+        (
+            await got(`${link}/posts.rss`, {
+                headers: {
+                    'User-Api-Key': key,
+                },
+            })
+        ).data
+    );
+
+    feed.items = feed.items.map((e) => ({
+        description: e.content,
+        author: e.creator,
+        ...e,
+    }));
+
+    ctx.state.data = { item: feed.items, ...feed };
+};

lib/v2/discourse/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:configId/posts', require('./posts'));
+};

lib/v2/dlnews/category.js

@@ -0,0 +1,81 @@
+const cheerio = require('cheerio');
+const got = require('@/utils/got');
+const { getData, getList } = require('./utils');
+const { art } = require('@/utils/render');
+const path = require('path');
+const asyncPool = require('tiny-async-pool');
+
+const _website = 'dlnews';
+const topics = {
+    defi: 'DeFi',
+    fintech: 'Fintech/VC/Deals',
+    'llama-u': 'Llama U',
+    markets: 'Markets',
+    'people-culture': 'People & Culture',
+    regulation: 'Regulation',
+    snapshot: 'Snapshot',
+    web3: 'Web3',
+};
+const extractArticle = (ctx, item) =>
+    ctx.cache.tryGet(item.link, async () => {
+        const { data: response } = await got(item.link);
+        const $ = cheerio.load(response);
+        const scriptTagContent = $('script#fusion-metadata').text();
+        const jsonData = JSON.parse(scriptTagContent.match(/Fusion\.globalContent=({.*?});Fusion\.globalContentConfig/)[1]).content_elements;
+        const filteredData = [];
+        for (const v of jsonData) {
+            if (v.type === 'header' && v.content.includes('What we’re reading')) {
+                break;
+            } else if (v.type === 'custom_embed' && Boolean(v.embed.config.text)) {
+                filteredData.push({ type: v.type, data: v.embed.config.text });
+            } else if (v.type === 'text' && !v.content.includes('<b>NOW READ: </b>')) {
+                filteredData.push({ type: v.type, data: v.content });
+            } else if (v.type === 'header') {
+                filteredData.push({ type: v.type, data: v.content });
+            } else if (v.type === 'list') {
+                filteredData.push({ type: v.type, list_type: v.list_type, items: v.items });
+            } else if (v.type === 'image') {
+                filteredData.push({ type: v.type, src: v.url, alt: v.alt_text, caption: v.subtitle });
+            }
+        }
+        item.description = art(path.resolve(__dirname, 'templates/description.art'), filteredData);
+        return item;
+    });
+
+module.exports = async (ctx) => {
+    const { category } = ctx.params;
+    const baseUrl = 'https://www.dlnews.com';
+    const apiPath = '/pf/api/v3/content/fetch/articles-api';
+    let vertical;
+    if (category) {
+        vertical = category;
+    } else {
+        vertical = '';
+    }
+
+    const query = {
+        author: '',
+        date: 'now-1y/d',
+        offset: 0,
+        query: '',
+        size: 15,
+        sort: 'display_date:desc',
+        vertical,
+    };
+    const data = await getData(`${baseUrl}${apiPath}?query=${encodeURIComponent(JSON.stringify(query))}&_website=${_website}`);
+    const list = getList(data);
+    const items = [];
+    for await (const data of asyncPool(3, list, (item) => extractArticle(ctx, item))) {
+        items.push(data);
+    }
+
+    ctx.state.data = {
+        title: topics.hasOwnProperty(category) ? `${topics[category]} : DL News` : 'DL News',
+        link: baseUrl,
+        item: items,
+        description: topics.hasOwnProperty(category) ? `${topics[category]} : News on dlnews.com` : 'Latest News on dlnews.com',
+        logo: 'https://www.dlnews.com/pf/resources/favicon.ico?d=284',
+        icon: 'https://www.dlnews.com/pf/resources/favicon.ico?d=284',
+        language: 'en-us',
+    };
+};

lib/v2/dlnews/maintainer.js

@@ -0,0 +1,3 @@
+module.exports = {
+    '/:category?': ['Rjnishant530'],
+};

lib/v2/dlnews/radar.js

@@ -0,0 +1,19 @@
+module.exports = {
+    'dlnews.com': {
+        _name: 'DL NEWS',
+        '.': [
+            {
+                title: 'All Articles',
+                docs: 'https://docs.rsshub.app/routes/finance#dl-news',
+                source: ['/articles/'],
+                target: '/dlnews/',
+            },
+            {
+                title: 'Topic',
+                docs: 'https://docs.rsshub.app/routes/finance#dl-news',
+                source: ['/articles/:category'],
+                target: '/dlnews/:category',
+            },
+        ],
+    },
+};

lib/v2/dlnews/router.js

@@ -0,0 +1,3 @@
+module.exports = (router) => {
+    router.get('/:category?', require('./category'));
+};

lib/v2/dlnews/templates/description.art

@@ -0,0 +1,24 @@
+{{ each $data d }}
+    {{ if d.type == 'custom_embed' }}
+        <ul>
+            {{ each d.data.split('\n') line }}
+                <li>{{@  line }}</li>
+            {{ /each }}
+        </ul>
+    {{ else if d.type == 'header' }}
+        <h2>{{@ d.data }}</h2>
+    {{ else if d.type == 'list' }}
+        {{ if d.list_type == 'unordered' }}<ul>{{ else }}<ol>{{ /if }}
+            {{ each d.items item }}
+                <li>{{@ item.content }}</li>
+            {{ /each }}
+        {{ if d.list_type == 'unordered' }}</ul>{{ else }}</ol>{{ /if }}
+    {{ else if d.type == 'image' }}
+        <figure>
+            <img src="{{ d.src }}" alt="{{ d.alt }}">
+            <figcaption>{{@  d.caption }}</figcaption>
+        </figure>
+    {{ else if d.type == 'text' }}
+        <p>{{@ d.data }}</p>
+    {{ /if }}
+{{ /each }}

lib/v2/dlnews/utils.js

@@ -0,0 +1,22 @@
+const got = require('@/utils/got');
+const { parseDate } = require('@/utils/parse-date');
+
+const baseUrl = 'https://www.dlnews.com';
+const getData = async (url) => (await got.get(url).json()).content_elements;
+
+const getList = (data) =>
+    data.map((value) => {
+        const { _id, headlines, description, publish_date, website_url, taxonomy, credits, promo_items } = value;
+        return {
+            id: _id,
+            title: headlines.basic,
+            link: `${baseUrl}${website_url}`,
+            description: description.basic,
+            author: credits.by.map((v) => v.name).join(', '),
+            itunes_item_image: promo_items.basic.url,
+            pubDate: parseDate(publish_date),
+            category: taxonomy.sections.map((v) => v.name).join(', '),
+        };
+    });
+
+module.exports = { getData, getList };