Merge pull request #992 from DIYgod/master

[pull] master from diygod:master

lib/v2/2048/index.js

@@ -17,26 +17,28 @@ module.exports = async (ctx) => {
     });
 
     const $ = cheerio.load(response.data);
+    const currentHost = `https://${new URL(response.url).host}`; // redirected host
 
     $('#shortcut').remove();
     $('tr[onmouseover="this.className=\'tr3 t_two\'"]').remove();
 
     const list = $('#ajaxtable tbody .tr2')
         .last()
         .nextAll('.tr3')
-        .map((_, item) => {
+        .toArray()
+        .map((item) => {
             item = $(item).find('.subject');
 
             return {
                 title: item.text(),
-                link: `${rootUrl}/2048/${item.attr('href')}`,
+                link: `${currentHost}/2048/${item.attr('href')}`,
+                guid: `${rootUrl}/2048/${item.attr('href')}`,
             };
-        })
-        .get();
+        });
 
     const items = await Promise.all(
         list.map((item) =>
-            ctx.cache.tryGet(item.link, async () => {
+            ctx.cache.tryGet(item.guid, async () => {
                 const detailResponse = await got({
                     method: 'get',
                     url: item.link,

lib/v2/apnews/topics.js

@@ -1,45 +1,39 @@
 const got = require('@/utils/got');
 const cheerio = require('cheerio');
-const { parseDate } = require('@/utils/parse-date');
-const timezone = require('@/utils/timezone');
-const { art } = require('@/utils/render');
-const path = require('path');
 const HOME_PAGE = 'https://apnews.com';
 
 module.exports = async (ctx) => {
     const { topic = 'trending-news' } = ctx.params;
     const url = `${HOME_PAGE}/hub/${topic}`;
     const response = await got(url);
     const $ = cheerio.load(response.data);
-    const data = JSON.parse(
-        $('script')
-            .not('[src], [type]')
-            .text()
-            .match(/window\['titanium-state'\] = (.*)\nwindow\['titanium-cacheConfig'\]/)[1]
-    );
-    const meta = data.hub.data[`/${topic}`];
 
-    const items = meta.cards
-        .filter((c) => c.id.startsWith('urn:publicid:ap.org:'))
-        .map((item) => {
-            const description = cheerio.load(item.contents[0].storyHTML, null, false);
-            description('.ad-placeholder').remove();
-            return {
-                title: item.contents[0].headline,
-                description: art(path.join(__dirname, 'templates/description.art'), {
-                    media: item.contents[0].media,
-                    description: description.html(),
-                }),
-                link: `${HOME_PAGE}/article/${item.contents[0].canonicalUrl}-${item.contents[0].shortId}`,
-                author: item.contents[0].bylines ? item.contents[0].bylines.slice(3) : null,
-                pubDate: timezone(parseDate(item.contents[0].published), 0),
-                category: item.contents[0].tagObjs.map((tag) => tag.name),
-            };
-        });
+    const items = await Promise.all(
+        $('.PagePromo-content')
+            .get()
+            .map((e) => ({
+                title: $(e).find('span.PagePromoContentIcons-text').text(),
+                link: $(e).find('a').attr('href'),
+            }))
+            .map((item) =>
+                ctx.cache.tryGet(item.link, async () => {
+                    const { data: response } = await got(item.link);
+                    const $ = cheerio.load(response);
+                    $('div.Enhancement').remove();
+                    return Object.assign(item, {
+                        pubDate: new Date($("meta[property='article:published_time']").attr('content')),
+                        updated: new Date($("meta[property='article:modified_time']").attr('content')),
+                        description: $('div.RichTextStoryBody').html(),
+                        category: $("meta[property='article:section']").attr('content'),
+                        guid: $("meta[name='brightspot.contentId']").attr('content'),
+                    });
+                })
+            )
+    );
 
     ctx.state.data = {
-        title: meta.tagObjs[0].seoTitle,
-        description: meta.tagObjs[0].seoDescription,
+        title: $('title').text(),
+        description: $("meta[property='og:description']").text(),
         link: url,
         item: items,
         language: $('html').attr('lang'),

lib/v2/javdb/utils.js

@@ -7,15 +7,16 @@ const allowDomain = ['javdb.com', 'javdb36.com', 'javdb007.com'];
 module.exports = {
     ProcessItems: async (ctx, currentUrl, title) => {
         const domain = ctx.query.domain ?? 'javdb.com';
-        if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(new URL(`https://${domain}/`).hostname)) {
+        const url = new URL(currentUrl, `https://${domain}`);
+        if (!config.feature.allow_user_supply_unsafe_domain && !allowDomain.includes(url.hostname)) {
             ctx.throw(403, `This RSS is disabled unless 'ALLOW_USER_SUPPLY_UNSAFE_DOMAIN' is set to 'true'.`);
         }
 
         const rootUrl = `https://${domain}`;
 
         const response = await got({
             method: 'get',
-            url: `${rootUrl}${currentUrl}`,
+            url: url.href,
         });
 
         const $ = cheerio.load(response.data);
@@ -68,11 +69,11 @@ module.exports = {
         );
 
         const htmlTitle = $('title').text();
-        const subject = htmlTitle.indexOf('|') === -1 ? '' : htmlTitle.split('|')[0];
+        const subject = htmlTitle.includes('|') ? htmlTitle.split('|')[0] : '';
 
         return {
             title: subject === '' ? title : `${subject} - ${title}`,
-            link: `${rootUrl}${currentUrl}`,
+            link: url.href,
             item: items,
         };
     },

lib/v2/wikinews/index.js

@@ -1,43 +1,42 @@
 const got = require('@/utils/got');
-const { parseString } = require('xml2js');
 const cheerio = require('cheerio');
 const currentURL = 'https://zh.wikinews.org/wiki/Special:%E6%96%B0%E9%97%BB%E8%AE%A2%E9%98%85';
 const { parseDate } = require('@/utils/parse-date');
 
 module.exports = async (ctx) => {
     const resp = await got(currentURL);
-    const {
-        urlset: { url: urls },
-    } = await new Promise((resolve, reject) => {
-        parseString(resp.data, (err, result) => {
-            if (err) {
-                reject(err);
-                return;
-            }
-            resolve(result);
+    const $ = cheerio.load(resp.data);
+    const urls = $('url')
+        .toArray()
+        .map((item) => {
+            item = $(item);
+            return {
+                title: item.find('news\\:title').text(),
+                pubDate: parseDate(item.find('news\\:publication_date').text()),
+                category: item
+                    .find('news\\:keywords')
+                    .text()
+                    .split(',')
+                    .map((item) => item.trim()),
+                link: item.find('loc').text(),
+            };
         });
-    });
+
+    const items = await Promise.all(
+        urls.map((item) =>
+            ctx.cache.tryGet(item.link, async () => {
+                const resp = await got(item.link);
+                const $ = cheerio.load(resp.data);
+                item.description = $('#bodyContent').html();
+
+                return item;
+            })
+        )
+    );
 
     ctx.state.data = {
         title: '最新新闻 - 维基新闻',
         link: currentURL,
-        item: await Promise.all(
-            urls.map(async (url) => {
-                const loc = url.loc[0];
-                const news = url['news:news'][0];
-                const description = await ctx.cache.tryGet(loc, async () => {
-                    const resp = await got(loc);
-                    const $ = cheerio.load(resp.data);
-                    return $('#bodyContent').html();
-                });
-                return {
-                    title: news['news:title'][0],
-                    pubDate: parseDate(news['news:publication_date'][0]),
-                    category: news['news:keywords'][0].split(',').map((item) => item.trim()),
-                    link: loc,
-                    description,
-                };
-            })
-        ),
+        item: items,
     };
 };

package.json

@@ -137,8 +137,7 @@
     "tiny-async-pool": "2.1.0",
     "tough-cookie": "4.1.3",
     "twitter-api-v2": "1.15.0",
-    "winston": "3.9.0",
-    "xml2js": "0.5.0"
+    "winston": "3.9.0"
   },
   "devDependencies": {
     "@microsoft/eslint-formatter-sarif": "3.0.0",