From 3c5c0bdba94aa7596488459e4ec8012dd14c42e3 Mon Sep 17 00:00:00 2001 From: John Brayton Date: Wed, 10 Aug 2022 18:02:27 -0400 Subject: [PATCH] feat: Add a custom extractor for www.engadget.com. (#552) * feat:Add a custom extractor for ma.ttias.be. When parsing content for cron.weekly issues, such as the one at https://ma.ttias.be/cronweekly/issue-130/, Mercury Parser would remove headings and ordered lists that were part of the content. This resolves that as follows: * Remove "id" attributes from "h1" and "h2" elements. Those attributes would result in the elements having a low weight. * Since Mercury Parser demotes "h1" elements to "h2", demote "h2" elements to "h3". * Add class="entry-content-asset" to "ul" elements to avoid them being removed. * removed redundant comment. * feat: Add a custom extractor for engadget.com. Co-authored-by: John Holdun --- cli.js | 2 +- fixtures/www.engadget.com/1587759947079.html | 1566 +++++++++++++++++ src/extractors/custom/index.js | 1 + .../custom/www.engadget.com/index.js | 53 + .../custom/www.engadget.com/index.test.js | 120 ++ 5 files changed, 1741 insertions(+), 1 deletion(-) create mode 100644 fixtures/www.engadget.com/1587759947079.html create mode 100644 src/extractors/custom/www.engadget.com/index.js create mode 100644 src/extractors/custom/www.engadget.com/index.test.js diff --git a/cli.js b/cli.js index 5be1a7b94..10f5adec8 100755 --- a/cli.js +++ b/cli.js @@ -26,7 +26,7 @@ const { extendedListTypes, headers, addExtractor, - version, + version ) => { if (version) { console.log(package_info.version); diff --git a/fixtures/www.engadget.com/1587759947079.html b/fixtures/www.engadget.com/1587759947079.html new file mode 100644 index 000000000..e02780c87 --- /dev/null +++ b/fixtures/www.engadget.com/1587759947079.html @@ -0,0 +1,1566 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + Lyft extends free scooter rides for critical workers through May | Engadget + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+ + +
+
+
+
+
+
+ +
+
+ +

+ +

+ +
+
+ + + + +
+ +
+ +
+
+
+ + + +
+
+
+
+ + + + + + + +
+
+
+
+
+
+
+
+
+
+
+
+ + +
+ + + + +
+ + + + + + + + + + + + + +
+
+ +
+ + + + + + +
+
+ +
+

+ Latest in Gear +

+
+ + + +
+
+ + + +
+ + +
+ +
+ +
+
Image credit: REUTERS/Brendan McDermid
+
+
+ +
+ + +
+ +
+ +
+ +
+

Lyft extends free scooter rides for critical workers through May

+
Emergency, healthcare and transit workers won't need to pay.
+
+ +
+
+
+
+ + + +
+ +
Jon Fingas, @jonfingas
+ 1h ago +
+
+
+
+
+
+ + +Comments +
+
+ +33 +Shares +
+
+
+ + +
+ +
+
+
+ Share + +
+
+
+
+
+ +
+
+
+ Tweet + +
+
+
+
+
+ +
+
+
+ Share + +
+
+
+
+ + +
+
+
+
+
+
+ +
+ +
+ +
+ +
+ +
+ + + + + + + + + + +
+ +
+ +
+
+
+ + + +
+
+

Sponsored Links

+ + +
+ + +
+ + + + +
+ +
+
+ A Lyft Scooter is parked in Washington, U.S., March 29, 2019. REUTERS/Brendan McDermid +
+ + + REUTERS/Brendan McDermid + +
+
+
+
+
+ + +
+
+
+
+ + + +

Lyft isn’t waiting until the end of April to decide on the fate of its free scooter rides for critical workers during the COVID-19 pandemic. The ridesharing firm has extended its LyftUp Scooter Program through May, giving free rides to healthcare, emergency and transit workers in six cities another month. Employers will still have to enroll staff, but registered workers will continue get as many no-cost 30-minute rides as they need to commute without having to get close to others on public transportation or in rideshare cars.

The program is currently available in Austin, Denver, Los Angeles, San Diego, Santa Monica and the metro DC area. Lyft has also promised stricter cleaning protocols to make sure the scooters themselves are cleaner.

+ +
+
+
+
+ + + +
+ +
+
+
+
+ + + +

This isn’t the only scooter program available to vital workers. Uber, for instance, pledged 10 million free rides and deliveries for people on the front line of the COVID-19 fight. Lyft’s extension could still be important in the cities where it’s offered, though, and it suggests these programs will continue for as long as nurses, bus drivers and other key staff need to get to work safely.

+ +
+
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + +
+
+
+
+
In this article: + + Lyft, scooter, scooter sharing, Covid-19, coronavirus, transportation, internet, Services, ride sharing, Ridesharing, news, gear +
+
+
+
+ All products recommended by Engadget are selected by our editorial team, independent of our parent company. Some of our stories include affiliate links. If you buy something through one of these links, we may earn an affiliate commission. +
+
+
+
+ Comment +
+
+ + + + Comments +
+ +
+
+
+ Share +
+
+ 33 + Shares +
+
+
+
+ + +
+ +
+
+
+ Share + +
+
+
+
+
+ +
+
+
+ Tweet + +
+
+
+
+
+ +
+
+
+ Share + +
+
+
+
+ + +
+
+
+ +
+
+
+
+
+ +
+ + +
+ +
+ +
+ +
+ +
+ + + + + + +
+ + + + +
+
+
+ +
+

+ Popular on Engadget +

+ +
+ + + +
+
+
+
+
+
+
+ + +
+
+
+
+ Dyson won't build ventilators for the UK after all +
+
+
+

+ + Dyson won't build ventilators for the UK after all + +

+
+ View +
+
+ + +
+
+
+
+ Apple Watch Series 5 cellular models are $100 off +
+
+
+

+ + Apple Watch Series 5 cellular models are $100 off + +

+
+ View +
+
+ + +
+
+
+
+ SpaceX will make its Starlink satellites less of a blight on the night sky +
+
+
+

+ + SpaceX will make its Starlink satellites less of a blight on the night sky + +

+
+ View +
+
+ + +
+
+
+
+ Facebook takes on Zoom with 'Messenger Rooms' +
+
+
+

+ + Facebook takes on Zoom with 'Messenger Rooms' + +

+
+ View +
+
+ + +
+
+
+
+ Tesla competitor Polestar 2 will start at $59,900 in the US +
+
+
+

+ + Tesla competitor Polestar 2 will start at $59,900 in the US + +

+
+ View +
+
+
+
+
+
+ +
+
+
+
+ + +
+ +
+
+
+
+
+
+ + + +
+

+ From around the web

+
+ +
+
+
+
+ +
+
+
+ + +
+ + +
+ +
+ +
+ + + + +
+ + + + + + + +
+ + +
+ + +
+ +
+ Page 1Page 1ear iconeye iconFill 23text filevr
+ +
+
+ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/src/extractors/custom/index.js b/src/extractors/custom/index.js index 3597a3392..8813e52f5 100644 --- a/src/extractors/custom/index.js +++ b/src/extractors/custom/index.js @@ -138,3 +138,4 @@ export * from './ma.ttias.be'; export * from './pastebin.com'; export * from './www.abendblatt.de'; export * from './www.gruene.de'; +export * from './www.engadget.com'; diff --git a/src/extractors/custom/www.engadget.com/index.js b/src/extractors/custom/www.engadget.com/index.js new file mode 100644 index 000000000..aef64bd70 --- /dev/null +++ b/src/extractors/custom/www.engadget.com/index.js @@ -0,0 +1,53 @@ +export const WwwEngadgetComExtractor = { + domain: 'www.engadget.com', + + title: { + selectors: [['meta[name="og:title"]', 'value']], + }, + + author: { + selectors: ['a.th-meta[data-ylk*="subsec:author"]'], + }, + + // Engadget stories have publish dates, but the only representation of them on the page + // is in a format like "2h ago". There are also these tags with blank values: + // + date_published: { + selectors: [ + // enter selectors + ], + }, + + dek: { + selectors: ['div[class*="o-title_mark"] div'], + }, + + // Engadget stories do have lead images specified by an og:image meta tag, but selecting + // the value attribute of that tag fails. I believe the "ℑ" sequence of characters + // is triggering this inability to select the attribute value. + lead_image_url: { + selectors: [ + // enter selectors + ], + }, + + content: { + selectors: [ + [ + // Some figures will be inside div.article-text, but some header figures/images + // will not. + '#page_body figure:not(div.article-text figure)', + 'div.article-text', + ], + ], + + // Is there anything in the content you selected that needs transformed + // before it's consumable content? E.g., unusual lazy loaded images + transforms: {}, + + // Is there anything that is in the result that shouldn't be? + // The clean selectors will remove anything that matches from + // the result + clean: [], + }, +}; diff --git a/src/extractors/custom/www.engadget.com/index.test.js b/src/extractors/custom/www.engadget.com/index.test.js new file mode 100644 index 000000000..db691fe91 --- /dev/null +++ b/src/extractors/custom/www.engadget.com/index.test.js @@ -0,0 +1,120 @@ +import assert from 'assert'; +import URL from 'url'; +import cheerio from 'cheerio'; + +import Mercury from 'mercury'; +import getExtractor from 'extractors/get-extractor'; +import { excerptContent } from 'utils/text'; + +const fs = require('fs'); + +describe('WwwEngadgetComExtractor', () => { + describe('initial test case', () => { + let result; + let url; + beforeAll(() => { + url = + 'https://www.engadget.com/lyft-extends-free-scooter-rides-for-critical-workers-184144961.html'; + const html = fs.readFileSync( + './fixtures/www.engadget.com/1587759947079.html' + ); + result = Mercury.parse(url, { html, fallback: false }); + }); + + it('is selected properly', () => { + // This test should be passing by default. + // It sanity checks that the correct parser + // is being selected for URLs from this domain + const extractor = getExtractor(url); + assert.equal(extractor.domain, URL.parse(url).hostname); + }); + + it('returns the title', async () => { + // To pass this test, fill out the title selector + // in ./src/extractors/custom/www.engadget.com/index.js. + const { title } = await result; + + // Update these values with the expected values from + // the article. + assert.equal( + title, + `Lyft extends free scooter rides for critical workers through May` + ); + }); + + it('returns the author', async () => { + // To pass this test, fill out the author selector + // in ./src/extractors/custom/www.engadget.com/index.js. + const { author } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(author, 'Jon Fingas'); + }); + + it('returns the date_published', async () => { + // To pass this test, fill out the date_published selector + // in ./src/extractors/custom/www.engadget.com/index.js. + const { date_published } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(date_published, null); + }); + + it('returns the dek', async () => { + // To pass this test, fill out the dek selector + // in ./src/extractors/custom/www.engadget.com/index.js. + const { dek } = await result; + + // Update these values with the expected values from + // the article. + assert.equal( + dek, + "Emergency, healthcare and transit workers won't need to pay." + ); + }); + + it('returns the lead_image_url', async () => { + // To pass this test, fill out the lead_image_url selector + // in ./src/extractors/custom/www.engadget.com/index.js. + const { lead_image_url } = await result; + + // Update these values with the expected values from + // the article. + assert.equal(lead_image_url, null); + }); + + it('returns the content', async () => { + // To pass this test, fill out the content selector + // in ./src/extractors/custom/www.engadget.com/index.js. + // You may also want to make use of the clean and transform + // options. + const { content } = await result; + + const $ = cheerio.load(content || ''); + + const first13 = excerptContent( + $('*') + .first() + .text(), + 13 + ); + + // Update these values with the expected values from + // the article. + assert.equal( + first13, + 'REUTERS/Brendan McDermid Lyft isn’t waiting until the end of April to decide on' + ); + + // Ensure that the figure is present. + const figure = $('figure'); + assert.equal(figure.length, 1); + + // Ensure that there are 3 paragraphs + const p = $('p'); + assert.equal(p.length, 3); + }); + }); +});