Add custom parser - 1pezeshk.com

package.json

@@ -127,7 +127,6 @@
     "postman-request": "^2.88.1-postman.31",
     "string-direction": "^0.1.2",
     "turndown": "^7.1.1",
-    "valid-url": "^1.0.9",
     "wuzzy": "^0.1.4",
     "yargs-parser": "^15.0.1"
   },

src/cleaners/lead-image-url.js

@@ -1,10 +1,7 @@
-import validUrl from 'valid-url';
-
 export default function clean(leadImageUrl) {
-  leadImageUrl = leadImageUrl.trim();
-  if (validUrl.isWebUri(leadImageUrl)) {
-    return leadImageUrl;
+  try {
+    return new URL(leadImageUrl.trim()).toString();
+  } catch {
+    return null;
   }
-
-  return null;
 }

src/cleaners/lead-image-url.test.js

@@ -4,7 +4,7 @@ import clean from './lead-image-url';
 
 describe('clean(leadImageUrl)', () => {
   it('returns the url if valid', () => {
-    const url = 'https://example.com';
+    const url = 'https://example.com/';
     assert.equal(clean(url), url);
   });
 

src/extractors/custom/index.js

@@ -142,3 +142,4 @@ export * from './postlight.com';
 export * from './www.investmentexecutive.com';
 export * from './www.cbc.ca';
 export * from './www.versants.com';
+export * from './www.1pezeshk.com';

src/extractors/custom/www.1pezeshk.com/index.js

@@ -0,0 +1,30 @@
+export const Www1pezeshkComExtractor = {
+  domain: 'www.1pezeshk.com',
+
+  title: {
+    selectors: [['meta[name="og:title"]', 'value'], 'h1.post-title'],
+  },
+  author: {
+    selectors: [['meta[name="author"]', 'value']],
+  },
+  date_published: {
+    selectors: [['meta[name="article:published_time"]', 'value']],
+  },
+  lead_image_url: {
+    selectors: [['.featured-area img', 'src']],
+  },
+  content: {
+    selectors: ['article > .entry-content'],
+
+    transforms: {
+      img: $node => {
+        $node.src = decodeURIComponent($node.src);
+      },
+    },
+
+    // Is there anything that is in the result that shouldn't be?
+    // The clean selectors will remove anything that matches from
+    // the result
+    clean: [],
+  },
+};

src/extractors/custom/www.1pezeshk.com/index.test.js

@@ -0,0 +1,81 @@
+import assert from 'assert';
+import URL from 'url';
+import cheerio from 'cheerio';
+
+import Parser from 'mercury';
+import getExtractor from 'extractors/get-extractor';
+import { excerptContent } from 'utils/text';
+
+const fs = require('fs');
+
+describe('Www1pezeshkComExtractor', () => {
+  describe('initial test case', () => {
+    let result;
+    let url;
+    beforeAll(() => {
+      url =
+        'https://www.1pezeshk.com/archives/2024/12/the-battle-of-dogs-utilizing-animals-in-warfare.html';
+      const html = fs.readFileSync(
+        './fixtures/www.1pezeshk.com/1733537734038.html'
+      );
+      result = Parser.parse(url, { html, fallback: false });
+    });
+
+    it('is selected properly', () => {
+      // This test should be passing by default.
+      // It sanity checks that the correct parser
+      // is being selected for URLs from this domain
+      const extractor = getExtractor(url);
+      assert.equal(extractor.domain, URL.parse(url).hostname);
+    });
+
+    it('returns the title', async () => {
+      // To pass this test, fill out the title selector
+      // in ./src/extractors/custom/www.1pezeshk.com/index.js.
+      const { title } = await result;
+
+      // Update these values with the expected values from
+      // the article.
+      assert.equal(title, `نبرد سگ‌ها: استفاده از حیوانات در میدان جنگ`);
+    });
+
+    it('returns the author', async () => {
+      const { author } = await result;
+
+      assert.equal(author, 'علیرضا مجیدی');
+    });
+
+    it('returns the date_published', async () => {
+      const { date_published } = await result;
+
+      assert.equal(date_published, `2024-12-06T21:30:19.000Z`);
+    });
+
+    it('returns the lead_image_url', async () => {
+      const { lead_image_url } = await result;
+
+      assert.equal(
+        lead_image_url,
+        'https://www.1pezeshk.com/wp-content/uploads/2024/12/12-09-1403-08-52-01-%D8%A8-%D8%B8.webp'
+      );
+    });
+
+    it('returns the content', async () => {
+      const { content } = await result;
+
+      const $ = cheerio.load(content || '');
+
+      const first13 = excerptContent(
+        $('*')
+          .first()
+          .text(),
+        13
+      );
+
+      assert.equal(
+        first13,
+        'در طول تاریخ، حیوانات همواره در میدان‌های نبرد در کنار انسان‌ها حضور داشته‌اند.'
+      );
+    });
+  });
+});

yarn.lock

@@ -10483,10 +10483,6 @@ uuid@^3.3.2:
   resolved "https://registry.yarnpkg.com/uuid/-/uuid-3.4.0.tgz#b23e4358afa8a202fe7a100af1f5f883f02007ee"
   integrity sha512-HjSDRw6gZE5JMggctHBcjVak08+KEVhSIiDzFnT9S9aegmp85S/bReBVTb4QTFaRNptJ9kuYaNhnbNEOkbKb/A==
 
-valid-url@^1.0.9:
-  version "1.0.9"
-  resolved "https://registry.yarnpkg.com/valid-url/-/valid-url-1.0.9.tgz#1c14479b40f1397a75782f115e4086447433a200"
-
 validate-npm-package-license@^3.0.1:
   version "3.0.1"
   resolved "https://registry.yarnpkg.com/validate-npm-package-license/-/validate-npm-package-license-3.0.1.tgz#2804babe712ad3379459acfbe24746ab2c303fbc"