Merge branch 'master' of https://github.com/unkn0w7n/calibre

recipes/afr.recipe

@@ -1,6 +1,10 @@
-from calibre.ptempfile import PersistentTemporaryFile
+#!/usr/bin/env python
 from calibre.web.feeds.news import BasicNewsRecipe
+from datetime import date
 
+def absurl(url):
+    if url.startswith('/'):
+        return 'https://www.afr.com' + url
 
 class afr(BasicNewsRecipe):
     title = 'Australian Financial Review'
@@ -40,50 +44,39 @@ class afr(BasicNewsRecipe):
         [data-testid="AuthorNames"], [data-testid="ArticleTimestamp"] {font-size:small;}
     '''
 
-    ignore_duplicate_articles = {'title'}
+    ignore_duplicate_articles = {'title', 'url'}
     resolve_internal_links  = True
     remove_empty_feeds = True
 
-    articles_are_obfuscated = True
-
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        try:
-            br.open(url)
-        except Exception as e:
-            url = e.hdrs.get('location')
-        soup = self.index_to_soup(url)
-        link = soup.find('a', href=True)
-        skip_sections =[ # add sections you want to skip
-            '/video/', '/videos/', '/media/', 'podcast-'
-        ]
-        if any(x in link['href'] for x in skip_sections):
-            self.log('Aborting Article ', link['href'])
-            self.abort_article('skipping video links')
-
-        self.log('Downloading ', link['href'])
-        html = br.open(link['href']).read()
-        pt = PersistentTemporaryFile('.html')
-        pt.write(html)
-        pt.close()
-        return pt.name
-
     def preprocess_html(self, soup):
         for img in soup.findAll('img', attrs={'data-src':True}):
             img['src'] = img['data-src']
         for fig in soup.findAll('figcaption'):
             fig['id'] = 'img-cap'
         return soup
 
-    feeds = []
-
-    sections = [
-        'companies', 'market', 'politics', 'policy', 'world', 'wealth', 'street-talk',
-        'chaticleer', 'rear-window', 'life-and-luxury', 'technology', 'property',
-        'work-and-careers',
-    ]
-
-    for sec in sections:
-        a = 'https://news.google.com/rss/search?q=when:27h+allinurl:https%3A%2F%2Fwww.afr.com{}&hl=en-AU&gl=AU&ceid=AU:en'
-        feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
-    feeds.append(('Others', a.format('')))
+    def parse_index(self):
+        index = 'https://www.afr.com/'
+        sections = [
+            'companies', 'market', 'politics', 'policy', 'world', 'wealth', 'street-talk',
+            'chaticleer', 'rear-window', 'life-and-luxury', 'technology', 'property',
+            'work-and-careers',
+        ]
+        feeds = []
+        soup = self.index_to_soup(index)
+        for sec in sections:
+            section = sec.capitalize()
+            self.log(section)
+            articles = []
+            for a in soup.findAll('a', attrs={'href':lambda x: x and x.startswith('/' + sec + '/')}):
+                url = absurl(a['href'].split('?')[0])
+                if url in {index + sec + '/', index + sec}:
+                    continue
+                if date.today().strftime('%Y') not in url:
+                    continue
+                title = self.tag_to_string(a)
+                self.log('\t', title, '\n\t\t', url)
+                articles.append({'title': title, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds

recipes/bar_and_bench.recipe

@@ -1,4 +1,4 @@
-from calibre.ptempfile import PersistentTemporaryFile
+#!/usr/bin/env python
 from calibre.web.feeds.news import BasicNewsRecipe, prefixed_classes
 
 
@@ -38,37 +38,24 @@ class bar(BasicNewsRecipe):
     resolve_internal_links  = True
     remove_empty_feeds = True
 
-    articles_are_obfuscated = True
-
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        try:
-            br.open(url)
-        except Exception as e:
-            url = e.hdrs.get('location')
-        soup = self.index_to_soup(url)
-        link = soup.find('a', href=True)
-        skip_sections =[ # add sections you want to skip
-            '/video/', '/videos/', '/media/', 'podcast-'
+    def parse_index(self):
+        index = 'https://www.barandbench.com/'
+        sections = [
+            'news', 'columns', 'interviews', 'law-firms', 'apprentice-lawyer', 'legal-jobs'
         ]
-        if any(x in link['href'] for x in skip_sections):
-            self.log('Aborting Article ', link['href'])
-            self.abort_article('skipping video links')
-
-        self.log('Downloading ', link['href'])
-        html = br.open(link['href']).read()
-        pt = PersistentTemporaryFile('.html')
-        pt.write(html)
-        pt.close()
-        return pt.name
-
-    feeds = []
-
-    sections = [
-        'news', 'columns', 'interviews', 'law-firms', 'apprentice-lawyer', 'legal-jobs'
-    ]
-
-    for sec in sections:
-        a = 'https://news.google.com/rss/search?q=when:27h+allinurl:barandbench.com{}&hl=en-IN&gl=IN&ceid=IN:en'
-        feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
-    feeds.append(('Others', a.format('')))
+        feeds = []
+        soup = self.index_to_soup(index)
+        for sec in sections:
+            section = sec.capitalize()
+            self.log(section)
+            articles = []
+            for a in soup.findAll('a', attrs={'href':lambda x: x and x.startswith(index + sec + '/')}):
+                url = a['href'].split('?')[0]
+                if url in {index + sec + '/', index + sec}:
+                    continue
+                title = self.tag_to_string(a)
+                self.log('\t', title, '\n\t\t', url)
+                articles.append({'title': title, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds

recipes/deccan_herald.recipe

@@ -1,7 +1,10 @@
-from calibre.ptempfile import PersistentTemporaryFile
 from calibre.web.feeds.news import BasicNewsRecipe, classes
 
 
+def absurl(url):
+    if url.startswith('/'):
+        return 'https://www.deccanherald.com' + url
+
 class herald(BasicNewsRecipe):
     title = 'Deccan Herald'
     __author__ = 'unkn0wn'
@@ -11,56 +14,40 @@ class herald(BasicNewsRecipe):
     remove_attributes = ['height', 'width', 'style']
     ignore_duplicate_articles = {'url', 'title'}
     encoding = 'utf-8'
-
-    articles_are_obfuscated = True
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        try:
-            br.open(url)
-        except Exception as e:
-            url = e.hdrs.get('location')
-        soup = self.index_to_soup(url)
-        link = soup.find('a', href=True)
-        skip_sections =[ # add sections you want to skip
-            '/video/', '/bengaluru-crime/', '/metrolife/',
-            '/karnataka-districts/', '/brandspot/', '/entertainment/',
-        ]
-        if any(x in link['href'] for x in skip_sections):
-            self.log('Aborting Article ', link['href'])
-            self.abort_article('skipping section')
-
-        self.log('Downloading ', link['href'])
-        html = br.open(link['href']).read()
-        pt = PersistentTemporaryFile('.html')
-        pt.write(html)
-        pt.close()
-        return pt.name
-
     keep_only_tags = [
         classes('article-title article-author__name'),
         dict(name='div', attrs={'id':'main-content'})
 
     ]
-    
+
     remove_tags = [
         classes(
             'storyShare social-media-icons in_article_video static_text'
             ' nl-optin-mobile dk_only article-banner-adver-wrapper wb_holder'
             ' field-name-field-tags section-full strip--business'
         )
     ]
-
-    feeds = [
-        ('Nation', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fnational%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('Karnataka', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fstate%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('Opinion', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fopinion%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('City', 
-            'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fcity%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('Business', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fbusiness%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('World', 
-            'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Finternational%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('Sports',
-            'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com%2Fsports%2F&hl=en-IN&gl=IN&ceid=IN:en'),
-        ('Others', 'https://news.google.com/rss/search?q=when:27h+allinurl:deccanherald.com&hl=en-IN&gl=IN&ceid=IN:en'),
-    ]
+
+    def parse_index(self):
+        index = 'https://www.deccanherald.com/'
+        sections = [
+            'india', 'world', 'elections', 'opinion', 'specials', 'business', 'sports'
+        ]
+        feeds = []
+
+        for sec in sections:
+            soup = self.index_to_soup(index + sec)
+            section = sec.capitalize()
+            self.log(section)
+            articles = []
+            for a in soup.findAll('a', attrs={'href':lambda x: x and x.startswith('/' + sec + '/')}):
+                url = absurl(a['href'].split('?')[0])
+                if url in {index + sec + '/', index + sec}:
+                    continue
+                title = self.tag_to_string(a)
+                self.log('\t', title, '\n\t\t', url)
+                articles.append({'title': title, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds

recipes/firstpost.recipe

@@ -1,9 +1,6 @@
-from calibre.ptempfile import PersistentTemporaryFile
+#!/usr/bin/env python
 from calibre.web.feeds.news import BasicNewsRecipe, classes
 
-# Firstpost feeds mix sections into other feeds, like explainers end up in opinion feed and opinions end up in India feed.
-# change google_feeds to True to fetch right sections.
-google_feeds = False
 
 class firstpost(BasicNewsRecipe):
     title = 'Firstpost'
@@ -43,35 +40,11 @@ class firstpost(BasicNewsRecipe):
         'world', 'web-stories', 'tech', 'artandculture', 'health', 'health-supplement',
         # 'photos', 'entertainment', 'living', 'education', 'sports', 'firstcricket',
     ]
-    if not google_feeds:
-        oldest_article = 1.2 # days
-        for sec in sections:
-            a = 'https://www.firstpost.com/rss/{}.xml'
-            feeds.append((sec.capitalize(), a.format(sec)))
-    else:
-        articles_are_obfuscated = True
 
-        def get_obfuscated_article(self, url):
-            br = self.get_browser()
-            soup = self.index_to_soup(url)
-            link = soup.find('a', href=True)
-            skip_sections =[ # add sections you want to skip
-                '/video/', '/videos/', '/media/', '/vantage/'
-            ]
-            if any(x in link['href'] for x in skip_sections):
-                self.log('Aborting Article ', link['href'])
-                self.abort_article('skipping video links')
-            self.log('Downloading ', link['href'])
-            html = br.open(link['href']).read()
-            pt = PersistentTemporaryFile('.html')
-            pt.write(html)
-            pt.close()
-            return pt.name
-
-        for sec in sections:
-            a = 'https://news.google.com/rss/search?q=when:27h+allinurl:firstpost.com{}&hl=en-IN&gl=IN&ceid=IN:en'
-            feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
-        # feeds.append(('Others', a.format('')))
+    oldest_article = 1.2 # days
+    for sec in sections:
+        a = 'https://www.firstpost.com/rss/{}.xml'
+        feeds.append((sec.capitalize(), a.format(sec)))
 
     def preprocess_html(self, soup):
         if h2 := soup.find('h2', attrs={'class':'category-name'}):

recipes/hamilton_spectator.recipe

@@ -1,10 +1,13 @@
-from calibre.ptempfile import PersistentTemporaryFile
+#!/usr/bin/env python
 from calibre.web.feeds.news import BasicNewsRecipe, classes
 
 '''
 Hamilton Spectator Calibre Recipe
 '''
 
+def absurl(url):
+    if url.startswith('/'):
+        return 'https://www.thespec.com' + url
 
 class HamiltonSpectator(BasicNewsRecipe):
     title = u'Hamilton Spectator'
@@ -21,8 +24,7 @@ class HamiltonSpectator(BasicNewsRecipe):
     remove_attributes = ['style', 'height', 'width']
     masthead_url = 'https://bloximages.chicago2.vip.townnews.com/thespec.com/content/tncms/custom/image/c0094646-1108-11ee-8af0-b3954ce40e5e.png'
 
-    ignore_duplicate_articles = {'title'}
-    articles_are_obfuscated = True
+    ignore_duplicate_articles = {'title', 'url'}
 
     extra_css = '''
         .caption { font-size:small; text-align:center; }
@@ -52,35 +54,26 @@ class HamiltonSpectator(BasicNewsRecipe):
                     img['src'] = x.split()[0]
         return soup
 
-    def get_obfuscated_article(self, url):
-        br = self.get_browser()
-        try:
-            br.open(url)
-        except Exception as e:
-            url = e.hdrs.get('location')
-        soup = self.index_to_soup(url)
-        link = soup.find('a', href=True)
-        skip_sections =[ # add sections you want to skip
-            '/video/', '/videos/', '/media/', 'podcast'
+    def parse_index(self):
+        index = 'https://www.thespec.com/'
+        sections = [
+            'news', 'politics', 'opinion', 'business', 'sports', 'life', 'entertainment'
         ]
-        if any(x in link['href'] for x in skip_sections):
-            self.log('Aborting Article ', link['href'])
-            self.abort_article('skipping video links')
-
-        self.log('Downloading ', link['href'])
-        html = br.open(link['href']).read()
-        pt = PersistentTemporaryFile('.html')
-        pt.write(html)
-        pt.close()
-        return pt.name
-
-    feeds = []
-
-    sections = [
-        'news', 'politics', 'opinion', 'business', 'sports', 'life', 'entertainment'
-    ]
-
-    for sec in sections:
-        a = 'https://news.google.com/rss/search?q=when:27h+allinurl:thespec.com{}&hl=en-CA&gl=IN&ceid=CA:en'
-        feeds.append((sec.capitalize(), a.format('%2F' + sec + '%2F')))
-    feeds.append(('Others', a.format('')))
+        feeds = []
+        soup = self.index_to_soup(index)
+        for sec in sections:
+            section = sec.capitalize()
+            self.log(section)
+            articles = []
+            for a in soup.findAll('a', attrs={'href':lambda x: x and x.startswith('/' + sec + '/')}):
+                url = absurl(a['href'].split('#')[0])
+                if url in {index + sec + '/', index + sec}:
+                    continue
+                if not url.endswith('.html'):
+                    continue
+                title = self.tag_to_string(a)
+                self.log('\t', title, '\n\t\t', url)
+                articles.append({'title': title, 'url': url})
+            if articles:
+                feeds.append((section, articles))
+        return feeds